1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.api;
18
19 import static com.google.common.base.MoreObjects.firstNonNull;
20 import static com.google.common.base.Strings.isNullOrEmpty;
21 import static com.google.common.collect.ImmutableList.toImmutableList;
22 import static com.linecorp.centraldogma.common.EntryType.DIRECTORY;
23 import static com.linecorp.centraldogma.internal.Util.isValidDirPath;
24 import static com.linecorp.centraldogma.internal.Util.isValidFilePath;
25 import static com.linecorp.centraldogma.server.internal.api.DtoConverter.convert;
26 import static com.linecorp.centraldogma.server.internal.api.HttpApiUtil.returnOrThrow;
27 import static com.linecorp.centraldogma.server.internal.api.RepositoryServiceV1.increaseCounterIfOldRevisionUsed;
28 import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.isMetaFile;
29 import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.isMirrorOrCredentialFile;
30 import static java.util.Objects.requireNonNull;
31
32 import java.util.Collection;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Objects;
36 import java.util.Optional;
37 import java.util.concurrent.CancellationException;
38 import java.util.concurrent.CompletableFuture;
39 import java.util.function.Function;
40
41 import javax.annotation.Nullable;
42
43 import com.fasterxml.jackson.databind.JsonNode;
44 import com.google.common.base.Throwables;
45 import com.google.common.collect.ImmutableList;
46 import com.google.common.collect.Iterables;
47 import com.google.common.collect.Streams;
48
49 import com.linecorp.armeria.common.HttpHeaderNames;
50 import com.linecorp.armeria.common.HttpResponse;
51 import com.linecorp.armeria.common.HttpStatus;
52 import com.linecorp.armeria.common.util.Exceptions;
53 import com.linecorp.armeria.server.ServiceRequestContext;
54 import com.linecorp.armeria.server.annotation.ConsumesJson;
55 import com.linecorp.armeria.server.annotation.Default;
56 import com.linecorp.armeria.server.annotation.Get;
57 import com.linecorp.armeria.server.annotation.Param;
58 import com.linecorp.armeria.server.annotation.Post;
59 import com.linecorp.armeria.server.annotation.ProducesJson;
60 import com.linecorp.armeria.server.annotation.RequestConverter;
61 import com.linecorp.centraldogma.common.Author;
62 import com.linecorp.centraldogma.common.Change;
63 import com.linecorp.centraldogma.common.Entry;
64 import com.linecorp.centraldogma.common.InvalidPushException;
65 import com.linecorp.centraldogma.common.Markup;
66 import com.linecorp.centraldogma.common.MergeQuery;
67 import com.linecorp.centraldogma.common.Query;
68 import com.linecorp.centraldogma.common.RepositoryRole;
69 import com.linecorp.centraldogma.common.Revision;
70 import com.linecorp.centraldogma.common.RevisionRange;
71 import com.linecorp.centraldogma.common.ShuttingDownException;
72 import com.linecorp.centraldogma.internal.api.v1.ChangeDto;
73 import com.linecorp.centraldogma.internal.api.v1.CommitMessageDto;
74 import com.linecorp.centraldogma.internal.api.v1.EntryDto;
75 import com.linecorp.centraldogma.internal.api.v1.MergedEntryDto;
76 import com.linecorp.centraldogma.internal.api.v1.PushResultDto;
77 import com.linecorp.centraldogma.internal.api.v1.WatchResultDto;
78 import com.linecorp.centraldogma.server.command.Command;
79 import com.linecorp.centraldogma.server.command.CommandExecutor;
80 import com.linecorp.centraldogma.server.command.CommitResult;
81 import com.linecorp.centraldogma.server.internal.api.auth.RequiresRepositoryRole;
82 import com.linecorp.centraldogma.server.internal.api.converter.ChangesRequestConverter;
83 import com.linecorp.centraldogma.server.internal.api.converter.CommitMessageRequestConverter;
84 import com.linecorp.centraldogma.server.internal.api.converter.MergeQueryRequestConverter;
85 import com.linecorp.centraldogma.server.internal.api.converter.QueryRequestConverter;
86 import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter;
87 import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter.WatchRequest;
88 import com.linecorp.centraldogma.server.storage.project.Project;
89 import com.linecorp.centraldogma.server.storage.repository.FindOption;
90 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
91 import com.linecorp.centraldogma.server.storage.repository.Repository;
92
93 import io.micrometer.core.instrument.MeterRegistry;
94 import io.netty.util.AttributeKey;
95
96
97
98
99 @ProducesJson
100 @RequiresRepositoryRole(RepositoryRole.READ)
101 @RequestConverter(CommitMessageRequestConverter.class)
102 public class ContentServiceV1 extends AbstractService {
103
104 static final AttributeKey<Boolean> IS_WATCH_REQUEST =
105 AttributeKey.valueOf(ContentServiceV1.class, "IS_WATCH_REQUEST");
106
107 private static final String MIRROR_LOCAL_REPO = "localRepo";
108
109 private final WatchService watchService;
110 private final MeterRegistry meterRegistry;
111
112 public ContentServiceV1(CommandExecutor executor, WatchService watchService, MeterRegistry meterRegistry) {
113 super(executor);
114 this.watchService = requireNonNull(watchService, "watchService");
115 this.meterRegistry = requireNonNull(meterRegistry, "meterRegistry");
116 }
117
118
119
120
121
122
123 @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/list(?<path>(|/.*))$")
124 public CompletableFuture<List<EntryDto<?>>> listFiles(ServiceRequestContext ctx,
125 @Param String path,
126 @Param @Default("-1") String revision,
127 Repository repository) {
128 final String normalizedPath = normalizePath(path);
129 final Revision normalizedRev = repository.normalizeNow(new Revision(revision));
130 increaseCounterIfOldRevisionUsed(ctx, repository, normalizedRev);
131 final CompletableFuture<List<EntryDto<?>>> future = new CompletableFuture<>();
132 listFiles(repository, normalizedPath, normalizedRev, false, future);
133 return future;
134 }
135
136 private static void listFiles(Repository repository, String pathPattern, Revision normalizedRev,
137 boolean withContent, CompletableFuture<List<EntryDto<?>>> result) {
138 final Map<FindOption<?>, ?> options = withContent ? FindOptions.FIND_ALL_WITH_CONTENT
139 : FindOptions.FIND_ALL_WITHOUT_CONTENT;
140
141 repository.find(normalizedRev, pathPattern, options).handle((entries, thrown) -> {
142 if (thrown != null) {
143 result.completeExceptionally(thrown);
144 return null;
145 }
146
147
148
149 if (isValidFilePath(pathPattern) && entries.size() == 1 &&
150 entries.values().iterator().next().type() == DIRECTORY) {
151 listFiles(repository, pathPattern + "/*", normalizedRev, withContent, result);
152 } else {
153 result.complete(entries.values().stream()
154 .map(entry -> convert(repository, normalizedRev, entry, withContent))
155 .collect(toImmutableList()));
156 }
157 return null;
158 });
159 }
160
161
162
163
164
165
166
167
168
169 private static String normalizePath(String path) {
170 if (path == null || path.isEmpty() || "/".equals(path)) {
171 return "/*";
172 }
173 if (isValidFilePath(path)) {
174 return path;
175 }
176 if (isValidDirPath(path)) {
177 if (path.endsWith("/")) {
178 return path + '*';
179 } else {
180 return path + "/*";
181 }
182 }
183 return path;
184 }
185
186
187
188
189
190
191 @Post("/projects/{projectName}/repos/{repoName}/contents")
192 @ConsumesJson
193 @RequiresRepositoryRole(RepositoryRole.WRITE)
194 public CompletableFuture<PushResultDto> push(
195 ServiceRequestContext ctx,
196 @Param @Default("-1") String revision,
197 Repository repository,
198 Author author,
199 CommitMessageDto commitMessage,
200 @RequestConverter(ChangesRequestConverter.class) Iterable<Change<?>> changes) {
201 checkMetaRepoPush(repository.name(), changes);
202 meterRegistry.counter("commits.push",
203 "project", repository.parent().name(),
204 "repository", repository.name())
205 .increment();
206
207 final long commitTimeMillis = System.currentTimeMillis();
208 return push(commitTimeMillis, author, repository, new Revision(revision), commitMessage, changes)
209 .toCompletableFuture()
210 .thenApply(rrev -> convert(rrev, commitTimeMillis));
211 }
212
213 private CompletableFuture<Revision> push(long commitTimeMills, Author author, Repository repository,
214 Revision revision, CommitMessageDto commitMessage,
215 Iterable<Change<?>> changes) {
216 final String summary = commitMessage.summary();
217 final String detail = commitMessage.detail();
218 final Markup markup = commitMessage.markup();
219
220 return execute(Command.push(
221 commitTimeMills, author, repository.parent().name(), repository.name(),
222 revision, summary, detail, markup, changes)).thenApply(CommitResult::revision);
223 }
224
225
226
227
228
229
230 @Post("/projects/{projectName}/repos/{repoName}/preview")
231 @ConsumesJson
232 public CompletableFuture<Iterable<ChangeDto<?>>> preview(
233 ServiceRequestContext ctx,
234 @Param @Default("-1") String revision,
235 Repository repository,
236 @RequestConverter(ChangesRequestConverter.class) Iterable<Change<?>> changes) {
237 final Revision baseRevision = new Revision(revision);
238 increaseCounterIfOldRevisionUsed(ctx, repository, baseRevision);
239 final CompletableFuture<Map<String, Change<?>>> changesFuture =
240 repository.previewDiff(baseRevision, changes);
241
242 return changesFuture.thenApply(previewDiffs -> previewDiffs.values().stream()
243 .map(DtoConverter::convert)
244 .collect(toImmutableList()));
245 }
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260 @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/contents(?<path>(|/.*))$")
261 public CompletableFuture<?> getFiles(
262 ServiceRequestContext ctx,
263 @Param String path, @Param @Default("-1") String revision,
264 Repository repository,
265 @RequestConverter(WatchRequestConverter.class) @Nullable WatchRequest watchRequest,
266 @RequestConverter(QueryRequestConverter.class) @Nullable Query<?> query) {
267 increaseCounterIfOldRevisionUsed(ctx, repository, new Revision(revision));
268 final String normalizedPath = normalizePath(path);
269
270
271 if (watchRequest != null) {
272 ctx.setAttr(IS_WATCH_REQUEST, true);
273 final Revision lastKnownRevision = watchRequest.lastKnownRevision();
274 final long timeOutMillis = watchRequest.timeoutMillis();
275 final boolean errorOnEntryNotFound = watchRequest.notifyEntryNotFound();
276 if (query != null) {
277 return watchFile(ctx, repository, lastKnownRevision, query, timeOutMillis,
278 errorOnEntryNotFound);
279 }
280
281 return watchRepository(ctx, repository, lastKnownRevision, normalizedPath,
282 timeOutMillis, errorOnEntryNotFound);
283 }
284
285 final Revision normalizedRev = repository.normalizeNow(new Revision(revision));
286 if (query != null) {
287
288 return repository.get(normalizedRev, query)
289 .handle(returnOrThrow((Entry<?> result) -> convert(repository, normalizedRev,
290 result, true)));
291 }
292
293
294 final CompletableFuture<List<EntryDto<?>>> future = new CompletableFuture<>();
295 listFiles(repository, normalizedPath, normalizedRev, true, future);
296 return future;
297 }
298
299 private CompletableFuture<?> watchFile(ServiceRequestContext ctx,
300 Repository repository, Revision lastKnownRevision,
301 Query<?> query, long timeOutMillis, boolean errorOnEntryNotFound) {
302 final CompletableFuture<? extends Entry<?>> future = watchService.watchFile(
303 repository, lastKnownRevision, query, timeOutMillis, errorOnEntryNotFound);
304
305 if (!future.isDone()) {
306 ctx.log().whenComplete().thenRun(() -> future.cancel(false));
307 }
308
309 return future.thenApply(entry -> {
310 final Revision revision = entry.revision();
311 final EntryDto<?> entryDto = convert(repository, revision, entry, true);
312 return (Object) new WatchResultDto(revision, entryDto);
313 }).exceptionally(ContentServiceV1::handleWatchFailure);
314 }
315
316 private CompletableFuture<?> watchRepository(ServiceRequestContext ctx,
317 Repository repository, Revision lastKnownRevision,
318 String pathPattern, long timeOutMillis,
319 boolean errorOnEntryNotFound) {
320 final CompletableFuture<Revision> future =
321 watchService.watchRepository(repository, lastKnownRevision, pathPattern,
322 timeOutMillis, errorOnEntryNotFound);
323
324 if (!future.isDone()) {
325 ctx.log().whenComplete().thenRun(() -> future.cancel(false));
326 }
327
328 return future.thenApply(revision -> (Object) new WatchResultDto(revision, null))
329 .exceptionally(ContentServiceV1::handleWatchFailure);
330 }
331
332 private static Object handleWatchFailure(Throwable thrown) {
333 final Throwable rootCause = Throwables.getRootCause(thrown);
334 if (rootCause instanceof CancellationException || rootCause instanceof ShuttingDownException) {
335
336 return HttpResponse.of(HttpStatus.NOT_MODIFIED);
337 }
338 return Exceptions.throwUnsafely(thrown);
339 }
340
341
342
343
344
345
346
347
348
349 @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/commits(?<revision>(|/.*))$")
350 public CompletableFuture<?> listCommits(ServiceRequestContext ctx,
351 @Param String revision,
352 @Param @Default("/**") String path,
353 @Param @Nullable String to,
354 @Param @Nullable Integer maxCommits,
355 Repository repository) {
356 final Revision fromRevision;
357 final Revision toRevision;
358
359
360
361
362
363 if (isNullOrEmpty(revision) || "/".equalsIgnoreCase(revision)) {
364 fromRevision = Revision.HEAD;
365 toRevision = to != null ? new Revision(to) : Revision.INIT;
366 } else {
367 fromRevision = new Revision(revision.substring(1));
368 toRevision = to != null ? new Revision(to) : fromRevision;
369 }
370
371 final RevisionRange range = repository.normalizeNow(fromRevision, toRevision).toDescending();
372
373 increaseCounterIfOldRevisionUsed(ctx, repository, range.from());
374 increaseCounterIfOldRevisionUsed(ctx, repository, range.to());
375
376 final int maxCommits0 = firstNonNull(maxCommits, Repository.DEFAULT_MAX_COMMITS);
377 return repository
378 .history(range.from(), range.to(), normalizePath(path), maxCommits0)
379 .thenApply(commits -> {
380 final boolean toList = to != null ||
381 isNullOrEmpty(revision) ||
382 "/".equalsIgnoreCase(revision);
383 return objectOrList(commits, toList, DtoConverter::convert);
384 });
385 }
386
387
388
389
390
391
392
393
394
395
396 @Get("/projects/{projectName}/repos/{repoName}/compare")
397 public CompletableFuture<?> getDiff(
398 ServiceRequestContext ctx,
399 @Param @Default("/**") String pathPattern,
400 @Param @Default("1") String from, @Param @Default("head") String to,
401 Repository repository,
402 @RequestConverter(QueryRequestConverter.class) @Nullable Query<?> query) {
403 final Revision fromRevision = new Revision(from);
404 final Revision toRevision = new Revision(to);
405 increaseCounterIfOldRevisionUsed(ctx, repository, fromRevision);
406 increaseCounterIfOldRevisionUsed(ctx, repository, toRevision);
407 if (query != null) {
408 return repository.diff(fromRevision, toRevision, query)
409 .thenApply(DtoConverter::convert);
410 } else {
411 return repository
412 .diff(fromRevision, toRevision, normalizePath(pathPattern))
413 .thenApply(changeMap -> changeMap.values().stream()
414 .map(DtoConverter::convert).collect(toImmutableList()));
415 }
416 }
417
418 private static <T> Object objectOrList(Collection<T> collection, boolean toList, Function<T, ?> converter) {
419 if (collection.isEmpty()) {
420 return ImmutableList.of();
421 }
422 if (toList) {
423 return collection.stream().map(converter).collect(toImmutableList());
424 }
425 return converter.apply(Iterables.getOnlyElement(collection));
426 }
427
428
429
430
431
432
433
434 @Get("/projects/{projectName}/repos/{repoName}/merge")
435 public <T> CompletableFuture<MergedEntryDto<T>> mergeFiles(
436 ServiceRequestContext ctx,
437 @Param @Default("-1") String revision, Repository repository,
438 @RequestConverter(MergeQueryRequestConverter.class) MergeQuery<T> query) {
439 final Revision rev = new Revision(revision);
440 increaseCounterIfOldRevisionUsed(ctx, repository, rev);
441 return repository.mergeFiles(rev, query).thenApply(DtoConverter::convert);
442 }
443
444 public static void checkMetaRepoPush(String repoName, Iterable<Change<?>> changes) {
445 if (Project.REPO_META.equals(repoName)) {
446 final boolean hasChangesOtherThanMetaRepoFiles =
447 Streams.stream(changes).anyMatch(change -> !isMetaFile(change.path()));
448 if (hasChangesOtherThanMetaRepoFiles) {
449 throw new InvalidPushException(
450 "The " + Project.REPO_META + " repository is reserved for internal usage.");
451 }
452
453
454
455 final Optional<String> notAllowedLocalRepo =
456 Streams.stream(changes)
457 .filter(change -> isMirrorOrCredentialFile(change.path()))
458 .filter(change -> change.content() != null)
459 .map(change -> {
460 final Object content = change.content();
461 if (content instanceof JsonNode) {
462 final JsonNode node = (JsonNode) content;
463 if (!node.isObject()) {
464 return null;
465 }
466 final JsonNode localRepoNode = node.get(MIRROR_LOCAL_REPO);
467 if (localRepoNode != null) {
468 final String localRepo = localRepoNode.textValue();
469 if (Project.isInternalRepo(localRepo)) {
470 return localRepo;
471 }
472 }
473 }
474 return null;
475 }).filter(Objects::nonNull).findFirst();
476 if (notAllowedLocalRepo.isPresent()) {
477 throw new InvalidPushException("invalid " + MIRROR_LOCAL_REPO + ": " +
478 notAllowedLocalRepo.get());
479 }
480 }
481 }
482 }