1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.api;
18
19 import static com.google.common.base.MoreObjects.firstNonNull;
20 import static com.google.common.base.Strings.isNullOrEmpty;
21 import static com.google.common.collect.ImmutableList.toImmutableList;
22 import static com.linecorp.centraldogma.common.EntryType.DIRECTORY;
23 import static com.linecorp.centraldogma.internal.Util.isValidDirPath;
24 import static com.linecorp.centraldogma.internal.Util.isValidFilePath;
25 import static com.linecorp.centraldogma.server.internal.api.DtoConverter.convert;
26 import static com.linecorp.centraldogma.server.internal.api.HttpApiUtil.returnOrThrow;
27 import static com.linecorp.centraldogma.server.internal.api.RepositoryServiceV1.increaseCounterIfOldRevisionUsed;
28 import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.metaRepoFiles;
29 import static java.util.Objects.requireNonNull;
30
31 import java.util.Collection;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Objects;
35 import java.util.Optional;
36 import java.util.concurrent.CancellationException;
37 import java.util.concurrent.CompletableFuture;
38 import java.util.function.Function;
39
40 import javax.annotation.Nullable;
41
42 import com.fasterxml.jackson.databind.JsonNode;
43 import com.google.common.base.Throwables;
44 import com.google.common.collect.ImmutableList;
45 import com.google.common.collect.Iterables;
46 import com.google.common.collect.Streams;
47
48 import com.linecorp.armeria.common.HttpHeaderNames;
49 import com.linecorp.armeria.common.HttpResponse;
50 import com.linecorp.armeria.common.HttpStatus;
51 import com.linecorp.armeria.common.util.Exceptions;
52 import com.linecorp.armeria.server.ServiceRequestContext;
53 import com.linecorp.armeria.server.annotation.Default;
54 import com.linecorp.armeria.server.annotation.ExceptionHandler;
55 import com.linecorp.armeria.server.annotation.Get;
56 import com.linecorp.armeria.server.annotation.Param;
57 import com.linecorp.armeria.server.annotation.Post;
58 import com.linecorp.armeria.server.annotation.ProducesJson;
59 import com.linecorp.armeria.server.annotation.RequestConverter;
60 import com.linecorp.centraldogma.common.Author;
61 import com.linecorp.centraldogma.common.Change;
62 import com.linecorp.centraldogma.common.Entry;
63 import com.linecorp.centraldogma.common.InvalidPushException;
64 import com.linecorp.centraldogma.common.Markup;
65 import com.linecorp.centraldogma.common.MergeQuery;
66 import com.linecorp.centraldogma.common.Query;
67 import com.linecorp.centraldogma.common.Revision;
68 import com.linecorp.centraldogma.common.RevisionRange;
69 import com.linecorp.centraldogma.common.ShuttingDownException;
70 import com.linecorp.centraldogma.internal.api.v1.ChangeDto;
71 import com.linecorp.centraldogma.internal.api.v1.CommitMessageDto;
72 import com.linecorp.centraldogma.internal.api.v1.EntryDto;
73 import com.linecorp.centraldogma.internal.api.v1.MergedEntryDto;
74 import com.linecorp.centraldogma.internal.api.v1.PushResultDto;
75 import com.linecorp.centraldogma.internal.api.v1.WatchResultDto;
76 import com.linecorp.centraldogma.server.command.Command;
77 import com.linecorp.centraldogma.server.command.CommandExecutor;
78 import com.linecorp.centraldogma.server.command.CommitResult;
79 import com.linecorp.centraldogma.server.internal.api.auth.RequiresReadPermission;
80 import com.linecorp.centraldogma.server.internal.api.auth.RequiresWritePermission;
81 import com.linecorp.centraldogma.server.internal.api.converter.ChangesRequestConverter;
82 import com.linecorp.centraldogma.server.internal.api.converter.CommitMessageRequestConverter;
83 import com.linecorp.centraldogma.server.internal.api.converter.MergeQueryRequestConverter;
84 import com.linecorp.centraldogma.server.internal.api.converter.QueryRequestConverter;
85 import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter;
86 import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter.WatchRequest;
87 import com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository;
88 import com.linecorp.centraldogma.server.storage.project.Project;
89 import com.linecorp.centraldogma.server.storage.repository.FindOption;
90 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
91 import com.linecorp.centraldogma.server.storage.repository.Repository;
92
93
94
95
96 @ProducesJson
97 @RequiresReadPermission
98 @RequestConverter(CommitMessageRequestConverter.class)
99 @ExceptionHandler(HttpApiExceptionHandler.class)
100 public class ContentServiceV1 extends AbstractService {
101
102 private static final String MIRROR_LOCAL_REPO = "localRepo";
103
104 private final WatchService watchService;
105
106 public ContentServiceV1(CommandExecutor executor, WatchService watchService) {
107 super(executor);
108 this.watchService = requireNonNull(watchService, "watchService");
109 }
110
111
112
113
114
115
116 @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/list(?<path>(|/.*))$")
117 public CompletableFuture<List<EntryDto<?>>> listFiles(ServiceRequestContext ctx,
118 @Param String path,
119 @Param @Default("-1") String revision,
120 Repository repository) {
121 final String normalizedPath = normalizePath(path);
122 final Revision normalizedRev = repository.normalizeNow(new Revision(revision));
123 increaseCounterIfOldRevisionUsed(ctx, repository, normalizedRev);
124 final CompletableFuture<List<EntryDto<?>>> future = new CompletableFuture<>();
125 listFiles(repository, normalizedPath, normalizedRev, false, future);
126 return future;
127 }
128
129 private static void listFiles(Repository repository, String pathPattern, Revision normalizedRev,
130 boolean withContent, CompletableFuture<List<EntryDto<?>>> result) {
131 final Map<FindOption<?>, ?> options = withContent ? FindOptions.FIND_ALL_WITH_CONTENT
132 : FindOptions.FIND_ALL_WITHOUT_CONTENT;
133
134 repository.find(normalizedRev, pathPattern, options).handle((entries, thrown) -> {
135 if (thrown != null) {
136 result.completeExceptionally(thrown);
137 return null;
138 }
139
140
141
142 if (isValidFilePath(pathPattern) && entries.size() == 1 &&
143 entries.values().iterator().next().type() == DIRECTORY) {
144 listFiles(repository, pathPattern + "/*", normalizedRev, withContent, result);
145 } else {
146 result.complete(entries.values().stream()
147 .map(entry -> convert(repository, normalizedRev, entry, withContent))
148 .collect(toImmutableList()));
149 }
150 return null;
151 });
152 }
153
154
155
156
157
158
159
160
161
162 private static String normalizePath(String path) {
163 if (path == null || path.isEmpty() || "/".equals(path)) {
164 return "/*";
165 }
166 if (isValidFilePath(path)) {
167 return path;
168 }
169 if (isValidDirPath(path)) {
170 if (path.endsWith("/")) {
171 return path + '*';
172 } else {
173 return path + "/*";
174 }
175 }
176 return path;
177 }
178
179
180
181
182
183
184 @Post("/projects/{projectName}/repos/{repoName}/contents")
185 @RequiresWritePermission
186 public CompletableFuture<PushResultDto> push(
187 @Param @Default("-1") String revision,
188 Repository repository,
189 Author author,
190 CommitMessageDto commitMessage,
191 @RequestConverter(ChangesRequestConverter.class) Iterable<Change<?>> changes) {
192 checkPush(repository.name(), changes);
193
194 final long commitTimeMillis = System.currentTimeMillis();
195 return push(commitTimeMillis, author, repository, new Revision(revision), commitMessage, changes)
196 .toCompletableFuture()
197 .thenApply(rrev -> convert(rrev, commitTimeMillis));
198 }
199
200 private CompletableFuture<Revision> push(long commitTimeMills, Author author, Repository repository,
201 Revision revision, CommitMessageDto commitMessage,
202 Iterable<Change<?>> changes) {
203 final String summary = commitMessage.summary();
204 final String detail = commitMessage.detail();
205 final Markup markup = commitMessage.markup();
206
207 return execute(Command.push(
208 commitTimeMills, author, repository.parent().name(), repository.name(),
209 revision, summary, detail, markup, changes)).thenApply(CommitResult::revision);
210 }
211
212
213
214
215
216
217 @Post("/projects/{projectName}/repos/{repoName}/preview")
218 public CompletableFuture<Iterable<ChangeDto<?>>> preview(
219 ServiceRequestContext ctx,
220 @Param @Default("-1") String revision,
221 Repository repository,
222 @RequestConverter(ChangesRequestConverter.class) Iterable<Change<?>> changes) {
223 final Revision baseRevision = new Revision(revision);
224 increaseCounterIfOldRevisionUsed(ctx, repository, baseRevision);
225 final CompletableFuture<Map<String, Change<?>>> changesFuture =
226 repository.previewDiff(baseRevision, changes);
227
228 return changesFuture.thenApply(previewDiffs -> previewDiffs.values().stream()
229 .map(DtoConverter::convert)
230 .collect(toImmutableList()));
231 }
232
233
234
235
236
237
238
239
240
241
242
243
244
245 @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/contents(?<path>(|/.*))$")
246 public CompletableFuture<?> getFiles(
247 ServiceRequestContext ctx,
248 @Param String path, @Param @Default("-1") String revision,
249 Repository repository,
250 @RequestConverter(WatchRequestConverter.class) @Nullable WatchRequest watchRequest,
251 @RequestConverter(QueryRequestConverter.class) @Nullable Query<?> query) {
252 increaseCounterIfOldRevisionUsed(ctx, repository, new Revision(revision));
253 final String normalizedPath = normalizePath(path);
254
255
256 if (watchRequest != null) {
257 final Revision lastKnownRevision = watchRequest.lastKnownRevision();
258 final long timeOutMillis = watchRequest.timeoutMillis();
259 final boolean errorOnEntryNotFound = watchRequest.notifyEntryNotFound();
260 if (query != null) {
261 return watchFile(ctx, repository, lastKnownRevision, query, timeOutMillis,
262 errorOnEntryNotFound);
263 }
264
265 return watchRepository(ctx, repository, lastKnownRevision, normalizedPath,
266 timeOutMillis, errorOnEntryNotFound);
267 }
268
269 final Revision normalizedRev = repository.normalizeNow(new Revision(revision));
270 if (query != null) {
271
272 return repository.get(normalizedRev, query)
273 .handle(returnOrThrow((Entry<?> result) -> convert(repository, normalizedRev,
274 result, true)));
275 }
276
277
278 final CompletableFuture<List<EntryDto<?>>> future = new CompletableFuture<>();
279 listFiles(repository, normalizedPath, normalizedRev, true, future);
280 return future;
281 }
282
283 private CompletableFuture<?> watchFile(ServiceRequestContext ctx,
284 Repository repository, Revision lastKnownRevision,
285 Query<?> query, long timeOutMillis, boolean errorOnEntryNotFound) {
286 final CompletableFuture<? extends Entry<?>> future = watchService.watchFile(
287 repository, lastKnownRevision, query, timeOutMillis, errorOnEntryNotFound);
288
289 if (!future.isDone()) {
290 ctx.log().whenComplete().thenRun(() -> future.cancel(false));
291 }
292
293 return future.thenApply(entry -> {
294 final Revision revision = entry.revision();
295 final EntryDto<?> entryDto = convert(repository, revision, entry, true);
296 return (Object) new WatchResultDto(revision, entryDto);
297 }).exceptionally(ContentServiceV1::handleWatchFailure);
298 }
299
300 private CompletableFuture<?> watchRepository(ServiceRequestContext ctx,
301 Repository repository, Revision lastKnownRevision,
302 String pathPattern, long timeOutMillis,
303 boolean errorOnEntryNotFound) {
304 final CompletableFuture<Revision> future =
305 watchService.watchRepository(repository, lastKnownRevision, pathPattern,
306 timeOutMillis, errorOnEntryNotFound);
307
308 if (!future.isDone()) {
309 ctx.log().whenComplete().thenRun(() -> future.cancel(false));
310 }
311
312 return future.thenApply(revision -> (Object) new WatchResultDto(revision, null))
313 .exceptionally(ContentServiceV1::handleWatchFailure);
314 }
315
316 private static Object handleWatchFailure(Throwable thrown) {
317 final Throwable rootCause = Throwables.getRootCause(thrown);
318 if (rootCause instanceof CancellationException || rootCause instanceof ShuttingDownException) {
319
320 return HttpResponse.of(HttpStatus.NOT_MODIFIED);
321 }
322 return Exceptions.throwUnsafely(thrown);
323 }
324
325
326
327
328
329
330
331
332
333 @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/commits(?<revision>(|/.*))$")
334 public CompletableFuture<?> listCommits(ServiceRequestContext ctx,
335 @Param String revision,
336 @Param @Default("/**") String path,
337 @Param @Nullable String to,
338 @Param @Nullable Integer maxCommits,
339 Repository repository) {
340 final Revision fromRevision;
341 final Revision toRevision;
342
343
344
345
346
347 if (isNullOrEmpty(revision) || "/".equalsIgnoreCase(revision)) {
348 fromRevision = Revision.HEAD;
349 toRevision = to != null ? new Revision(to) : Revision.INIT;
350 } else {
351 fromRevision = new Revision(revision.substring(1));
352 toRevision = to != null ? new Revision(to) : fromRevision;
353 }
354
355 final RevisionRange range = repository.normalizeNow(fromRevision, toRevision).toDescending();
356
357 increaseCounterIfOldRevisionUsed(ctx, repository, range.from());
358 increaseCounterIfOldRevisionUsed(ctx, repository, range.to());
359
360 final int maxCommits0 = firstNonNull(maxCommits, Repository.DEFAULT_MAX_COMMITS);
361 return repository
362 .history(range.from(), range.to(), normalizePath(path), maxCommits0)
363 .thenApply(commits -> {
364 final boolean toList = to != null ||
365 isNullOrEmpty(revision) ||
366 "/".equalsIgnoreCase(revision);
367 return objectOrList(commits, toList, DtoConverter::convert);
368 });
369 }
370
371
372
373
374
375
376
377
378
379
380 @Get("/projects/{projectName}/repos/{repoName}/compare")
381 public CompletableFuture<?> getDiff(
382 ServiceRequestContext ctx,
383 @Param @Default("/**") String pathPattern,
384 @Param @Default("1") String from, @Param @Default("head") String to,
385 Repository repository,
386 @RequestConverter(QueryRequestConverter.class) @Nullable Query<?> query) {
387 final Revision fromRevision = new Revision(from);
388 final Revision toRevision = new Revision(to);
389 increaseCounterIfOldRevisionUsed(ctx, repository, fromRevision);
390 increaseCounterIfOldRevisionUsed(ctx, repository, toRevision);
391 if (query != null) {
392 return repository.diff(fromRevision, toRevision, query)
393 .thenApply(DtoConverter::convert);
394 } else {
395 return repository
396 .diff(fromRevision, toRevision, normalizePath(pathPattern))
397 .thenApply(changeMap -> changeMap.values().stream()
398 .map(DtoConverter::convert).collect(toImmutableList()));
399 }
400 }
401
402 private static <T> Object objectOrList(Collection<T> collection, boolean toList, Function<T, ?> converter) {
403 if (collection.isEmpty()) {
404 return ImmutableList.of();
405 }
406 if (toList) {
407 return collection.stream().map(converter).collect(toImmutableList());
408 }
409 return converter.apply(Iterables.getOnlyElement(collection));
410 }
411
412
413
414
415
416
417
418 @Get("/projects/{projectName}/repos/{repoName}/merge")
419 public <T> CompletableFuture<MergedEntryDto<T>> mergeFiles(
420 ServiceRequestContext ctx,
421 @Param @Default("-1") String revision, Repository repository,
422 @RequestConverter(MergeQueryRequestConverter.class) MergeQuery<T> query) {
423 final Revision rev = new Revision(revision);
424 increaseCounterIfOldRevisionUsed(ctx, repository, rev);
425 return repository.mergeFiles(rev, query).thenApply(DtoConverter::convert);
426 }
427
428
429
430
431
432 public static void checkPush(String repoName, Iterable<Change<?>> changes) {
433 if (Project.REPO_META.equals(repoName)) {
434 final boolean hasChangesOtherThanMetaRepoFiles =
435 Streams.stream(changes).anyMatch(change -> !metaRepoFiles.contains(change.path()));
436 if (hasChangesOtherThanMetaRepoFiles) {
437 throw new InvalidPushException(
438 "The " + Project.REPO_META + " repository is reserved for internal usage.");
439 }
440
441 final Optional<String> notAllowedLocalRepo =
442 Streams.stream(changes)
443 .filter(change -> DefaultMetaRepository.PATH_MIRRORS.equals(change.path()))
444 .filter(change -> change.content() != null)
445 .map(change -> {
446 final Object content = change.content();
447 if (content instanceof JsonNode) {
448 final JsonNode node = (JsonNode) content;
449 if (!node.isArray()) {
450 return null;
451 }
452 for (JsonNode jsonNode : node) {
453 final JsonNode localRepoNode = jsonNode.get(MIRROR_LOCAL_REPO);
454 if (localRepoNode != null) {
455 final String localRepo = localRepoNode.textValue();
456 if (Project.isReservedRepoName(localRepo)) {
457 return localRepo;
458 }
459 }
460 }
461 }
462 return null;
463 }).filter(Objects::nonNull).findFirst();
464 if (notAllowedLocalRepo.isPresent()) {
465 throw new InvalidPushException("invalid " + MIRROR_LOCAL_REPO + ": " +
466 notAllowedLocalRepo.get());
467 }
468 }
469 }
470 }