1   /*
2    * Copyright 2017 LINE Corporation
3    *
4    * LINE Corporation licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  
17  package com.linecorp.centraldogma.server.internal.api;
18  
19  import static com.google.common.base.MoreObjects.firstNonNull;
20  import static com.google.common.base.Strings.isNullOrEmpty;
21  import static com.google.common.collect.ImmutableList.toImmutableList;
22  import static com.linecorp.centraldogma.common.EntryType.DIRECTORY;
23  import static com.linecorp.centraldogma.internal.Util.isValidDirPath;
24  import static com.linecorp.centraldogma.internal.Util.isValidFilePath;
25  import static com.linecorp.centraldogma.server.internal.api.DtoConverter.convert;
26  import static com.linecorp.centraldogma.server.internal.api.HttpApiUtil.returnOrThrow;
27  import static com.linecorp.centraldogma.server.internal.api.RepositoryServiceV1.increaseCounterIfOldRevisionUsed;
28  import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.isMetaFile;
29  import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.isMirrorOrCredentialFile;
30  import static java.util.Objects.requireNonNull;
31  
32  import java.util.Collection;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Objects;
36  import java.util.Optional;
37  import java.util.concurrent.CancellationException;
38  import java.util.concurrent.CompletableFuture;
39  import java.util.function.Function;
40  
41  import javax.annotation.Nullable;
42  
43  import com.fasterxml.jackson.databind.JsonNode;
44  import com.google.common.base.Throwables;
45  import com.google.common.collect.ImmutableList;
46  import com.google.common.collect.Iterables;
47  import com.google.common.collect.Streams;
48  
49  import com.linecorp.armeria.common.HttpHeaderNames;
50  import com.linecorp.armeria.common.HttpResponse;
51  import com.linecorp.armeria.common.HttpStatus;
52  import com.linecorp.armeria.common.util.Exceptions;
53  import com.linecorp.armeria.server.ServiceRequestContext;
54  import com.linecorp.armeria.server.annotation.ConsumesJson;
55  import com.linecorp.armeria.server.annotation.Default;
56  import com.linecorp.armeria.server.annotation.Get;
57  import com.linecorp.armeria.server.annotation.Param;
58  import com.linecorp.armeria.server.annotation.Post;
59  import com.linecorp.armeria.server.annotation.ProducesJson;
60  import com.linecorp.armeria.server.annotation.RequestConverter;
61  import com.linecorp.centraldogma.common.Author;
62  import com.linecorp.centraldogma.common.Change;
63  import com.linecorp.centraldogma.common.Entry;
64  import com.linecorp.centraldogma.common.InvalidPushException;
65  import com.linecorp.centraldogma.common.Markup;
66  import com.linecorp.centraldogma.common.MergeQuery;
67  import com.linecorp.centraldogma.common.Query;
68  import com.linecorp.centraldogma.common.RepositoryRole;
69  import com.linecorp.centraldogma.common.Revision;
70  import com.linecorp.centraldogma.common.RevisionRange;
71  import com.linecorp.centraldogma.common.ShuttingDownException;
72  import com.linecorp.centraldogma.internal.api.v1.ChangeDto;
73  import com.linecorp.centraldogma.internal.api.v1.CommitMessageDto;
74  import com.linecorp.centraldogma.internal.api.v1.EntryDto;
75  import com.linecorp.centraldogma.internal.api.v1.MergedEntryDto;
76  import com.linecorp.centraldogma.internal.api.v1.PushResultDto;
77  import com.linecorp.centraldogma.internal.api.v1.WatchResultDto;
78  import com.linecorp.centraldogma.server.command.Command;
79  import com.linecorp.centraldogma.server.command.CommandExecutor;
80  import com.linecorp.centraldogma.server.command.CommitResult;
81  import com.linecorp.centraldogma.server.internal.api.auth.RequiresRepositoryRole;
82  import com.linecorp.centraldogma.server.internal.api.converter.ChangesRequestConverter;
83  import com.linecorp.centraldogma.server.internal.api.converter.CommitMessageRequestConverter;
84  import com.linecorp.centraldogma.server.internal.api.converter.MergeQueryRequestConverter;
85  import com.linecorp.centraldogma.server.internal.api.converter.QueryRequestConverter;
86  import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter;
87  import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter.WatchRequest;
88  import com.linecorp.centraldogma.server.storage.project.Project;
89  import com.linecorp.centraldogma.server.storage.repository.FindOption;
90  import com.linecorp.centraldogma.server.storage.repository.FindOptions;
91  import com.linecorp.centraldogma.server.storage.repository.Repository;
92  
93  import io.micrometer.core.instrument.MeterRegistry;
94  import io.netty.util.AttributeKey;
95  
96  /**
97   * Annotated service object for managing and watching contents.
98   */
99  @ProducesJson
100 @RequiresRepositoryRole(RepositoryRole.READ)
101 @RequestConverter(CommitMessageRequestConverter.class)
102 public class ContentServiceV1 extends AbstractService {
103 
104     static final AttributeKey<Boolean> IS_WATCH_REQUEST =
105             AttributeKey.valueOf(ContentServiceV1.class, "IS_WATCH_REQUEST");
106 
107     private static final String MIRROR_LOCAL_REPO = "localRepo";
108 
109     private final WatchService watchService;
110     private final MeterRegistry meterRegistry;
111 
112     public ContentServiceV1(CommandExecutor executor, WatchService watchService, MeterRegistry meterRegistry) {
113         super(executor);
114         this.watchService = requireNonNull(watchService, "watchService");
115         this.meterRegistry = requireNonNull(meterRegistry, "meterRegistry");
116     }
117 
118     /**
119      * GET /projects/{projectName}/repos/{repoName}/list{path}?revision={revision}
120      *
121      * <p>Returns the list of files in the path.
122      */
123     @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/list(?<path>(|/.*))$")
124     public CompletableFuture<List<EntryDto<?>>> listFiles(ServiceRequestContext ctx,
125                                                           @Param String path,
126                                                           @Param @Default("-1") String revision,
127                                                           Repository repository) {
128         final String normalizedPath = normalizePath(path);
129         final Revision normalizedRev = repository.normalizeNow(new Revision(revision));
130         increaseCounterIfOldRevisionUsed(ctx, repository, normalizedRev);
131         final CompletableFuture<List<EntryDto<?>>> future = new CompletableFuture<>();
132         listFiles(repository, normalizedPath, normalizedRev, false, future);
133         return future;
134     }
135 
136     private static void listFiles(Repository repository, String pathPattern, Revision normalizedRev,
137                                   boolean withContent, CompletableFuture<List<EntryDto<?>>> result) {
138         final Map<FindOption<?>, ?> options = withContent ? FindOptions.FIND_ALL_WITH_CONTENT
139                                                           : FindOptions.FIND_ALL_WITHOUT_CONTENT;
140 
141         repository.find(normalizedRev, pathPattern, options).handle((entries, thrown) -> {
142             if (thrown != null) {
143                 result.completeExceptionally(thrown);
144                 return null;
145             }
146             // If the pathPattern is a valid file path and the result is a directory, the client forgets to add
147             // "/*" to the end of the path. So, let's do it and invoke once more.
148             // This is called once at most, because the pathPattern is not a valid file path anymore.
149             if (isValidFilePath(pathPattern) && entries.size() == 1 &&
150                 entries.values().iterator().next().type() == DIRECTORY) {
151                 listFiles(repository, pathPattern + "/*", normalizedRev, withContent, result);
152             } else {
153                 result.complete(entries.values().stream()
154                                        .map(entry -> convert(repository, normalizedRev, entry, withContent))
155                                        .collect(toImmutableList()));
156             }
157             return null;
158         });
159     }
160 
161     /**
162      * Normalizes the path according to the following order.
163      * <ul>
164      *   <li>if the path is {@code null}, empty string or "/", normalize to {@code "/*"}</li>
165      *   <li>if the path is a valid file path, return the path as it is</li>
166      *   <li>if the path is a valid directory path, append "*" at the end</li>
167      * </ul>
168      */
169     private static String normalizePath(String path) {
170         if (path == null || path.isEmpty() || "/".equals(path)) {
171             return "/*";
172         }
173         if (isValidFilePath(path)) {
174             return path;
175         }
176         if (isValidDirPath(path)) {
177             if (path.endsWith("/")) {
178                 return path + '*';
179             } else {
180                 return path + "/*";
181             }
182         }
183         return path;
184     }
185 
186     /**
187      * POST /projects/{projectName}/repos/{repoName}/contents?revision={revision}
188      *
189      * <p>Pushes a commit.
190      */
191     @Post("/projects/{projectName}/repos/{repoName}/contents")
192     @ConsumesJson
193     @RequiresRepositoryRole(RepositoryRole.WRITE)
194     public CompletableFuture<PushResultDto> push(
195             ServiceRequestContext ctx,
196             @Param @Default("-1") String revision,
197             Repository repository,
198             Author author,
199             CommitMessageDto commitMessage,
200             @RequestConverter(ChangesRequestConverter.class) Iterable<Change<?>> changes) {
201         checkMetaRepoPush(repository.name(), changes);
202         meterRegistry.counter("commits.push",
203                               "project", repository.parent().name(),
204                               "repository", repository.name())
205                      .increment();
206 
207         final long commitTimeMillis = System.currentTimeMillis();
208         return push(commitTimeMillis, author, repository, new Revision(revision), commitMessage, changes)
209                 .toCompletableFuture()
210                 .thenApply(rrev -> convert(rrev, commitTimeMillis));
211     }
212 
213     private CompletableFuture<Revision> push(long commitTimeMills, Author author, Repository repository,
214                                              Revision revision, CommitMessageDto commitMessage,
215                                              Iterable<Change<?>> changes) {
216         final String summary = commitMessage.summary();
217         final String detail = commitMessage.detail();
218         final Markup markup = commitMessage.markup();
219 
220         return execute(Command.push(
221                 commitTimeMills, author, repository.parent().name(), repository.name(),
222                 revision, summary, detail, markup, changes)).thenApply(CommitResult::revision);
223     }
224 
225     /**
226      * POST /projects/{projectName}/repos/{repoName}/preview?revision={revision}
227      *
228      * <p>Previews the actual changes which will be resulted by the given changes.
229      */
230     @Post("/projects/{projectName}/repos/{repoName}/preview")
231     @ConsumesJson
232     public CompletableFuture<Iterable<ChangeDto<?>>> preview(
233             ServiceRequestContext ctx,
234             @Param @Default("-1") String revision,
235             Repository repository,
236             @RequestConverter(ChangesRequestConverter.class) Iterable<Change<?>> changes) {
237         final Revision baseRevision = new Revision(revision);
238         increaseCounterIfOldRevisionUsed(ctx, repository, baseRevision);
239         final CompletableFuture<Map<String, Change<?>>> changesFuture =
240                 repository.previewDiff(baseRevision, changes);
241 
242         return changesFuture.thenApply(previewDiffs -> previewDiffs.values().stream()
243                                                                    .map(DtoConverter::convert)
244                                                                    .collect(toImmutableList()));
245     }
246 
247     /**
248      * GET /projects/{projectName}/repos/{repoName}/contents{path}?revision={revision}&amp;
249      * jsonpath={jsonpath}
250      *
251      * <p>Returns the entry of files in the path. This is same with
252      * {@link #listFiles(ServiceRequestContext, String, String, Repository)} except that containing
253      * the content of the files.
254      * Note that if the {@link HttpHeaderNames#IF_NONE_MATCH} in which has a revision is sent with,
255      * this will await for the time specified in {@link HttpHeaderNames#PREFER}.
256      * During the time if the specified revision becomes different with the latest revision, this will
257      * response back right away to the client.
258      * {@link HttpStatus#NOT_MODIFIED} otherwise.
259      */
260     @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/contents(?<path>(|/.*))$")
261     public CompletableFuture<?> getFiles(
262             ServiceRequestContext ctx,
263             @Param String path, @Param @Default("-1") String revision,
264             Repository repository,
265             @RequestConverter(WatchRequestConverter.class) @Nullable WatchRequest watchRequest,
266             @RequestConverter(QueryRequestConverter.class) @Nullable Query<?> query) {
267         increaseCounterIfOldRevisionUsed(ctx, repository, new Revision(revision));
268         final String normalizedPath = normalizePath(path);
269 
270         // watch repository or a file
271         if (watchRequest != null) {
272             ctx.setAttr(IS_WATCH_REQUEST, true);
273             final Revision lastKnownRevision = watchRequest.lastKnownRevision();
274             final long timeOutMillis = watchRequest.timeoutMillis();
275             final boolean errorOnEntryNotFound = watchRequest.notifyEntryNotFound();
276             if (query != null) {
277                 return watchFile(ctx, repository, lastKnownRevision, query, timeOutMillis,
278                                  errorOnEntryNotFound);
279             }
280 
281             return watchRepository(ctx, repository, lastKnownRevision, normalizedPath,
282                                    timeOutMillis, errorOnEntryNotFound);
283         }
284 
285         final Revision normalizedRev = repository.normalizeNow(new Revision(revision));
286         if (query != null) {
287             // get a file
288             return repository.get(normalizedRev, query)
289                              .handle(returnOrThrow((Entry<?> result) -> convert(repository, normalizedRev,
290                                                                                 result, true)));
291         }
292 
293         // get files
294         final CompletableFuture<List<EntryDto<?>>> future = new CompletableFuture<>();
295         listFiles(repository, normalizedPath, normalizedRev, true, future);
296         return future;
297     }
298 
299     private CompletableFuture<?> watchFile(ServiceRequestContext ctx,
300                                            Repository repository, Revision lastKnownRevision,
301                                            Query<?> query, long timeOutMillis, boolean errorOnEntryNotFound) {
302         final CompletableFuture<? extends Entry<?>> future = watchService.watchFile(
303                 repository, lastKnownRevision, query, timeOutMillis, errorOnEntryNotFound);
304 
305         if (!future.isDone()) {
306             ctx.log().whenComplete().thenRun(() -> future.cancel(false));
307         }
308 
309         return future.thenApply(entry -> {
310             final Revision revision = entry.revision();
311             final EntryDto<?> entryDto = convert(repository, revision, entry, true);
312             return (Object) new WatchResultDto(revision, entryDto);
313         }).exceptionally(ContentServiceV1::handleWatchFailure);
314     }
315 
316     private CompletableFuture<?> watchRepository(ServiceRequestContext ctx,
317                                                  Repository repository, Revision lastKnownRevision,
318                                                  String pathPattern, long timeOutMillis,
319                                                  boolean errorOnEntryNotFound) {
320         final CompletableFuture<Revision> future =
321                 watchService.watchRepository(repository, lastKnownRevision, pathPattern,
322                                              timeOutMillis, errorOnEntryNotFound);
323 
324         if (!future.isDone()) {
325             ctx.log().whenComplete().thenRun(() -> future.cancel(false));
326         }
327 
328         return future.thenApply(revision -> (Object) new WatchResultDto(revision, null))
329                      .exceptionally(ContentServiceV1::handleWatchFailure);
330     }
331 
332     private static Object handleWatchFailure(Throwable thrown) {
333         final Throwable rootCause = Throwables.getRootCause(thrown);
334         if (rootCause instanceof CancellationException || rootCause instanceof ShuttingDownException) {
335             // timeout happens
336             return HttpResponse.of(HttpStatus.NOT_MODIFIED);
337         }
338         return Exceptions.throwUnsafely(thrown);
339     }
340 
341     /**
342      * GET /projects/{projectName}/repos/{repoName}/commits/{revision}?
343      * path={path}&amp;to={to}&amp;maxCommits={maxCommits}
344      *
345      * <p>Returns a commit or the list of commits in the path. If the user specify the {@code revision} only,
346      * this will return the corresponding commit. If the user does not specify the {@code revision} or
347      * specify {@code to}, this will return the list of commits.
348      */
349     @Get("regex:/projects/(?<projectName>[^/]+)/repos/(?<repoName>[^/]+)/commits(?<revision>(|/.*))$")
350     public CompletableFuture<?> listCommits(ServiceRequestContext ctx,
351                                             @Param String revision,
352                                             @Param @Default("/**") String path,
353                                             @Param @Nullable String to,
354                                             @Param @Nullable Integer maxCommits,
355                                             Repository repository) {
356         final Revision fromRevision;
357         final Revision toRevision;
358 
359         // 1. only the "revision" is specified:       get the "revision" and return just one commit
360         // 2. only the "to" is specified:             get from "HEAD" to "to" and return the list
361         // 3. the "revision" and "to" is specified:   get from the "revision" to "to" and return the list
362         // 4. nothing is specified:                   get from "HEAD" to "INIT" and return the list
363         if (isNullOrEmpty(revision) || "/".equalsIgnoreCase(revision)) {
364             fromRevision = Revision.HEAD;
365             toRevision = to != null ? new Revision(to) : Revision.INIT;
366         } else {
367             fromRevision = new Revision(revision.substring(1));
368             toRevision = to != null ? new Revision(to) : fromRevision;
369         }
370 
371         final RevisionRange range = repository.normalizeNow(fromRevision, toRevision).toDescending();
372 
373         increaseCounterIfOldRevisionUsed(ctx, repository, range.from());
374         increaseCounterIfOldRevisionUsed(ctx, repository, range.to());
375 
376         final int maxCommits0 = firstNonNull(maxCommits, Repository.DEFAULT_MAX_COMMITS);
377         return repository
378                 .history(range.from(), range.to(), normalizePath(path), maxCommits0)
379                 .thenApply(commits -> {
380                     final boolean toList = to != null ||
381                                            isNullOrEmpty(revision) ||
382                                            "/".equalsIgnoreCase(revision);
383                     return objectOrList(commits, toList, DtoConverter::convert);
384                 });
385     }
386 
387     /**
388      * GET /projects/{projectName}/repos/{repoName}/compare?
389      * path={path}&amp;from={from}&amp;to={to}&amp;jsonpath={jsonpath} returns a diff.
390      *
391      * <p>or,
392      *
393      * <p>GET /projects/{projectName}/repos/{repoName}/compare?
394      * pathPattern={pathPattern}&amp;from={from}&amp;to={to} returns diffs.
395      */
396     @Get("/projects/{projectName}/repos/{repoName}/compare")
397     public CompletableFuture<?> getDiff(
398             ServiceRequestContext ctx,
399             @Param @Default("/**") String pathPattern,
400             @Param @Default("1") String from, @Param @Default("head") String to,
401             Repository repository,
402             @RequestConverter(QueryRequestConverter.class) @Nullable Query<?> query) {
403         final Revision fromRevision = new Revision(from);
404         final Revision toRevision = new Revision(to);
405         increaseCounterIfOldRevisionUsed(ctx, repository, fromRevision);
406         increaseCounterIfOldRevisionUsed(ctx, repository, toRevision);
407         if (query != null) {
408             return repository.diff(fromRevision, toRevision, query)
409                              .thenApply(DtoConverter::convert);
410         } else {
411             return repository
412                     .diff(fromRevision, toRevision, normalizePath(pathPattern))
413                     .thenApply(changeMap -> changeMap.values().stream()
414                                                      .map(DtoConverter::convert).collect(toImmutableList()));
415         }
416     }
417 
418     private static <T> Object objectOrList(Collection<T> collection, boolean toList, Function<T, ?> converter) {
419         if (collection.isEmpty()) {
420             return ImmutableList.of();
421         }
422         if (toList) {
423             return collection.stream().map(converter).collect(toImmutableList());
424         }
425         return converter.apply(Iterables.getOnlyElement(collection));
426     }
427 
428     /**
429      * GET /projects/{projectName}/repos/{repoName}/merge?
430      * revision={revision}&amp;path={path}&amp;optional_path={optional_path}
431      *
432      * <p>Returns a merged entry of files which are specified in the query string.
433      */
434     @Get("/projects/{projectName}/repos/{repoName}/merge")
435     public <T> CompletableFuture<MergedEntryDto<T>> mergeFiles(
436             ServiceRequestContext ctx,
437             @Param @Default("-1") String revision, Repository repository,
438             @RequestConverter(MergeQueryRequestConverter.class) MergeQuery<T> query) {
439         final Revision rev = new Revision(revision);
440         increaseCounterIfOldRevisionUsed(ctx, repository, rev);
441         return repository.mergeFiles(rev, query).thenApply(DtoConverter::convert);
442     }
443 
444     public static void checkMetaRepoPush(String repoName, Iterable<Change<?>> changes) {
445         if (Project.REPO_META.equals(repoName)) {
446             final boolean hasChangesOtherThanMetaRepoFiles =
447                     Streams.stream(changes).anyMatch(change -> !isMetaFile(change.path()));
448             if (hasChangesOtherThanMetaRepoFiles) {
449                 throw new InvalidPushException(
450                         "The " + Project.REPO_META + " repository is reserved for internal usage.");
451             }
452 
453             // TODO(ikhoon): Disallow creating a mirror with the commit API. Mirroring REST API should be used
454             //               to validate the input.
455             final Optional<String> notAllowedLocalRepo =
456                     Streams.stream(changes)
457                            .filter(change -> isMirrorOrCredentialFile(change.path()))
458                            .filter(change -> change.content() != null)
459                            .map(change -> {
460                                final Object content = change.content();
461                                if (content instanceof JsonNode) {
462                                    final JsonNode node = (JsonNode) content;
463                                    if (!node.isObject()) {
464                                        return null;
465                                    }
466                                    final JsonNode localRepoNode = node.get(MIRROR_LOCAL_REPO);
467                                    if (localRepoNode != null) {
468                                        final String localRepo = localRepoNode.textValue();
469                                        if (Project.isInternalRepo(localRepo)) {
470                                            return localRepo;
471                                        }
472                                    }
473                                }
474                                return null;
475                            }).filter(Objects::nonNull).findFirst();
476             if (notAllowedLocalRepo.isPresent()) {
477                 throw new InvalidPushException("invalid " + MIRROR_LOCAL_REPO + ": " +
478                                                notAllowedLocalRepo.get());
479             }
480         }
481     }
482 }