1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage.repository.git;
18
19 import static com.google.common.base.Preconditions.checkState;
20 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context;
21 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut;
22 import static java.nio.charset.StandardCharsets.UTF_8;
23 import static java.util.Objects.requireNonNull;
24 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
25 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION;
26
27 import java.io.File;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.lang.reflect.Field;
31 import java.util.ArrayList;
32 import java.util.Collection;
33 import java.util.Collections;
34 import java.util.LinkedHashMap;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.Properties;
38 import java.util.concurrent.CompletableFuture;
39 import java.util.concurrent.CopyOnWriteArrayList;
40 import java.util.concurrent.Executor;
41 import java.util.concurrent.atomic.AtomicReference;
42 import java.util.concurrent.locks.ReadWriteLock;
43 import java.util.concurrent.locks.ReentrantReadWriteLock;
44 import java.util.function.BiConsumer;
45 import java.util.function.Function;
46 import java.util.function.Supplier;
47 import java.util.regex.Pattern;
48
49 import javax.annotation.Nullable;
50
51 import org.eclipse.jgit.diff.DiffEntry;
52 import org.eclipse.jgit.diff.DiffFormatter;
53 import org.eclipse.jgit.dircache.DirCache;
54 import org.eclipse.jgit.dircache.DirCacheIterator;
55 import org.eclipse.jgit.lib.Constants;
56 import org.eclipse.jgit.lib.ObjectId;
57 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
58 import org.eclipse.jgit.lib.ObjectReader;
59 import org.eclipse.jgit.lib.PersonIdent;
60 import org.eclipse.jgit.lib.Ref;
61 import org.eclipse.jgit.lib.RefUpdate;
62 import org.eclipse.jgit.lib.RefUpdate.Result;
63 import org.eclipse.jgit.lib.RepositoryBuilder;
64 import org.eclipse.jgit.revwalk.RevCommit;
65 import org.eclipse.jgit.revwalk.RevTree;
66 import org.eclipse.jgit.revwalk.RevWalk;
67 import org.eclipse.jgit.revwalk.TreeRevFilter;
68 import org.eclipse.jgit.revwalk.filter.RevFilter;
69 import org.eclipse.jgit.storage.file.FileBasedConfig;
70 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
71 import org.eclipse.jgit.treewalk.TreeWalk;
72 import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
73 import org.eclipse.jgit.treewalk.filter.TreeFilter;
74 import org.eclipse.jgit.util.SystemReader;
75 import org.slf4j.Logger;
76 import org.slf4j.LoggerFactory;
77
78 import com.fasterxml.jackson.databind.JsonNode;
79 import com.google.common.annotations.VisibleForTesting;
80 import com.google.common.base.MoreObjects;
81 import com.google.common.collect.ImmutableList;
82 import com.google.common.collect.ImmutableMap;
83
84 import com.linecorp.armeria.common.util.Exceptions;
85 import com.linecorp.armeria.server.ServiceRequestContext;
86 import com.linecorp.centraldogma.common.Author;
87 import com.linecorp.centraldogma.common.CentralDogmaException;
88 import com.linecorp.centraldogma.common.Change;
89 import com.linecorp.centraldogma.common.Commit;
90 import com.linecorp.centraldogma.common.Entry;
91 import com.linecorp.centraldogma.common.EntryNotFoundException;
92 import com.linecorp.centraldogma.common.EntryType;
93 import com.linecorp.centraldogma.common.Markup;
94 import com.linecorp.centraldogma.common.RedundantChangeException;
95 import com.linecorp.centraldogma.common.RepositoryNotFoundException;
96 import com.linecorp.centraldogma.common.Revision;
97 import com.linecorp.centraldogma.common.RevisionNotFoundException;
98 import com.linecorp.centraldogma.common.RevisionRange;
99 import com.linecorp.centraldogma.common.ShuttingDownException;
100 import com.linecorp.centraldogma.internal.Jackson;
101 import com.linecorp.centraldogma.internal.Util;
102 import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch;
103 import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode;
104 import com.linecorp.centraldogma.server.command.CommitResult;
105 import com.linecorp.centraldogma.server.command.ContentTransformer;
106 import com.linecorp.centraldogma.server.internal.IsolatedSystemReader;
107 import com.linecorp.centraldogma.server.internal.JGitUtil;
108 import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
109 import com.linecorp.centraldogma.server.internal.storage.repository.git.Watch.WatchListener;
110 import com.linecorp.centraldogma.server.storage.StorageException;
111 import com.linecorp.centraldogma.server.storage.project.Project;
112 import com.linecorp.centraldogma.server.storage.repository.CacheableCall;
113 import com.linecorp.centraldogma.server.storage.repository.DiffResultType;
114 import com.linecorp.centraldogma.server.storage.repository.FindOption;
115 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
116 import com.linecorp.centraldogma.server.storage.repository.Repository;
117 import com.linecorp.centraldogma.server.storage.repository.RepositoryListener;
118
119
120
121
122 class GitRepository implements Repository {
123
124 private static final Logger logger = LoggerFactory.getLogger(GitRepository.class);
125
126 static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER;
127
128 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
129
130 private static final Field revWalkObjectsField;
131
132 static {
133 final String jgitPomProperties = "META-INF/maven/org.eclipse.jgit/org.eclipse.jgit/pom.properties";
134 try (InputStream is = SystemReader.class.getClassLoader().getResourceAsStream(jgitPomProperties)) {
135 final Properties props = new Properties();
136 props.load(is);
137 final Object jgitVersion = props.get("version");
138 if (jgitVersion != null) {
139 logger.info("Using JGit: {}", jgitVersion);
140 }
141 } catch (IOException e) {
142 logger.debug("Failed to read JGit version", e);
143 }
144
145 IsolatedSystemReader.install();
146
147 Field field = null;
148 try {
149 field = RevWalk.class.getDeclaredField("objects");
150 if (field.getType() != ObjectIdOwnerMap.class) {
151 throw new IllegalStateException(
152 RevWalk.class.getSimpleName() + ".objects is not an " +
153 ObjectIdOwnerMap.class.getSimpleName() + '.');
154 }
155 field.setAccessible(true);
156 } catch (NoSuchFieldException e) {
157 throw new IllegalStateException(
158 RevWalk.class.getSimpleName() + ".objects does not exist.");
159 }
160
161 revWalkObjectsField = field;
162 }
163
164 private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
165 private final Project parent;
166 private final Executor repositoryWorker;
167 private final long creationTimeMillis;
168 private final Author author;
169 @VisibleForTesting
170 @Nullable
171 final RepositoryCache cache;
172 private final String name;
173 private final org.eclipse.jgit.lib.Repository jGitRepository;
174 private final CommitIdDatabase commitIdDatabase;
175 @VisibleForTesting
176 final CommitWatchers commitWatchers = new CommitWatchers();
177 private final AtomicReference<Supplier<CentralDogmaException>> closePending = new AtomicReference<>();
178 private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
179 private final List<RepositoryListener> listeners = new CopyOnWriteArrayList<>();
180
181
182
183
184 private volatile Revision headRevision;
185
186
187
188
189
190
191
192
193
194
195
196 @VisibleForTesting
197 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
198 long creationTimeMillis, Author author) {
199 this(parent, repoDir, repositoryWorker, creationTimeMillis, author, null);
200 }
201
202
203
204
205
206
207
208
209
210
211
212 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
213 long creationTimeMillis, Author author, @Nullable RepositoryCache cache) {
214
215 this.parent = requireNonNull(parent, "parent");
216 name = requireNonNull(repoDir, "repoDir").getName();
217 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
218 this.creationTimeMillis = creationTimeMillis;
219 this.author = requireNonNull(author, "author");
220 this.cache = cache;
221
222 final RepositoryBuilder repositoryBuilder = new RepositoryBuilder().setGitDir(repoDir).setBare();
223 boolean success = false;
224 try {
225
226 try (org.eclipse.jgit.lib.Repository initRepo = repositoryBuilder.build()) {
227 if (exist(repoDir)) {
228 throw new StorageException(
229 "failed to create a repository at: " + repoDir + " (exists already)");
230 }
231 initRepo.create(true);
232
233
234 JGitUtil.applyDefaultsAndSave(initRepo.getConfig());
235 }
236
237
238 jGitRepository = new RepositoryBuilder().setGitDir(repoDir).build();
239
240
241 final RefUpdate head = jGitRepository.updateRef(Constants.HEAD);
242 head.disableRefLog();
243 head.link(Constants.R_HEADS + Constants.MASTER);
244
245
246 commitIdDatabase = new CommitIdDatabase(jGitRepository);
247
248 new CommitExecutor(this, creationTimeMillis, author, "Create a new repository", "",
249 Markup.PLAINTEXT, true)
250 .executeInitialCommit();
251
252 headRevision = Revision.INIT;
253 success = true;
254 } catch (IOException e) {
255 throw new StorageException("failed to create a repository at: " + repoDir, e);
256 } finally {
257 if (!success) {
258 internalClose();
259
260 deleteCruft(repoDir);
261 }
262 }
263 }
264
265
266
267
268
269
270
271
272
273 GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache) {
274 this.parent = requireNonNull(parent, "parent");
275 name = requireNonNull(repoDir, "repoDir").getName();
276 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
277 this.cache = cache;
278
279 final RepositoryBuilder repositoryBuilder = new RepositoryBuilder().setGitDir(repoDir).setBare();
280 try {
281 jGitRepository = repositoryBuilder.build();
282 if (!exist(repoDir)) {
283 throw new RepositoryNotFoundException(repoDir.toString());
284 }
285
286
287 final int formatVersion = jGitRepository.getConfig().getInt(
288 CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 0);
289 if (formatVersion != JGitUtil.REPO_FORMAT_VERSION) {
290 throw new StorageException("unsupported repository format version: " + formatVersion +
291 " (expected: " + JGitUtil.REPO_FORMAT_VERSION + ')');
292 }
293
294
295 JGitUtil.applyDefaultsAndSave(jGitRepository.getConfig());
296 } catch (IOException e) {
297 throw new StorageException("failed to open a repository at: " + repoDir, e);
298 }
299
300 boolean success = false;
301 try {
302 headRevision = uncachedHeadRevision();
303 commitIdDatabase = new CommitIdDatabase(jGitRepository);
304 if (!headRevision.equals(commitIdDatabase.headRevision())) {
305 commitIdDatabase.rebuild(jGitRepository);
306 assert headRevision.equals(commitIdDatabase.headRevision());
307 }
308 final Commit initialCommit = blockingHistory(Revision.INIT, Revision.INIT, ALL_PATH, 1).get(0);
309 creationTimeMillis = initialCommit.when();
310 author = initialCommit.author();
311 success = true;
312 } finally {
313 if (!success) {
314 internalClose();
315 }
316 }
317 }
318
319 private static boolean exist(File repoDir) {
320 try {
321 final RepositoryBuilder repositoryBuilder = new RepositoryBuilder().setGitDir(repoDir);
322 final org.eclipse.jgit.lib.Repository repository = repositoryBuilder.build();
323 if (repository.getConfig() instanceof FileBasedConfig) {
324 return ((FileBasedConfig) repository.getConfig()).getFile().exists();
325 }
326 return repository.getDirectory().exists();
327 } catch (IOException e) {
328 throw new StorageException("failed to check if repository exists at " + repoDir, e);
329 }
330 }
331
332
333
334
335
336
337
338 void close(Supplier<CentralDogmaException> failureCauseSupplier) {
339 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
340 if (closePending.compareAndSet(null, failureCauseSupplier)) {
341 repositoryWorker.execute(() -> {
342
343 rwLock.writeLock().lock();
344 try {
345 if (commitIdDatabase != null) {
346 try {
347 commitIdDatabase.close();
348 } catch (Exception e) {
349 logger.warn("Failed to close a commitId database:", e);
350 }
351 }
352
353 if (jGitRepository != null) {
354 try {
355 jGitRepository.close();
356 } catch (Exception e) {
357 logger.warn("Failed to close a Git repository: {}",
358 jGitRepository.getDirectory(), e);
359 }
360 }
361 } finally {
362 try {
363 rwLock.writeLock().unlock();
364 } finally {
365 commitWatchers.close(failureCauseSupplier);
366 closeFuture.complete(null);
367 }
368 }
369 });
370 }
371
372 closeFuture.join();
373 }
374
375 void internalClose() {
376 close(() -> new CentralDogmaException("should never reach here"));
377 }
378
379 CommitIdDatabase commitIdDatabase() {
380 return commitIdDatabase;
381 }
382
383 @Override
384 public org.eclipse.jgit.lib.Repository jGitRepository() {
385 return jGitRepository;
386 }
387
388 @Override
389 public Project parent() {
390 return parent;
391 }
392
393 @Override
394 public String name() {
395 return name;
396 }
397
398 @Override
399 public long creationTimeMillis() {
400 return creationTimeMillis;
401 }
402
403 @Override
404 public Author author() {
405 return author;
406 }
407
408 @Override
409 public Revision normalizeNow(Revision revision) {
410 return normalizeNow(revision, cachedHeadRevision().major());
411 }
412
413 private static Revision normalizeNow(Revision revision, int baseMajor) {
414 requireNonNull(revision, "revision");
415
416 int major = revision.major();
417
418 if (major >= 0) {
419 if (major > baseMajor) {
420 throw new RevisionNotFoundException(revision);
421 }
422 } else {
423 major = baseMajor + major + 1;
424 if (major <= 0) {
425 throw new RevisionNotFoundException(revision);
426 }
427 }
428
429
430 if (revision.major() == major) {
431 return revision;
432 } else {
433 return new Revision(major);
434 }
435 }
436
437 @Override
438 public RevisionRange normalizeNow(Revision from, Revision to) {
439 final int baseMajor = cachedHeadRevision().major();
440 return new RevisionRange(normalizeNow(from, baseMajor), normalizeNow(to, baseMajor));
441 }
442
443 @Override
444 public CompletableFuture<Map<String, Entry<?>>> find(
445 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
446 final ServiceRequestContext ctx = context();
447 return CompletableFuture.supplyAsync(() -> {
448 failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options);
449 return blockingFind(revision, pathPattern, options);
450 }, repositoryWorker);
451 }
452
453 private Map<String, Entry<?>> blockingFind(
454 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
455
456 requireNonNull(pathPattern, "pathPattern");
457 requireNonNull(revision, "revision");
458 requireNonNull(options, "options");
459
460 final Revision normRevision = normalizeNow(revision);
461 final boolean fetchContent = FindOption.FETCH_CONTENT.get(options);
462 final int maxEntries = FindOption.MAX_ENTRIES.get(options);
463
464 readLock();
465 try (ObjectReader reader = jGitRepository.newObjectReader();
466 TreeWalk treeWalk = new TreeWalk(reader);
467 RevWalk revWalk = newRevWalk(reader)) {
468
469
470 final Revision headRevision = cachedHeadRevision();
471 if (normRevision.compareTo(headRevision) > 0) {
472 return Collections.emptyMap();
473 }
474
475 if ("/".equals(pathPattern)) {
476 return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/"));
477 }
478
479 final Map<String, Entry<?>> result = new LinkedHashMap<>();
480 final ObjectId commitId = commitIdDatabase.get(normRevision);
481 final RevCommit revCommit = revWalk.parseCommit(commitId);
482 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
483
484 final RevTree revTree = revCommit.getTree();
485 treeWalk.addTree(revTree.getId());
486 while (treeWalk.next() && result.size() < maxEntries) {
487 final boolean matches = filter.matches(treeWalk);
488 final String path = '/' + treeWalk.getPathString();
489
490
491 if (treeWalk.isSubtree()) {
492 if (matches) {
493
494 result.put(path, Entry.ofDirectory(normRevision, path));
495 }
496
497 treeWalk.enterSubtree();
498 continue;
499 }
500
501 if (!matches) {
502 continue;
503 }
504
505
506 final Entry<?> entry;
507 final EntryType entryType = EntryType.guessFromPath(path);
508 if (fetchContent) {
509 final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes();
510 switch (entryType) {
511 case JSON:
512 final JsonNode jsonNode = Jackson.readTree(content);
513 entry = Entry.ofJson(normRevision, path, jsonNode);
514 break;
515 case TEXT:
516 final String strVal = sanitizeText(new String(content, UTF_8));
517 entry = Entry.ofText(normRevision, path, strVal);
518 break;
519 default:
520 throw new Error("unexpected entry type: " + entryType);
521 }
522 } else {
523 switch (entryType) {
524 case JSON:
525 entry = Entry.ofJson(normRevision, path, Jackson.nullNode);
526 break;
527 case TEXT:
528 entry = Entry.ofText(normRevision, path, "");
529 break;
530 default:
531 throw new Error("unexpected entry type: " + entryType);
532 }
533 }
534
535 result.put(path, entry);
536 }
537
538 return Util.unsafeCast(result);
539 } catch (CentralDogmaException | IllegalArgumentException e) {
540 throw e;
541 } catch (Exception e) {
542 throw new StorageException(
543 "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern +
544 " for " + revision, e);
545 } finally {
546 readUnlock();
547 }
548 }
549
550 @Override
551 public CompletableFuture<List<Commit>> history(
552 Revision from, Revision to, String pathPattern, int maxCommits) {
553
554 final ServiceRequestContext ctx = context();
555 return CompletableFuture.supplyAsync(() -> {
556 failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits);
557 return blockingHistory(from, to, pathPattern, maxCommits);
558 }, repositoryWorker);
559 }
560
561 @VisibleForTesting
562 List<Commit> blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) {
563 requireNonNull(pathPattern, "pathPattern");
564 requireNonNull(from, "from");
565 requireNonNull(to, "to");
566 if (maxCommits <= 0) {
567 throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)");
568 }
569
570 maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS);
571
572 final RevisionRange range = normalizeNow(from, to);
573 final RevisionRange descendingRange = range.toDescending();
574
575
576 readLock();
577 final RepositoryCache cache =
578
579 (descendingRange.from().major() < headRevision.major() - MAX_MAX_COMMITS * 3) ? null
580 : this.cache;
581 try (ObjectReader objectReader = jGitRepository.newObjectReader();
582 RevWalk revWalk = newRevWalk(new CachingTreeObjectReader(this, objectReader, cache))) {
583 final ObjectIdOwnerMap<?> revWalkInternalMap =
584 (ObjectIdOwnerMap<?>) revWalkObjectsField.get(revWalk);
585
586 final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from());
587 final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to());
588
589 revWalk.markStart(revWalk.parseCommit(fromCommitId));
590 revWalk.setRetainBody(false);
591
592
593
594
595
596
597 final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create(
598 TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern)));
599
600
601
602 final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS);
603
604 final List<Commit> commitList = new ArrayList<>();
605 int numProcessedCommits = 0;
606 for (RevCommit revCommit : revWalk) {
607 numProcessedCommits++;
608
609 if (filter.include(revWalk, revCommit)) {
610 revWalk.parseBody(revCommit);
611 commitList.add(toCommit(revCommit));
612 revCommit.disposeBody();
613 }
614
615 if (revCommit.getId().equals(toCommitId) ||
616 commitList.size() >= maxCommits ||
617
618 numProcessedCommits >= maxNumProcessedCommits) {
619 break;
620 }
621
622
623
624 if (numProcessedCommits % 16 == 0) {
625 revWalkInternalMap.clear();
626 }
627 }
628
629
630
631 if (commitList.size() < maxCommits &&
632 descendingRange.to().major() == 1 &&
633 pathPattern.contains(ALL_PATH)) {
634 try (RevWalk tmpRevWalk = newRevWalk()) {
635 final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId);
636 commitList.add(toCommit(lastRevCommit));
637 }
638 }
639
640 if (!descendingRange.equals(range)) {
641 Collections.reverse(commitList);
642 }
643
644 return commitList;
645 } catch (CentralDogmaException e) {
646 throw e;
647 } catch (Exception e) {
648 throw new StorageException(
649 "failed to retrieve the history: " + parent.name() + '/' + name +
650 " (" + pathPattern + ", " + from + ".." + to + ')', e);
651 } finally {
652 readUnlock();
653 }
654 }
655
656 private static Commit toCommit(RevCommit revCommit) {
657 final Author author;
658 final PersonIdent committerIdent = revCommit.getCommitterIdent();
659 final long when;
660 if (committerIdent == null) {
661 author = Author.UNKNOWN;
662 when = 0;
663 } else {
664 author = new Author(committerIdent.getName(), committerIdent.getEmailAddress());
665 when = committerIdent.getWhen().getTime();
666 }
667
668 try {
669 return CommitUtil.newCommit(author, when, revCommit.getFullMessage());
670 } catch (Exception e) {
671 throw new StorageException("failed to create a Commit", e);
672 }
673 }
674
675 @Override
676 public CompletableFuture<Map<String, Change<?>>> diff(Revision from, Revision to, String pathPattern,
677 DiffResultType diffResultType) {
678 final ServiceRequestContext ctx = context();
679 return CompletableFuture.supplyAsync(() -> {
680 requireNonNull(from, "from");
681 requireNonNull(to, "to");
682 requireNonNull(pathPattern, "pathPattern");
683
684 failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern);
685
686 final RevisionRange range = normalizeNow(from, to).toAscending();
687 readLock();
688 try (RevWalk rw = newRevWalk()) {
689 final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from()));
690 final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to()));
691
692
693
694 return toChangeMap(blockingCompareTreesUncached(
695 treeA, treeB, pathPatternFilterOrTreeFilter(pathPattern)), diffResultType);
696 } catch (StorageException e) {
697 throw e;
698 } catch (Exception e) {
699 throw new StorageException("failed to parse two trees: range=" + range, e);
700 } finally {
701 readUnlock();
702 }
703 }, repositoryWorker);
704 }
705
706 private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) {
707 if (pathPattern == null) {
708 return TreeFilter.ALL;
709 }
710
711 final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern);
712 return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter;
713 }
714
715 @Override
716 public CompletableFuture<Map<String, Change<?>>> previewDiff(Revision baseRevision,
717 Iterable<Change<?>> changes) {
718 final ServiceRequestContext ctx = context();
719 return CompletableFuture.supplyAsync(() -> {
720 failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision);
721 return blockingPreviewDiff(baseRevision, new DefaultChangesApplier(changes));
722 }, repositoryWorker);
723 }
724
725 Map<String, Change<?>> blockingPreviewDiff(Revision baseRevision, AbstractChangesApplier changesApplier) {
726 baseRevision = normalizeNow(baseRevision);
727
728 readLock();
729 try (ObjectReader reader = jGitRepository.newObjectReader();
730 RevWalk revWalk = newRevWalk(reader);
731 DiffFormatter diffFormatter = new DiffFormatter(null)) {
732
733 final ObjectId baseTreeId = toTree(revWalk, baseRevision);
734 final DirCache dirCache = DirCache.newInCore();
735 final int numEdits = changesApplier.apply(jGitRepository, baseRevision, baseTreeId, dirCache);
736 if (numEdits == 0) {
737 return Collections.emptyMap();
738 }
739
740 final CanonicalTreeParser p = new CanonicalTreeParser();
741 p.reset(reader, baseTreeId);
742 diffFormatter.setRepository(jGitRepository);
743 final List<DiffEntry> result = diffFormatter.scan(p, new DirCacheIterator(dirCache));
744 return toChangeMap(result, DiffResultType.NORMAL);
745 } catch (IOException e) {
746 throw new StorageException("failed to perform a dry-run diff", e);
747 } finally {
748 readUnlock();
749 }
750 }
751
752 private Map<String, Change<?>> toChangeMap(List<DiffEntry> diffEntryList, DiffResultType diffResultType) {
753 try (ObjectReader reader = jGitRepository.newObjectReader()) {
754 final Map<String, Change<?>> changeMap = new LinkedHashMap<>();
755
756 for (DiffEntry diffEntry : diffEntryList) {
757 final String oldPath = '/' + diffEntry.getOldPath();
758 final String newPath = '/' + diffEntry.getNewPath();
759
760 switch (diffEntry.getChangeType()) {
761 case MODIFY:
762 final EntryType oldEntryType = EntryType.guessFromPath(oldPath);
763 switch (oldEntryType) {
764 case JSON:
765 if (!oldPath.equals(newPath)) {
766 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
767 }
768
769 final JsonNode oldJsonNode =
770 Jackson.readTree(
771 reader.open(diffEntry.getOldId().toObjectId()).getBytes());
772 final JsonNode newJsonNode =
773 Jackson.readTree(
774 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
775 final JsonPatch patch =
776 JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE);
777
778 if (!patch.isEmpty()) {
779 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
780 putChange(changeMap, newPath,
781 Change.ofJsonUpsert(newPath, newJsonNode));
782 } else {
783 putChange(changeMap, newPath,
784 Change.ofJsonPatch(newPath, Jackson.valueToTree(patch)));
785 }
786 }
787 break;
788 case TEXT:
789 final String oldText = sanitizeText(new String(
790 reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8));
791
792 final String newText = sanitizeText(new String(
793 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
794
795 if (!oldPath.equals(newPath)) {
796 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
797 }
798
799 if (!oldText.equals(newText)) {
800 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
801 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, newText));
802 } else {
803 putChange(changeMap, newPath,
804 Change.ofTextPatch(newPath, oldText, newText));
805 }
806 }
807 break;
808 default:
809 throw new Error("unexpected old entry type: " + oldEntryType);
810 }
811 break;
812 case ADD:
813 final EntryType newEntryType = EntryType.guessFromPath(newPath);
814 switch (newEntryType) {
815 case JSON: {
816 final JsonNode jsonNode = Jackson.readTree(
817 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
818
819 putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode));
820 break;
821 }
822 case TEXT: {
823 final String text = sanitizeText(new String(
824 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
825
826 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text));
827 break;
828 }
829 default:
830 throw new Error("unexpected new entry type: " + newEntryType);
831 }
832 break;
833 case DELETE:
834 putChange(changeMap, oldPath, Change.ofRemoval(oldPath));
835 break;
836 default:
837 throw new Error();
838 }
839 }
840 return changeMap;
841 } catch (Exception e) {
842 throw new StorageException("failed to convert list of DiffEntry to Changes map", e);
843 }
844 }
845
846 private static void putChange(Map<String, Change<?>> changeMap, String path, Change<?> change) {
847 final Change<?> oldChange = changeMap.put(path, change);
848 assert oldChange == null;
849 }
850
851 @Override
852 public CompletableFuture<CommitResult> commit(
853 Revision baseRevision, long commitTimeMillis, Author author, String summary,
854 String detail, Markup markup, Iterable<Change<?>> changes, boolean directExecution) {
855 requireNonNull(baseRevision, "baseRevision");
856 requireNonNull(author, "author");
857 requireNonNull(summary, "summary");
858 requireNonNull(detail, "detail");
859 requireNonNull(markup, "markup");
860 requireNonNull(changes, "changes");
861 final CommitExecutor commitExecutor =
862 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
863 return commit(baseRevision, commitExecutor, normBaseRevision -> {
864 if (!directExecution) {
865 return changes;
866 }
867 return blockingPreviewDiff(normBaseRevision, new DefaultChangesApplier(changes)).values();
868 });
869 }
870
871 @Override
872 public CompletableFuture<CommitResult> commit(Revision baseRevision, long commitTimeMillis, Author author,
873 String summary, String detail, Markup markup,
874 ContentTransformer<?> transformer) {
875 requireNonNull(baseRevision, "baseRevision");
876 requireNonNull(author, "author");
877 requireNonNull(summary, "summary");
878 requireNonNull(detail, "detail");
879 requireNonNull(markup, "markup");
880 requireNonNull(transformer, "transformer");
881 final CommitExecutor commitExecutor =
882 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
883 return commit(baseRevision, commitExecutor,
884 normBaseRevision -> blockingPreviewDiff(
885 normBaseRevision, new TransformingChangesApplier(transformer)).values());
886 }
887
888 private CompletableFuture<CommitResult> commit(
889 Revision baseRevision,
890 CommitExecutor commitExecutor,
891 Function<Revision, Iterable<Change<?>>> applyingChangesProvider) {
892 final ServiceRequestContext ctx = context();
893 return CompletableFuture.supplyAsync(() -> {
894 failFastIfTimedOut(this, logger, ctx, "commit", baseRevision,
895 commitExecutor.author(), commitExecutor.summary());
896 return commitExecutor.execute(baseRevision, applyingChangesProvider);
897 }, repositoryWorker);
898 }
899
900
901
902
903 static String sanitizeText(String text) {
904 if (text.indexOf('\r') >= 0) {
905 text = CR.matcher(text).replaceAll("");
906 }
907 if (!text.isEmpty() && !text.endsWith("\n")) {
908 text += "\n";
909 }
910 return text;
911 }
912
913 static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
914 String ref, ObjectId commitId) throws IOException {
915
916 if (ref.startsWith(Constants.R_TAGS)) {
917 final Ref oldRef = jGitRepository.exactRef(ref);
918 if (oldRef != null) {
919 throw new StorageException("tag ref exists already: " + ref);
920 }
921 }
922
923 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
924 refUpdate.setNewObjectId(commitId);
925
926 final Result res = refUpdate.update(revWalk);
927 switch (res) {
928 case NEW:
929 case FAST_FORWARD:
930
931 break;
932 default:
933 throw new StorageException("unexpected refUpdate state: " + res);
934 }
935 }
936
937 @Override
938 public CompletableFuture<Revision> findLatestRevision(Revision lastKnownRevision, String pathPattern,
939 boolean errorOnEntryNotFound) {
940 requireNonNull(lastKnownRevision, "lastKnownRevision");
941 requireNonNull(pathPattern, "pathPattern");
942
943 final ServiceRequestContext ctx = context();
944 return CompletableFuture.supplyAsync(() -> {
945 failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern);
946 return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound);
947 }, repositoryWorker);
948 }
949
950 @Nullable
951 private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern,
952 boolean errorOnEntryNotFound) {
953 final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD);
954 if (range.from().equals(range.to())) {
955
956 if (!errorOnEntryNotFound) {
957 return null;
958 }
959
960 final Map<String, Entry<?>> entries =
961 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
962 if (!entries.isEmpty()) {
963
964 return null;
965 }
966 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
967 }
968
969 if (range.from().major() == 1) {
970
971 final Map<String, Entry<?>> entries =
972 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
973 if (entries.isEmpty()) {
974 if (!errorOnEntryNotFound) {
975 return null;
976 }
977 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
978 } else {
979 return range.to();
980 }
981 }
982
983
984 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
985
986 final List<DiffEntry> diffEntries;
987 readLock();
988 try (RevWalk revWalk = newRevWalk()) {
989 final RevTree treeA = toTree(revWalk, range.from());
990 final RevTree treeB = toTree(revWalk, range.to());
991 diffEntries = blockingCompareTrees(treeA, treeB);
992 } finally {
993 readUnlock();
994 }
995
996
997 for (DiffEntry e : diffEntries) {
998 final String path;
999 switch (e.getChangeType()) {
1000 case ADD:
1001 path = e.getNewPath();
1002 break;
1003 case MODIFY:
1004 case DELETE:
1005 path = e.getOldPath();
1006 break;
1007 default:
1008 throw new Error();
1009 }
1010
1011 if (filter.matches(path)) {
1012 return range.to();
1013 }
1014 }
1015
1016 if (!errorOnEntryNotFound) {
1017 return null;
1018 }
1019 if (!blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) {
1020
1021
1022 return null;
1023 }
1024 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
1025 }
1026
1027
1028
1029
1030 private List<DiffEntry> blockingCompareTrees(RevTree treeA, RevTree treeB) {
1031 if (cache == null) {
1032 return blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL);
1033 }
1034
1035 final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB);
1036 return cache.get(key).join();
1037 }
1038
1039 List<DiffEntry> blockingCompareTreesUncached(@Nullable RevTree treeA,
1040 @Nullable RevTree treeB,
1041 TreeFilter filter) {
1042 readLock();
1043 try (DiffFormatter diffFormatter = new DiffFormatter(null)) {
1044 diffFormatter.setRepository(jGitRepository);
1045 diffFormatter.setPathFilter(filter);
1046 return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB));
1047 } catch (IOException e) {
1048 throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e);
1049 } finally {
1050 readUnlock();
1051 }
1052 }
1053
1054 @Override
1055 public CompletableFuture<Revision> watch(Revision lastKnownRevision, String pathPattern,
1056 boolean errorOnEntryNotFound) {
1057 requireNonNull(lastKnownRevision, "lastKnownRevision");
1058 requireNonNull(pathPattern, "pathPattern");
1059 final ServiceRequestContext ctx = context();
1060 final Revision normLastKnownRevision = normalizeNow(lastKnownRevision);
1061 final CompletableFuture<Revision> future = new CompletableFuture<>();
1062 CompletableFuture.runAsync(() -> {
1063 failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern);
1064 readLock();
1065 try {
1066
1067
1068 final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern,
1069 errorOnEntryNotFound);
1070 if (latestRevision != null) {
1071 future.complete(latestRevision);
1072 } else {
1073 commitWatchers.add(normLastKnownRevision, pathPattern, future, null);
1074 }
1075 } finally {
1076 readUnlock();
1077 }
1078 }, repositoryWorker).exceptionally(cause -> {
1079 future.completeExceptionally(cause);
1080 return null;
1081 });
1082
1083 return future;
1084 }
1085
1086 private void recursiveWatch(String pathPattern, WatchListener listener) {
1087 requireNonNull(pathPattern, "pathPattern");
1088 CompletableFuture.runAsync(() -> {
1089 final Revision headRevision = this.headRevision;
1090
1091 commitWatchers.add(headRevision, pathPattern, null, listener);
1092 listener.onUpdate(headRevision, null);
1093 }, repositoryWorker);
1094 }
1095
1096 @Override
1097 public <T> CompletableFuture<T> execute(CacheableCall<T> cacheableCall) {
1098
1099 requireNonNull(cacheableCall, "cacheableCall");
1100 final ServiceRequestContext ctx = context();
1101
1102 return CompletableFuture.supplyAsync(() -> {
1103 failFastIfTimedOut(this, logger, ctx, "execute", cacheableCall);
1104 return cacheableCall.execute();
1105 }, repositoryWorker).thenCompose(Function.identity());
1106 }
1107
1108 @Override
1109 public void addListener(RepositoryListener listener) {
1110 listeners.add(listener);
1111
1112 final String pathPattern = listener.pathPattern();
1113 recursiveWatch(pathPattern, (newRevision, cause) -> {
1114 if (shouldStopListening()) {
1115 return;
1116 }
1117
1118 if (cause != null) {
1119 cause = Exceptions.peel(cause);
1120 if (cause instanceof ShuttingDownException) {
1121 return;
1122 }
1123
1124 logger.warn("Failed to watch {} file in {}/{}.", pathPattern, parent.name(), name, cause);
1125 return;
1126 }
1127
1128 try {
1129 assert newRevision != null;
1130
1131 listener.onUpdate(blockingFind(headRevision, pathPattern, ImmutableMap.of()));
1132 } catch (Exception ex) {
1133 logger.warn("Unexpected exception while invoking {}.onUpdate(). listener: {}",
1134 RepositoryListener.class.getSimpleName(), listener, ex);
1135 }
1136 });
1137 }
1138
1139 private boolean shouldStopListening() {
1140 return closePending.get() != null;
1141 }
1142
1143 void notifyWatchers(Revision newRevision, List<DiffEntry> diffEntries) {
1144 for (DiffEntry entry : diffEntries) {
1145 switch (entry.getChangeType()) {
1146 case ADD:
1147 commitWatchers.notify(newRevision, entry.getNewPath());
1148 break;
1149 case MODIFY:
1150 case DELETE:
1151 commitWatchers.notify(newRevision, entry.getOldPath());
1152 break;
1153 default:
1154 throw new Error();
1155 }
1156 }
1157 }
1158
1159 Revision cachedHeadRevision() {
1160 return headRevision;
1161 }
1162
1163 void setHeadRevision(Revision headRevision) {
1164 this.headRevision = headRevision;
1165 }
1166
1167
1168
1169
1170 private Revision uncachedHeadRevision() {
1171 try (RevWalk revWalk = newRevWalk()) {
1172 final ObjectId headRevisionId = jGitRepository.resolve(R_HEADS_MASTER);
1173 if (headRevisionId != null) {
1174 final RevCommit revCommit = revWalk.parseCommit(headRevisionId);
1175 return CommitUtil.extractRevision(revCommit.getFullMessage());
1176 }
1177 } catch (CentralDogmaException e) {
1178 throw e;
1179 } catch (Exception e) {
1180 throw new StorageException("failed to get the current revision", e);
1181 }
1182
1183 throw new StorageException("failed to determine the HEAD: " + parent.name() + '/' + name);
1184 }
1185
1186 private RevTree toTree(RevWalk revWalk, Revision revision) {
1187 return toTree(commitIdDatabase, revWalk, revision);
1188 }
1189
1190 static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) {
1191 final ObjectId commitId = commitIdDatabase.get(revision);
1192 try {
1193 return revWalk.parseCommit(commitId).getTree();
1194 } catch (IOException e) {
1195 throw new StorageException("failed to parse a commit: " + commitId, e);
1196 }
1197 }
1198
1199 private RevWalk newRevWalk() {
1200 final RevWalk revWalk = new RevWalk(jGitRepository);
1201 configureRevWalk(revWalk);
1202 return revWalk;
1203 }
1204
1205 static RevWalk newRevWalk(ObjectReader reader) {
1206 final RevWalk revWalk = new RevWalk(reader);
1207 configureRevWalk(revWalk);
1208 return revWalk;
1209 }
1210
1211 private static void configureRevWalk(RevWalk revWalk) {
1212
1213 revWalk.setRewriteParents(false);
1214 }
1215
1216 private void readLock() {
1217 rwLock.readLock().lock();
1218 if (closePending.get() != null) {
1219 rwLock.readLock().unlock();
1220 throw closePending.get().get();
1221 }
1222 }
1223
1224 private void readUnlock() {
1225 rwLock.readLock().unlock();
1226 }
1227
1228 void writeLock() {
1229 rwLock.writeLock().lock();
1230 if (closePending.get() != null) {
1231 writeUnLock();
1232 throw closePending.get().get();
1233 }
1234 }
1235
1236 void writeUnLock() {
1237 rwLock.writeLock().unlock();
1238 }
1239
1240
1241
1242
1243 public void cloneTo(File newRepoDir) {
1244 cloneTo(newRepoDir, (current, total) -> { });
1245 }
1246
1247
1248
1249
1250 public void cloneTo(File newRepoDir, BiConsumer<Integer, Integer> progressListener) {
1251 requireNonNull(newRepoDir, "newRepoDir");
1252 requireNonNull(progressListener, "progressListener");
1253
1254 final Revision endRevision = normalizeNow(Revision.HEAD);
1255 final GitRepository newRepo = new GitRepository(parent, newRepoDir, repositoryWorker,
1256 creationTimeMillis(), author(), cache);
1257
1258 progressListener.accept(1, endRevision.major());
1259 boolean success = false;
1260 try {
1261
1262 Revision previousNonEmptyRevision = null;
1263 for (int i = 2; i <= endRevision.major();) {
1264
1265 final int batch = 16;
1266 final List<Commit> commits = blockingHistory(
1267 new Revision(i), new Revision(Math.min(endRevision.major(), i + batch - 1)),
1268 ALL_PATH, batch);
1269 checkState(!commits.isEmpty(), "empty commits");
1270
1271 if (previousNonEmptyRevision == null) {
1272 previousNonEmptyRevision = commits.get(0).revision().backward(1);
1273 }
1274 for (Commit c : commits) {
1275 final Revision revision = c.revision();
1276 checkState(revision.major() == i,
1277 "mismatching revision: %s (expected: %s)", revision.major(), i);
1278
1279 final Revision baseRevision = revision.backward(1);
1280 final Collection<Change<?>> changes =
1281 diff(previousNonEmptyRevision, revision, ALL_PATH).join().values();
1282
1283 try {
1284 new CommitExecutor(newRepo, c.when(), c.author(), c.summary(),
1285 c.detail(), c.markup(), false)
1286 .execute(baseRevision, normBaseRevision -> blockingPreviewDiff(
1287 normBaseRevision, new DefaultChangesApplier(changes)).values());
1288 previousNonEmptyRevision = revision;
1289 } catch (RedundantChangeException e) {
1290
1291
1292 new CommitExecutor(newRepo, c.when(), c.author(), c.summary(),
1293 c.detail(), c.markup(), true)
1294 .execute(baseRevision, normBaseRevision -> blockingPreviewDiff(
1295 normBaseRevision, new DefaultChangesApplier(changes)).values());
1296 }
1297
1298 progressListener.accept(i, endRevision.major());
1299 i++;
1300 }
1301 }
1302
1303 success = true;
1304 } finally {
1305 newRepo.internalClose();
1306 if (!success) {
1307 deleteCruft(newRepoDir);
1308 }
1309 }
1310 }
1311
1312 private static void deleteCruft(File repoDir) {
1313 try {
1314 Util.deleteFileTree(repoDir);
1315 } catch (IOException e) {
1316 logger.error("Failed to delete a half-created repository at: {}", repoDir, e);
1317 }
1318 }
1319
1320 @Override
1321 public String toString() {
1322 return MoreObjects.toStringHelper(this)
1323 .add("dir", jGitRepository.getDirectory())
1324 .toString();
1325 }
1326 }