1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage.repository.git;
18
19 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context;
20 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut;
21 import static java.nio.charset.StandardCharsets.UTF_8;
22 import static java.util.Objects.requireNonNull;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.lang.reflect.Field;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.LinkedHashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Properties;
34 import java.util.concurrent.CompletableFuture;
35 import java.util.concurrent.CopyOnWriteArrayList;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.atomic.AtomicReference;
38 import java.util.concurrent.locks.ReadWriteLock;
39 import java.util.concurrent.locks.ReentrantReadWriteLock;
40 import java.util.function.Function;
41 import java.util.function.Supplier;
42 import java.util.regex.Pattern;
43
44 import javax.annotation.Nullable;
45
46 import org.eclipse.jgit.diff.DiffEntry;
47 import org.eclipse.jgit.diff.DiffFormatter;
48 import org.eclipse.jgit.dircache.DirCache;
49 import org.eclipse.jgit.dircache.DirCacheIterator;
50 import org.eclipse.jgit.lib.Constants;
51 import org.eclipse.jgit.lib.ObjectId;
52 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
53 import org.eclipse.jgit.lib.ObjectReader;
54 import org.eclipse.jgit.lib.PersonIdent;
55 import org.eclipse.jgit.lib.RefUpdate;
56 import org.eclipse.jgit.lib.RefUpdate.Result;
57 import org.eclipse.jgit.revwalk.RevCommit;
58 import org.eclipse.jgit.revwalk.RevTree;
59 import org.eclipse.jgit.revwalk.RevWalk;
60 import org.eclipse.jgit.revwalk.TreeRevFilter;
61 import org.eclipse.jgit.revwalk.filter.RevFilter;
62 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
63 import org.eclipse.jgit.treewalk.TreeWalk;
64 import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
65 import org.eclipse.jgit.treewalk.filter.TreeFilter;
66 import org.eclipse.jgit.util.SystemReader;
67 import org.slf4j.Logger;
68 import org.slf4j.LoggerFactory;
69
70 import com.fasterxml.jackson.databind.JsonNode;
71 import com.google.common.annotations.VisibleForTesting;
72 import com.google.common.base.MoreObjects;
73 import com.google.common.collect.ImmutableList;
74 import com.google.common.collect.ImmutableMap;
75
76 import com.linecorp.armeria.common.util.Exceptions;
77 import com.linecorp.armeria.server.ServiceRequestContext;
78 import com.linecorp.centraldogma.common.Author;
79 import com.linecorp.centraldogma.common.CentralDogmaException;
80 import com.linecorp.centraldogma.common.Change;
81 import com.linecorp.centraldogma.common.Commit;
82 import com.linecorp.centraldogma.common.Entry;
83 import com.linecorp.centraldogma.common.EntryNotFoundException;
84 import com.linecorp.centraldogma.common.EntryType;
85 import com.linecorp.centraldogma.common.Markup;
86 import com.linecorp.centraldogma.common.Revision;
87 import com.linecorp.centraldogma.common.RevisionNotFoundException;
88 import com.linecorp.centraldogma.common.RevisionRange;
89 import com.linecorp.centraldogma.common.ShuttingDownException;
90 import com.linecorp.centraldogma.internal.Jackson;
91 import com.linecorp.centraldogma.internal.Util;
92 import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch;
93 import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode;
94 import com.linecorp.centraldogma.server.command.CommitResult;
95 import com.linecorp.centraldogma.server.command.ContentTransformer;
96 import com.linecorp.centraldogma.server.internal.IsolatedSystemReader;
97 import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
98 import com.linecorp.centraldogma.server.internal.storage.repository.git.Watch.WatchListener;
99 import com.linecorp.centraldogma.server.internal.storage.repository.git.rocksdb.RocksDbRepository;
100 import com.linecorp.centraldogma.server.storage.StorageException;
101 import com.linecorp.centraldogma.server.storage.project.Project;
102 import com.linecorp.centraldogma.server.storage.repository.CacheableCall;
103 import com.linecorp.centraldogma.server.storage.repository.DiffResultType;
104 import com.linecorp.centraldogma.server.storage.repository.FindOption;
105 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
106 import com.linecorp.centraldogma.server.storage.repository.Repository;
107 import com.linecorp.centraldogma.server.storage.repository.RepositoryListener;
108
109
110
111
112 class GitRepository implements Repository {
113
114 private static final Logger logger = LoggerFactory.getLogger(GitRepository.class);
115
116 static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER;
117
118 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
119
120 private static final Field revWalkObjectsField;
121
122 static {
123 final String jgitPomProperties = "META-INF/maven/org.eclipse.jgit/org.eclipse.jgit/pom.properties";
124 try (InputStream is = SystemReader.class.getClassLoader().getResourceAsStream(jgitPomProperties)) {
125 final Properties props = new Properties();
126 props.load(is);
127 final Object jgitVersion = props.get("version");
128 if (jgitVersion != null) {
129 logger.info("Using JGit: {}", jgitVersion);
130 }
131 } catch (IOException e) {
132 logger.debug("Failed to read JGit version", e);
133 }
134
135 IsolatedSystemReader.install();
136
137 Field field = null;
138 try {
139 field = RevWalk.class.getDeclaredField("objects");
140 if (field.getType() != ObjectIdOwnerMap.class) {
141 throw new IllegalStateException(
142 RevWalk.class.getSimpleName() + ".objects is not an " +
143 ObjectIdOwnerMap.class.getSimpleName() + '.');
144 }
145 field.setAccessible(true);
146 } catch (NoSuchFieldException e) {
147 throw new IllegalStateException(
148 RevWalk.class.getSimpleName() + ".objects does not exist.");
149 }
150
151 revWalkObjectsField = field;
152 }
153
154 private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
155 private final Project parent;
156 private final Executor repositoryWorker;
157 private final long creationTimeMillis;
158 private final Author author;
159 @VisibleForTesting
160 @Nullable
161 final RepositoryCache cache;
162 private final File repoDir;
163 private final String name;
164 private final org.eclipse.jgit.lib.Repository jGitRepository;
165 private final boolean isEncrypted;
166 private final CommitIdDatabase commitIdDatabase;
167 @VisibleForTesting
168 final CommitWatchers commitWatchers = new CommitWatchers();
169 private final AtomicReference<Supplier<CentralDogmaException>> closePending = new AtomicReference<>();
170 private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
171 private final List<RepositoryListener> listeners = new CopyOnWriteArrayList<>();
172
173
174
175
176 private volatile Revision headRevision;
177
178
179
180
181 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
182 long creationTimeMillis, Author author, @Nullable RepositoryCache cache,
183 org.eclipse.jgit.lib.Repository jGitRepository, CommitIdDatabase commitIdDatabase) {
184 this.parent = parent;
185 this.repoDir = repoDir;
186 name = repoDir.getName();
187 this.repositoryWorker = repositoryWorker;
188 this.creationTimeMillis = creationTimeMillis;
189 this.author = author;
190 this.cache = cache;
191 this.jGitRepository = jGitRepository;
192 isEncrypted = jGitRepository instanceof RocksDbRepository;
193 this.commitIdDatabase = commitIdDatabase;
194 new CommitExecutor(this, creationTimeMillis, author, "Create a new repository", "",
195 Markup.PLAINTEXT, true)
196 .executeInitialCommit();
197
198 headRevision = Revision.INIT;
199 }
200
201
202
203
204 GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache,
205 org.eclipse.jgit.lib.Repository jGitRepository, CommitIdDatabase commitIdDatabase,
206 Revision headRevision) {
207 this.parent = requireNonNull(parent, "parent");
208 this.repoDir = requireNonNull(repoDir, "repoDir");
209 name = requireNonNull(repoDir, "repoDir").getName();
210 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
211 this.cache = cache;
212 this.jGitRepository = requireNonNull(jGitRepository, "jGitRepository");
213 isEncrypted = jGitRepository instanceof RocksDbRepository;
214 this.commitIdDatabase = requireNonNull(commitIdDatabase, "commitIdDatabase");
215 this.headRevision = requireNonNull(headRevision, "headRevision");
216 final Commit initialCommit = blockingHistory(Revision.INIT, Revision.INIT, ALL_PATH, 1).get(0);
217 creationTimeMillis = initialCommit.when();
218 author = initialCommit.author();
219 }
220
221
222
223
224
225
226
227 void close(Supplier<CentralDogmaException> failureCauseSupplier) {
228 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
229 if (closePending.compareAndSet(null, failureCauseSupplier)) {
230 repositoryWorker.execute(() -> {
231
232 rwLock.writeLock().lock();
233 try {
234 closeRepository(commitIdDatabase, jGitRepository);
235 } finally {
236 try {
237 rwLock.writeLock().unlock();
238 } finally {
239 commitWatchers.close(failureCauseSupplier);
240 closeFuture.complete(null);
241 }
242 }
243 });
244 }
245
246 closeFuture.join();
247 }
248
249 static void closeRepository(@Nullable CommitIdDatabase commitIdDatabase,
250 @Nullable org.eclipse.jgit.lib.Repository jGitRepository) {
251 if (commitIdDatabase != null) {
252 try {
253 commitIdDatabase.close();
254 } catch (Exception e) {
255 logger.warn("Failed to close a commitId database:", e);
256 }
257 }
258
259 if (jGitRepository != null) {
260 try {
261 jGitRepository.close();
262 } catch (Exception e) {
263 logger.warn("Failed to close a Git repository: {}", jGitRepository.getDirectory(), e);
264 }
265 }
266 }
267
268 void internalClose() {
269 close(() -> new CentralDogmaException("should never reach here"));
270 }
271
272 CommitIdDatabase commitIdDatabase() {
273 return commitIdDatabase;
274 }
275
276 @Override
277 public org.eclipse.jgit.lib.Repository jGitRepository() {
278 return jGitRepository;
279 }
280
281 @Override
282 public Project parent() {
283 return parent;
284 }
285
286 @Override
287 public File repoDir() {
288 return repoDir;
289 }
290
291 @Override
292 public String name() {
293 return name;
294 }
295
296 @Override
297 public long creationTimeMillis() {
298 return creationTimeMillis;
299 }
300
301 @Override
302 public Author author() {
303 return author;
304 }
305
306 @Override
307 public Revision normalizeNow(Revision revision) {
308 return normalizeNow(revision, cachedHeadRevision().major());
309 }
310
311 private static Revision normalizeNow(Revision revision, int baseMajor) {
312 requireNonNull(revision, "revision");
313
314 int major = revision.major();
315
316 if (major >= 0) {
317 if (major > baseMajor) {
318 throw new RevisionNotFoundException(revision);
319 }
320 } else {
321 major = baseMajor + major + 1;
322 if (major <= 0) {
323 throw new RevisionNotFoundException(revision);
324 }
325 }
326
327
328 if (revision.major() == major) {
329 return revision;
330 } else {
331 return new Revision(major);
332 }
333 }
334
335 @Override
336 public RevisionRange normalizeNow(Revision from, Revision to) {
337 final int baseMajor = cachedHeadRevision().major();
338 return new RevisionRange(normalizeNow(from, baseMajor), normalizeNow(to, baseMajor));
339 }
340
341 @Override
342 public CompletableFuture<Map<String, Entry<?>>> find(
343 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
344 final ServiceRequestContext ctx = context();
345 return CompletableFuture.supplyAsync(() -> {
346 failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options);
347 return blockingFind(revision, pathPattern, options);
348 }, repositoryWorker);
349 }
350
351 private Map<String, Entry<?>> blockingFind(
352 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
353
354 requireNonNull(pathPattern, "pathPattern");
355 requireNonNull(revision, "revision");
356 requireNonNull(options, "options");
357
358 final Revision normRevision = normalizeNow(revision);
359 final boolean fetchContent = FindOption.FETCH_CONTENT.get(options);
360 final int maxEntries = FindOption.MAX_ENTRIES.get(options);
361
362 readLock();
363 try (ObjectReader reader = jGitRepository.newObjectReader();
364 TreeWalk treeWalk = new TreeWalk(reader);
365 RevWalk revWalk = newRevWalk(reader)) {
366
367
368 final Revision headRevision = cachedHeadRevision();
369 if (normRevision.compareTo(headRevision) > 0) {
370 return Collections.emptyMap();
371 }
372
373 if ("/".equals(pathPattern)) {
374 return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/"));
375 }
376
377 final Map<String, Entry<?>> result = new LinkedHashMap<>();
378 final ObjectId commitId = commitIdDatabase.get(normRevision);
379 final RevCommit revCommit = revWalk.parseCommit(commitId);
380 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
381
382 final RevTree revTree = revCommit.getTree();
383 treeWalk.addTree(revTree.getId());
384 while (treeWalk.next() && result.size() < maxEntries) {
385 final boolean matches = filter.matches(treeWalk);
386 final String path = '/' + treeWalk.getPathString();
387
388
389 if (treeWalk.isSubtree()) {
390 if (matches) {
391
392 result.put(path, Entry.ofDirectory(normRevision, path));
393 }
394
395 treeWalk.enterSubtree();
396 continue;
397 }
398
399 if (!matches) {
400 continue;
401 }
402
403
404 final Entry<?> entry;
405 final EntryType entryType = EntryType.guessFromPath(path);
406 if (fetchContent) {
407 final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes();
408 final String string = new String(content, UTF_8);
409 switch (entryType) {
410 case JSON:
411 entry = Entry.ofJson(normRevision, path, string);
412 break;
413 case YAML:
414 entry = Entry.ofYaml(normRevision, path, string);
415 break;
416 case TEXT:
417 final String strVal = sanitizeText(string);
418 entry = Entry.ofText(normRevision, path, strVal);
419 break;
420 default:
421 throw new Error("unexpected entry type: " + entryType);
422 }
423 } else {
424 switch (entryType) {
425 case JSON:
426 entry = Entry.ofJson(normRevision, path, "");
427 break;
428 case YAML:
429 entry = Entry.ofYaml(normRevision, path, "");
430 break;
431 case TEXT:
432 entry = Entry.ofText(normRevision, path, "");
433 break;
434 default:
435 throw new Error("unexpected entry type: " + entryType);
436 }
437 }
438
439 result.put(path, entry);
440 }
441
442 return Util.unsafeCast(result);
443 } catch (CentralDogmaException | IllegalArgumentException e) {
444 throw e;
445 } catch (Exception e) {
446 throw new StorageException(
447 "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern +
448 " for " + revision, e);
449 } finally {
450 readUnlock();
451 }
452 }
453
454 @Override
455 public CompletableFuture<List<Commit>> history(
456 Revision from, Revision to, String pathPattern, int maxCommits) {
457
458 final ServiceRequestContext ctx = context();
459 return CompletableFuture.supplyAsync(() -> {
460 failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits);
461 return blockingHistory(from, to, pathPattern, maxCommits);
462 }, repositoryWorker);
463 }
464
465 @VisibleForTesting
466 List<Commit> blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) {
467 requireNonNull(pathPattern, "pathPattern");
468 requireNonNull(from, "from");
469 requireNonNull(to, "to");
470 if (maxCommits <= 0) {
471 throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)");
472 }
473
474 maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS);
475
476 final RevisionRange range = normalizeNow(from, to);
477 final RevisionRange descendingRange = range.toDescending();
478
479
480 readLock();
481 final RepositoryCache cache =
482
483 (descendingRange.from().major() < headRevision.major() - MAX_MAX_COMMITS * 3) ? null
484 : this.cache;
485 try (ObjectReader objectReader = jGitRepository.newObjectReader();
486 RevWalk revWalk = newRevWalk(new CachingTreeObjectReader(this, objectReader, cache))) {
487 final ObjectIdOwnerMap<?> revWalkInternalMap =
488 (ObjectIdOwnerMap<?>) revWalkObjectsField.get(revWalk);
489
490 final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from());
491 final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to());
492
493 revWalk.markStart(revWalk.parseCommit(fromCommitId));
494 revWalk.setRetainBody(false);
495
496
497
498
499
500
501 final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create(
502 TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern)));
503
504
505
506 final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS);
507
508 final List<Commit> commitList = new ArrayList<>();
509 int numProcessedCommits = 0;
510 for (RevCommit revCommit : revWalk) {
511 numProcessedCommits++;
512
513 if (filter.include(revWalk, revCommit)) {
514 revWalk.parseBody(revCommit);
515 commitList.add(toCommit(revCommit));
516 revCommit.disposeBody();
517 }
518
519 if (revCommit.getId().equals(toCommitId) ||
520 commitList.size() >= maxCommits ||
521
522 numProcessedCommits >= maxNumProcessedCommits) {
523 break;
524 }
525
526
527
528 if (numProcessedCommits % 16 == 0) {
529 revWalkInternalMap.clear();
530 }
531 }
532
533
534
535 if (commitList.size() < maxCommits &&
536 descendingRange.to().major() == 1 &&
537 pathPattern.contains(ALL_PATH)) {
538 try (RevWalk tmpRevWalk = newRevWalk()) {
539 final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId);
540 commitList.add(toCommit(lastRevCommit));
541 }
542 }
543
544 if (!descendingRange.equals(range)) {
545 Collections.reverse(commitList);
546 }
547
548 return commitList;
549 } catch (CentralDogmaException e) {
550 throw e;
551 } catch (Exception e) {
552 throw new StorageException(
553 "failed to retrieve the history: " + parent.name() + '/' + name +
554 " (" + pathPattern + ", " + from + ".." + to + ')', e);
555 } finally {
556 readUnlock();
557 }
558 }
559
560 private static Commit toCommit(RevCommit revCommit) {
561 final Author author;
562 final PersonIdent committerIdent = revCommit.getCommitterIdent();
563 final long when;
564 if (committerIdent == null) {
565 author = Author.UNKNOWN;
566 when = 0;
567 } else {
568 author = new Author(committerIdent.getName(), committerIdent.getEmailAddress());
569 when = committerIdent.getWhen().getTime();
570 }
571
572 try {
573 return CommitUtil.newCommit(author, when, revCommit.getFullMessage());
574 } catch (Exception e) {
575 throw new StorageException("failed to create a Commit", e);
576 }
577 }
578
579 @Override
580 public CompletableFuture<Map<String, Change<?>>> diff(Revision from, Revision to, String pathPattern,
581 DiffResultType diffResultType) {
582 final ServiceRequestContext ctx = context();
583 return CompletableFuture.supplyAsync(() -> {
584 requireNonNull(from, "from");
585 requireNonNull(to, "to");
586 requireNonNull(pathPattern, "pathPattern");
587
588 failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern);
589
590 final RevisionRange range = normalizeNow(from, to).toAscending();
591 readLock();
592 try (RevWalk rw = newRevWalk()) {
593 final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from()));
594 final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to()));
595
596
597
598 return toChangeMap(blockingCompareTreesUncached(
599 treeA, treeB, pathPatternFilterOrTreeFilter(pathPattern)), diffResultType);
600 } catch (StorageException e) {
601 throw e;
602 } catch (Exception e) {
603 throw new StorageException("failed to parse two trees: range=" + range, e);
604 } finally {
605 readUnlock();
606 }
607 }, repositoryWorker);
608 }
609
610 private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) {
611 if (pathPattern == null) {
612 return TreeFilter.ALL;
613 }
614
615 final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern);
616 return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter;
617 }
618
619 @Override
620 public CompletableFuture<Map<String, Change<?>>> previewDiff(Revision baseRevision,
621 Iterable<Change<?>> changes) {
622 final ServiceRequestContext ctx = context();
623 return CompletableFuture.supplyAsync(() -> {
624 failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision);
625 return blockingPreviewDiff(baseRevision, new DefaultChangesApplier(changes));
626 }, repositoryWorker);
627 }
628
629 Map<String, Change<?>> blockingPreviewDiff(Revision baseRevision, AbstractChangesApplier changesApplier) {
630 baseRevision = normalizeNow(baseRevision);
631
632 readLock();
633 try (ObjectReader reader = jGitRepository.newObjectReader();
634 RevWalk revWalk = newRevWalk(reader);
635 DiffFormatter diffFormatter = new DiffFormatter(null)) {
636
637 final ObjectId baseTreeId = toTree(revWalk, baseRevision);
638 final DirCache dirCache = DirCache.newInCore();
639 final int numEdits = changesApplier.apply(jGitRepository, baseRevision, baseTreeId, dirCache);
640 if (numEdits == 0) {
641 return Collections.emptyMap();
642 }
643
644 final CanonicalTreeParser p = new CanonicalTreeParser();
645 p.reset(reader, baseTreeId);
646 diffFormatter.setRepository(jGitRepository);
647 final List<DiffEntry> result = diffFormatter.scan(p, new DirCacheIterator(dirCache));
648 return toChangeMap(result, DiffResultType.NORMAL);
649 } catch (IOException e) {
650 throw new StorageException("failed to perform a dry-run diff", e);
651 } finally {
652 readUnlock();
653 }
654 }
655
656 private Map<String, Change<?>> toChangeMap(List<DiffEntry> diffEntryList, DiffResultType diffResultType) {
657 try (ObjectReader reader = jGitRepository.newObjectReader()) {
658 final Map<String, Change<?>> changeMap = new LinkedHashMap<>();
659
660 for (DiffEntry diffEntry : diffEntryList) {
661 final String oldPath = '/' + diffEntry.getOldPath();
662 final String newPath = '/' + diffEntry.getNewPath();
663
664 switch (diffEntry.getChangeType()) {
665 case MODIFY:
666 final EntryType oldEntryType = EntryType.guessFromPath(oldPath);
667 switch (oldEntryType) {
668 case JSON:
669 if (!oldPath.equals(newPath)) {
670 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
671 }
672
673 final JsonNode oldJsonNode =
674 Jackson.readTree(
675 reader.open(diffEntry.getOldId().toObjectId()).getBytes());
676 final JsonNode newJsonNode =
677 Jackson.readTree(
678 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
679 final JsonPatch patch =
680 JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE);
681
682 if (!patch.isEmpty()) {
683 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
684 putChange(changeMap, newPath,
685 Change.ofJsonUpsert(newPath, newJsonNode));
686 } else {
687 putChange(changeMap, newPath,
688 Change.ofJsonPatch(newPath, Jackson.valueToTree(patch)));
689 }
690 }
691 break;
692 case TEXT:
693 final String oldText = sanitizeText(new String(
694 reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8));
695
696 final String newText = sanitizeText(new String(
697 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
698
699 if (!oldPath.equals(newPath)) {
700 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
701 }
702
703 if (!oldText.equals(newText)) {
704 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
705 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, newText));
706 } else {
707 putChange(changeMap, newPath,
708 Change.ofTextPatch(newPath, oldText, newText));
709 }
710 }
711 break;
712 default:
713 throw new Error("unexpected old entry type: " + oldEntryType);
714 }
715 break;
716 case ADD:
717 final EntryType newEntryType = EntryType.guessFromPath(newPath);
718 switch (newEntryType) {
719 case JSON: {
720 final JsonNode jsonNode = Jackson.readTree(
721 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
722
723 putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode));
724 break;
725 }
726 case TEXT: {
727 final String text = sanitizeText(new String(
728 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
729
730 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text));
731 break;
732 }
733 default:
734 throw new Error("unexpected new entry type: " + newEntryType);
735 }
736 break;
737 case DELETE:
738 putChange(changeMap, oldPath, Change.ofRemoval(oldPath));
739 break;
740 default:
741 throw new Error();
742 }
743 }
744 return changeMap;
745 } catch (Exception e) {
746 throw new StorageException("failed to convert list of DiffEntry to Changes map", e);
747 }
748 }
749
750 private static void putChange(Map<String, Change<?>> changeMap, String path, Change<?> change) {
751 final Change<?> oldChange = changeMap.put(path, change);
752 assert oldChange == null;
753 }
754
755 @Override
756 public CompletableFuture<CommitResult> commit(
757 Revision baseRevision, long commitTimeMillis, Author author, String summary,
758 String detail, Markup markup, Iterable<Change<?>> changes, boolean directExecution) {
759 requireNonNull(baseRevision, "baseRevision");
760 requireNonNull(author, "author");
761 requireNonNull(summary, "summary");
762 requireNonNull(detail, "detail");
763 requireNonNull(markup, "markup");
764 requireNonNull(changes, "changes");
765 final CommitExecutor commitExecutor =
766 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
767 return commit(baseRevision, commitExecutor, normBaseRevision -> changes);
768 }
769
770 @Override
771 public CompletableFuture<CommitResult> commit(Revision baseRevision, long commitTimeMillis, Author author,
772 String summary, String detail, Markup markup,
773 ContentTransformer<?> transformer) {
774 requireNonNull(baseRevision, "baseRevision");
775 requireNonNull(author, "author");
776 requireNonNull(summary, "summary");
777 requireNonNull(detail, "detail");
778 requireNonNull(markup, "markup");
779 requireNonNull(transformer, "transformer");
780 final CommitExecutor commitExecutor =
781 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
782 return commit(baseRevision, commitExecutor,
783 normBaseRevision -> blockingPreviewDiff(
784 normBaseRevision, new TransformingChangesApplier(transformer)).values());
785 }
786
787 private CompletableFuture<CommitResult> commit(
788 Revision baseRevision,
789 CommitExecutor commitExecutor,
790 Function<Revision, Iterable<Change<?>>> applyingChangesProvider) {
791 final ServiceRequestContext ctx = context();
792 return CompletableFuture.supplyAsync(() -> {
793 failFastIfTimedOut(this, logger, ctx, "commit", baseRevision,
794 commitExecutor.author(), commitExecutor.summary());
795 return commitExecutor.execute(baseRevision, applyingChangesProvider);
796 }, repositoryWorker);
797 }
798
799
800
801
802 static String sanitizeText(String text) {
803 if (text.indexOf('\r') >= 0) {
804 text = CR.matcher(text).replaceAll("");
805 }
806 if (!text.isEmpty() && !text.endsWith("\n")) {
807 text += "\n";
808 }
809 return text;
810 }
811
812 static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
813 String ref, ObjectId commitId) throws IOException {
814 if (ref.startsWith(Constants.R_TAGS)) {
815 throw new StorageException("Using a tag is not allowed. ref: " + ref);
816 }
817
818 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
819 refUpdate.setNewObjectId(commitId);
820
821 final Result res = refUpdate.update(revWalk);
822 switch (res) {
823 case NEW:
824 case FAST_FORWARD:
825
826 break;
827 default:
828 throw new StorageException("unexpected refUpdate state: " + res);
829 }
830 }
831
832 @Override
833 public CompletableFuture<Revision> findLatestRevision(Revision lastKnownRevision, String pathPattern,
834 boolean errorOnEntryNotFound) {
835 requireNonNull(lastKnownRevision, "lastKnownRevision");
836 requireNonNull(pathPattern, "pathPattern");
837
838 final ServiceRequestContext ctx = context();
839 return CompletableFuture.supplyAsync(() -> {
840 failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern);
841 return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound);
842 }, repositoryWorker);
843 }
844
845 @Nullable
846 private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern,
847 boolean errorOnEntryNotFound) {
848 final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD);
849 if (range.from().equals(range.to())) {
850
851 if (!errorOnEntryNotFound) {
852 return null;
853 }
854
855 final Map<String, Entry<?>> entries =
856 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
857 if (!entries.isEmpty()) {
858
859 return null;
860 }
861 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
862 }
863
864 if (range.from().major() == 1) {
865
866 final Map<String, Entry<?>> entries =
867 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
868 if (entries.isEmpty()) {
869 if (!errorOnEntryNotFound) {
870 return null;
871 }
872 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
873 } else {
874 return range.to();
875 }
876 }
877
878
879 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
880
881 final List<DiffEntry> diffEntries;
882 readLock();
883 try (RevWalk revWalk = newRevWalk()) {
884 final RevTree treeA = toTree(revWalk, range.from());
885 final RevTree treeB = toTree(revWalk, range.to());
886 diffEntries = blockingCompareTrees(treeA, treeB);
887 } finally {
888 readUnlock();
889 }
890
891
892 for (DiffEntry e : diffEntries) {
893 final String path;
894 switch (e.getChangeType()) {
895 case ADD:
896 path = e.getNewPath();
897 break;
898 case MODIFY:
899 case DELETE:
900 path = e.getOldPath();
901 break;
902 default:
903 throw new Error();
904 }
905
906 if (filter.matches(path)) {
907 return range.to();
908 }
909 }
910
911 if (!errorOnEntryNotFound) {
912 return null;
913 }
914 if (!blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) {
915
916
917 return null;
918 }
919 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
920 }
921
922
923
924
925 private List<DiffEntry> blockingCompareTrees(RevTree treeA, RevTree treeB) {
926 if (cache == null) {
927 return blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL);
928 }
929
930 final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB);
931 return cache.get(key).join();
932 }
933
934 List<DiffEntry> blockingCompareTreesUncached(@Nullable RevTree treeA,
935 @Nullable RevTree treeB,
936 TreeFilter filter) {
937 readLock();
938 try (DiffFormatter diffFormatter = new DiffFormatter(null)) {
939 diffFormatter.setRepository(jGitRepository);
940 diffFormatter.setPathFilter(filter);
941 return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB));
942 } catch (IOException e) {
943 throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e);
944 } finally {
945 readUnlock();
946 }
947 }
948
949 @Override
950 public CompletableFuture<Revision> watch(Revision lastKnownRevision, String pathPattern,
951 boolean errorOnEntryNotFound) {
952 requireNonNull(lastKnownRevision, "lastKnownRevision");
953 requireNonNull(pathPattern, "pathPattern");
954 final ServiceRequestContext ctx = context();
955 final Revision normLastKnownRevision = normalizeNow(lastKnownRevision);
956 final CompletableFuture<Revision> future = new CompletableFuture<>();
957 CompletableFuture.runAsync(() -> {
958 failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern);
959 readLock();
960 try {
961
962
963 final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern,
964 errorOnEntryNotFound);
965 if (latestRevision != null) {
966 future.complete(latestRevision);
967 } else {
968 commitWatchers.add(normLastKnownRevision, pathPattern, future, null);
969 }
970 } finally {
971 readUnlock();
972 }
973 }, repositoryWorker).exceptionally(cause -> {
974 future.completeExceptionally(cause);
975 return null;
976 });
977
978 return future;
979 }
980
981 private void recursiveWatch(String pathPattern, WatchListener listener) {
982 requireNonNull(pathPattern, "pathPattern");
983 CompletableFuture.runAsync(() -> {
984 final Revision headRevision = this.headRevision;
985
986 commitWatchers.add(headRevision, pathPattern, null, listener);
987 listener.onUpdate(headRevision, null);
988 }, repositoryWorker);
989 }
990
991 @Override
992 public <T> CompletableFuture<T> execute(CacheableCall<T> cacheableCall) {
993
994 requireNonNull(cacheableCall, "cacheableCall");
995 final ServiceRequestContext ctx = context();
996
997 return CompletableFuture.supplyAsync(() -> {
998 failFastIfTimedOut(this, logger, ctx, "execute", cacheableCall);
999 return cacheableCall.execute();
1000 }, repositoryWorker).thenCompose(Function.identity());
1001 }
1002
1003 @Override
1004 public void addListener(RepositoryListener listener) {
1005 listeners.add(listener);
1006
1007 final String pathPattern = listener.pathPattern();
1008 recursiveWatch(pathPattern, (newRevision, cause) -> {
1009 if (shouldStopListening()) {
1010 return;
1011 }
1012
1013 if (cause != null) {
1014 cause = Exceptions.peel(cause);
1015 if (cause instanceof ShuttingDownException) {
1016 return;
1017 }
1018
1019 logger.warn("Failed to watch {} file in {}/{}.", pathPattern, parent.name(), name, cause);
1020 return;
1021 }
1022
1023 try {
1024 assert newRevision != null;
1025
1026 listener.onUpdate(blockingFind(headRevision, pathPattern, ImmutableMap.of()));
1027 } catch (Exception ex) {
1028 logger.warn("Unexpected exception while invoking {}.onUpdate(). listener: {}",
1029 RepositoryListener.class.getSimpleName(), listener, ex);
1030 }
1031 });
1032 }
1033
1034 private boolean shouldStopListening() {
1035 return closePending.get() != null;
1036 }
1037
1038 void notifyWatchers(Revision newRevision, List<DiffEntry> diffEntries) {
1039 for (DiffEntry entry : diffEntries) {
1040 switch (entry.getChangeType()) {
1041 case ADD:
1042 commitWatchers.notify(newRevision, entry.getNewPath());
1043 break;
1044 case MODIFY:
1045 case DELETE:
1046 commitWatchers.notify(newRevision, entry.getOldPath());
1047 break;
1048 default:
1049 throw new Error();
1050 }
1051 }
1052 }
1053
1054 Revision cachedHeadRevision() {
1055 return headRevision;
1056 }
1057
1058 void setHeadRevision(Revision headRevision) {
1059 this.headRevision = headRevision;
1060 }
1061
1062 private RevTree toTree(RevWalk revWalk, Revision revision) {
1063 return toTree(commitIdDatabase, revWalk, revision);
1064 }
1065
1066 static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) {
1067 final ObjectId commitId = commitIdDatabase.get(revision);
1068 try {
1069 return revWalk.parseCommit(commitId).getTree();
1070 } catch (IOException e) {
1071 throw new StorageException("failed to parse a commit: " + commitId, e);
1072 }
1073 }
1074
1075 private RevWalk newRevWalk() {
1076 final RevWalk revWalk = new RevWalk(jGitRepository);
1077 configureRevWalk(revWalk);
1078 return revWalk;
1079 }
1080
1081 static RevWalk newRevWalk(ObjectReader reader) {
1082 final RevWalk revWalk = new RevWalk(reader);
1083 configureRevWalk(revWalk);
1084 return revWalk;
1085 }
1086
1087 private static void configureRevWalk(RevWalk revWalk) {
1088
1089 revWalk.setRewriteParents(false);
1090 }
1091
1092 private void readLock() {
1093 rwLock.readLock().lock();
1094 if (closePending.get() != null) {
1095 rwLock.readLock().unlock();
1096 throw closePending.get().get();
1097 }
1098 }
1099
1100 private void readUnlock() {
1101 rwLock.readLock().unlock();
1102 }
1103
1104 void writeLock() {
1105 rwLock.writeLock().lock();
1106 if (closePending.get() != null) {
1107 writeUnLock();
1108 throw closePending.get().get();
1109 }
1110 }
1111
1112 void writeUnLock() {
1113 rwLock.writeLock().unlock();
1114 }
1115
1116 static void deleteCruft(File repoDir) {
1117 try {
1118 Util.deleteFileTree(repoDir);
1119 } catch (IOException e) {
1120 logger.error("Failed to delete a half-created repository at: {}", repoDir, e);
1121 }
1122 }
1123
1124 @Override
1125 public boolean isEncrypted() {
1126 return isEncrypted;
1127 }
1128
1129 @Override
1130 public String toString() {
1131 return MoreObjects.toStringHelper(this)
1132 .add("dir", jGitRepository.getDirectory())
1133 .toString();
1134 }
1135 }