1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage.repository.git;
18
19 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context;
20 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut;
21 import static java.nio.charset.StandardCharsets.UTF_8;
22 import static java.util.Objects.requireNonNull;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.lang.reflect.Field;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.LinkedHashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Properties;
34 import java.util.concurrent.CompletableFuture;
35 import java.util.concurrent.CopyOnWriteArrayList;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.atomic.AtomicReference;
38 import java.util.concurrent.locks.ReadWriteLock;
39 import java.util.concurrent.locks.ReentrantReadWriteLock;
40 import java.util.function.Function;
41 import java.util.function.Supplier;
42 import java.util.regex.Pattern;
43
44 import org.eclipse.jgit.diff.DiffEntry;
45 import org.eclipse.jgit.diff.DiffFormatter;
46 import org.eclipse.jgit.dircache.DirCache;
47 import org.eclipse.jgit.dircache.DirCacheIterator;
48 import org.eclipse.jgit.lib.Constants;
49 import org.eclipse.jgit.lib.ObjectId;
50 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
51 import org.eclipse.jgit.lib.ObjectReader;
52 import org.eclipse.jgit.lib.PersonIdent;
53 import org.eclipse.jgit.lib.RefUpdate;
54 import org.eclipse.jgit.lib.RefUpdate.Result;
55 import org.eclipse.jgit.revwalk.RevCommit;
56 import org.eclipse.jgit.revwalk.RevTree;
57 import org.eclipse.jgit.revwalk.RevWalk;
58 import org.eclipse.jgit.revwalk.TreeRevFilter;
59 import org.eclipse.jgit.revwalk.filter.RevFilter;
60 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
61 import org.eclipse.jgit.treewalk.TreeWalk;
62 import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
63 import org.eclipse.jgit.treewalk.filter.TreeFilter;
64 import org.eclipse.jgit.util.SystemReader;
65 import org.jspecify.annotations.Nullable;
66 import org.slf4j.Logger;
67 import org.slf4j.LoggerFactory;
68
69 import com.fasterxml.jackson.core.JsonProcessingException;
70 import com.fasterxml.jackson.databind.JsonNode;
71 import com.google.common.annotations.VisibleForTesting;
72 import com.google.common.base.MoreObjects;
73 import com.google.common.collect.ImmutableList;
74 import com.google.common.collect.ImmutableMap;
75
76 import com.linecorp.armeria.common.util.Exceptions;
77 import com.linecorp.armeria.server.ServiceRequestContext;
78 import com.linecorp.centraldogma.common.Author;
79 import com.linecorp.centraldogma.common.CentralDogmaException;
80 import com.linecorp.centraldogma.common.Change;
81 import com.linecorp.centraldogma.common.ChangeFormatException;
82 import com.linecorp.centraldogma.common.Commit;
83 import com.linecorp.centraldogma.common.Entry;
84 import com.linecorp.centraldogma.common.EntryNotFoundException;
85 import com.linecorp.centraldogma.common.EntryType;
86 import com.linecorp.centraldogma.common.Markup;
87 import com.linecorp.centraldogma.common.Revision;
88 import com.linecorp.centraldogma.common.RevisionNotFoundException;
89 import com.linecorp.centraldogma.common.RevisionRange;
90 import com.linecorp.centraldogma.common.ShuttingDownException;
91 import com.linecorp.centraldogma.internal.Jackson;
92 import com.linecorp.centraldogma.internal.Json5;
93 import com.linecorp.centraldogma.internal.Util;
94 import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch;
95 import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode;
96 import com.linecorp.centraldogma.server.command.CommitResult;
97 import com.linecorp.centraldogma.server.command.ContentTransformer;
98 import com.linecorp.centraldogma.server.internal.IsolatedSystemReader;
99 import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
100 import com.linecorp.centraldogma.server.internal.storage.repository.git.Watch.WatchListener;
101 import com.linecorp.centraldogma.server.internal.storage.repository.git.rocksdb.RocksDbRepository;
102 import com.linecorp.centraldogma.server.storage.StorageException;
103 import com.linecorp.centraldogma.server.storage.project.Project;
104 import com.linecorp.centraldogma.server.storage.repository.CacheableCall;
105 import com.linecorp.centraldogma.server.storage.repository.DiffResultType;
106 import com.linecorp.centraldogma.server.storage.repository.FindOption;
107 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
108 import com.linecorp.centraldogma.server.storage.repository.Repository;
109 import com.linecorp.centraldogma.server.storage.repository.RepositoryListener;
110
111
112
113
114 class GitRepository implements Repository {
115
116 private static final Logger logger = LoggerFactory.getLogger(GitRepository.class);
117
118 static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER;
119
120 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
121
122 private static final Field revWalkObjectsField;
123
124 static {
125 final String jgitPomProperties = "META-INF/maven/org.eclipse.jgit/org.eclipse.jgit/pom.properties";
126 try (InputStream is = SystemReader.class.getClassLoader().getResourceAsStream(jgitPomProperties)) {
127 final Properties props = new Properties();
128 props.load(is);
129 final Object jgitVersion = props.get("version");
130 if (jgitVersion != null) {
131 logger.info("Using JGit: {}", jgitVersion);
132 }
133 } catch (IOException e) {
134 logger.debug("Failed to read JGit version", e);
135 }
136
137 IsolatedSystemReader.install();
138
139 Field field = null;
140 try {
141 field = RevWalk.class.getDeclaredField("objects");
142 if (field.getType() != ObjectIdOwnerMap.class) {
143 throw new IllegalStateException(
144 RevWalk.class.getSimpleName() + ".objects is not an " +
145 ObjectIdOwnerMap.class.getSimpleName() + '.');
146 }
147 field.setAccessible(true);
148 } catch (NoSuchFieldException e) {
149 throw new IllegalStateException(
150 RevWalk.class.getSimpleName() + ".objects does not exist.");
151 }
152
153 revWalkObjectsField = field;
154 }
155
156 private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
157 private final Project parent;
158 private final Executor repositoryWorker;
159 private final long creationTimeMillis;
160 private final Author author;
161 @VisibleForTesting
162 @Nullable
163 final RepositoryCache cache;
164 private final File repoDir;
165 private final String name;
166 private final org.eclipse.jgit.lib.Repository jGitRepository;
167 private final boolean isEncrypted;
168 private final CommitIdDatabase commitIdDatabase;
169 @VisibleForTesting
170 final CommitWatchers commitWatchers = new CommitWatchers();
171 private final AtomicReference<Supplier<CentralDogmaException>> closePending = new AtomicReference<>();
172 private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
173 private final List<RepositoryListener> listeners = new CopyOnWriteArrayList<>();
174
175 List<RepositoryListener> listeners() {
176 return listeners;
177 }
178
179
180
181
182 private volatile Revision headRevision;
183
184
185
186
187 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
188 long creationTimeMillis, Author author, @Nullable RepositoryCache cache,
189 org.eclipse.jgit.lib.Repository jGitRepository, CommitIdDatabase commitIdDatabase) {
190 this.parent = parent;
191 this.repoDir = repoDir;
192 name = repoDir.getName();
193 this.repositoryWorker = repositoryWorker;
194 this.creationTimeMillis = creationTimeMillis;
195 this.author = author;
196 this.cache = cache;
197 this.jGitRepository = jGitRepository;
198 isEncrypted = jGitRepository instanceof RocksDbRepository;
199 this.commitIdDatabase = commitIdDatabase;
200 new CommitExecutor(this, creationTimeMillis, author, "Create a new repository", "",
201 Markup.PLAINTEXT, true)
202 .executeInitialCommit();
203
204 headRevision = Revision.INIT;
205 }
206
207
208
209
210 GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache,
211 org.eclipse.jgit.lib.Repository jGitRepository, CommitIdDatabase commitIdDatabase,
212 Revision headRevision) {
213 this.parent = requireNonNull(parent, "parent");
214 this.repoDir = requireNonNull(repoDir, "repoDir");
215 name = requireNonNull(repoDir, "repoDir").getName();
216 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
217 this.cache = cache;
218 this.jGitRepository = requireNonNull(jGitRepository, "jGitRepository");
219 isEncrypted = jGitRepository instanceof RocksDbRepository;
220 this.commitIdDatabase = requireNonNull(commitIdDatabase, "commitIdDatabase");
221 this.headRevision = requireNonNull(headRevision, "headRevision");
222 final Commit initialCommit = blockingHistory(Revision.INIT, Revision.INIT, ALL_PATH, 1).get(0);
223 creationTimeMillis = initialCommit.when();
224 author = initialCommit.author();
225 }
226
227
228
229
230
231
232
233 void close(Supplier<CentralDogmaException> failureCauseSupplier) {
234 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
235 if (closePending.compareAndSet(null, failureCauseSupplier)) {
236 repositoryWorker.execute(() -> {
237
238 rwLock.writeLock().lock();
239 try {
240 closeRepository(commitIdDatabase, jGitRepository);
241 } finally {
242 try {
243 rwLock.writeLock().unlock();
244 } finally {
245 commitWatchers.close(failureCauseSupplier);
246 closeFuture.complete(null);
247 }
248 }
249 });
250 }
251
252 closeFuture.join();
253 }
254
255 static void closeRepository(@Nullable CommitIdDatabase commitIdDatabase,
256 org.eclipse.jgit.lib.@Nullable Repository jGitRepository) {
257 if (commitIdDatabase != null) {
258 try {
259 commitIdDatabase.close();
260 } catch (Exception e) {
261 logger.warn("Failed to close a commitId database:", e);
262 }
263 }
264
265 if (jGitRepository != null) {
266 try {
267 jGitRepository.close();
268 } catch (Exception e) {
269 logger.warn("Failed to close a Git repository: {}", jGitRepository.getDirectory(), e);
270 }
271 }
272 }
273
274 void internalClose() {
275 close(() -> new CentralDogmaException("should never reach here"));
276 }
277
278 CommitIdDatabase commitIdDatabase() {
279 return commitIdDatabase;
280 }
281
282 @Override
283 public org.eclipse.jgit.lib.Repository jGitRepository() {
284 return jGitRepository;
285 }
286
287 @Override
288 public Project parent() {
289 return parent;
290 }
291
292 @Override
293 public File repoDir() {
294 return repoDir;
295 }
296
297 @Override
298 public String name() {
299 return name;
300 }
301
302 @Override
303 public long creationTimeMillis() {
304 return creationTimeMillis;
305 }
306
307 @Override
308 public Author author() {
309 return author;
310 }
311
312 @Override
313 public Revision normalizeNow(Revision revision) {
314 return normalizeNow(revision, cachedHeadRevision().major());
315 }
316
317 private static Revision normalizeNow(Revision revision, int baseMajor) {
318 requireNonNull(revision, "revision");
319
320 int major = revision.major();
321
322 if (major >= 0) {
323 if (major > baseMajor) {
324 throw new RevisionNotFoundException(revision);
325 }
326 } else {
327 major = baseMajor + major + 1;
328 if (major <= 0) {
329 throw new RevisionNotFoundException(revision);
330 }
331 }
332
333
334 if (revision.major() == major) {
335 return revision;
336 } else {
337 return new Revision(major);
338 }
339 }
340
341 @Override
342 public RevisionRange normalizeNow(Revision from, Revision to) {
343 final int baseMajor = cachedHeadRevision().major();
344 return new RevisionRange(normalizeNow(from, baseMajor), normalizeNow(to, baseMajor));
345 }
346
347 @Override
348 public CompletableFuture<Map<String, Entry<?>>> find(
349 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
350 final ServiceRequestContext ctx = context();
351 return CompletableFuture.supplyAsync(() -> {
352 failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options);
353 return blockingFind(revision, pathPattern, options);
354 }, repositoryWorker);
355 }
356
357 private Map<String, Entry<?>> blockingFind(
358 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
359
360 requireNonNull(pathPattern, "pathPattern");
361 requireNonNull(revision, "revision");
362 requireNonNull(options, "options");
363
364 final Revision normRevision = normalizeNow(revision);
365 final boolean fetchContent = FindOption.FETCH_CONTENT.get(options);
366 final int maxEntries = FindOption.MAX_ENTRIES.get(options);
367
368 readLock();
369 try (ObjectReader reader = jGitRepository.newObjectReader();
370 TreeWalk treeWalk = new TreeWalk(reader);
371 RevWalk revWalk = newRevWalk(reader)) {
372
373
374 final Revision headRevision = cachedHeadRevision();
375 if (normRevision.compareTo(headRevision) > 0) {
376 return Collections.emptyMap();
377 }
378
379 if ("/".equals(pathPattern)) {
380 return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/"));
381 }
382
383 final Map<String, Entry<?>> result = new LinkedHashMap<>();
384 final ObjectId commitId = commitIdDatabase.get(normRevision);
385 final RevCommit revCommit = revWalk.parseCommit(commitId);
386 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
387
388 final RevTree revTree = revCommit.getTree();
389 treeWalk.addTree(revTree.getId());
390 while (treeWalk.next() && result.size() < maxEntries) {
391 final boolean matches = filter.matches(treeWalk);
392 final String path = '/' + treeWalk.getPathString();
393
394 try {
395
396 if (treeWalk.isSubtree()) {
397 if (matches) {
398
399 result.put(path, Entry.ofDirectory(normRevision, path));
400 }
401
402 treeWalk.enterSubtree();
403 continue;
404 }
405
406 if (!matches) {
407 continue;
408 }
409
410
411 final Entry<?> entry;
412 final EntryType entryType = EntryType.guessFromPath(path);
413 if (fetchContent) {
414 final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes();
415 final String string = new String(content, UTF_8);
416 switch (entryType) {
417 case JSON:
418 entry = Entry.ofJson(normRevision, path, string);
419 break;
420 case YAML:
421 Entry<?> maybeYaml;
422 try {
423 maybeYaml = Entry.ofYaml(normRevision, path, string);
424 } catch (JsonProcessingException e) {
425 logger.debug("Failed to parse YAML content at {}/{}{} (rev: {})",
426 parent.name(), name, path, normRevision, e);
427
428 maybeYaml = Entry.ofText(normRevision, path, string);
429 }
430 entry = maybeYaml;
431 break;
432 case TEXT:
433 final String strVal = sanitizeText(string);
434 entry = Entry.ofText(normRevision, path, strVal);
435 break;
436 default:
437 throw new Error("unexpected entry type: " + entryType);
438 }
439 } else {
440 switch (entryType) {
441 case JSON:
442 entry = Entry.ofJson(normRevision, path, "");
443 break;
444 case YAML:
445 entry = Entry.ofYaml(normRevision, path, "");
446 break;
447 case TEXT:
448 entry = Entry.ofText(normRevision, path, "");
449 break;
450 default:
451 throw new Error("unexpected entry type: " + entryType);
452 }
453 }
454
455 result.put(path, entry);
456 } catch (Exception e) {
457 throw new StorageException(
458 "failed to get data from '" + parent.name() + '/' + name + "' at " + path +
459 " for " + revision, e);
460 }
461 }
462
463 return Util.unsafeCast(result);
464 } catch (CentralDogmaException | IllegalArgumentException | StorageException e) {
465 throw e;
466 } catch (Exception e) {
467 throw new StorageException(
468 "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern +
469 " for " + revision, e);
470 } finally {
471 readUnlock();
472 }
473 }
474
475 @Override
476 public CompletableFuture<List<Commit>> history(
477 Revision from, Revision to, String pathPattern, int maxCommits) {
478
479 final ServiceRequestContext ctx = context();
480 return CompletableFuture.supplyAsync(() -> {
481 failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits);
482 return blockingHistory(from, to, pathPattern, maxCommits);
483 }, repositoryWorker);
484 }
485
486 @VisibleForTesting
487 List<Commit> blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) {
488 requireNonNull(pathPattern, "pathPattern");
489 requireNonNull(from, "from");
490 requireNonNull(to, "to");
491 if (maxCommits <= 0) {
492 throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)");
493 }
494
495 maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS);
496
497 final RevisionRange range = normalizeNow(from, to);
498 final RevisionRange descendingRange = range.toDescending();
499
500
501 readLock();
502 final RepositoryCache cache =
503
504 (descendingRange.from().major() < headRevision.major() - MAX_MAX_COMMITS * 3) ? null
505 : this.cache;
506 try (ObjectReader objectReader = jGitRepository.newObjectReader();
507 RevWalk revWalk = newRevWalk(new CachingTreeObjectReader(this, objectReader, cache))) {
508 final ObjectIdOwnerMap<?> revWalkInternalMap =
509 (ObjectIdOwnerMap<?>) revWalkObjectsField.get(revWalk);
510
511 final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from());
512 final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to());
513
514 revWalk.markStart(revWalk.parseCommit(fromCommitId));
515 revWalk.setRetainBody(false);
516
517
518
519
520
521
522 final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create(
523 TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern)));
524
525
526
527 final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS);
528
529 final List<Commit> commitList = new ArrayList<>();
530 int numProcessedCommits = 0;
531 for (RevCommit revCommit : revWalk) {
532 numProcessedCommits++;
533
534 if (filter.include(revWalk, revCommit)) {
535 revWalk.parseBody(revCommit);
536 commitList.add(toCommit(revCommit));
537 revCommit.disposeBody();
538 }
539
540 if (revCommit.getId().equals(toCommitId) ||
541 commitList.size() >= maxCommits ||
542
543 numProcessedCommits >= maxNumProcessedCommits) {
544 break;
545 }
546
547
548
549 if (numProcessedCommits % 16 == 0) {
550 revWalkInternalMap.clear();
551 }
552 }
553
554
555
556 if (commitList.size() < maxCommits &&
557 descendingRange.to().major() == 1 &&
558 pathPattern.contains(ALL_PATH)) {
559 try (RevWalk tmpRevWalk = newRevWalk()) {
560 final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId);
561 commitList.add(toCommit(lastRevCommit));
562 }
563 }
564
565 if (!descendingRange.equals(range)) {
566 Collections.reverse(commitList);
567 }
568
569 return commitList;
570 } catch (CentralDogmaException e) {
571 throw e;
572 } catch (Exception e) {
573 throw new StorageException(
574 "failed to retrieve the history: " + parent.name() + '/' + name +
575 " (" + pathPattern + ", " + from + ".." + to + ')', e);
576 } finally {
577 readUnlock();
578 }
579 }
580
581 private static Commit toCommit(RevCommit revCommit) {
582 final Author author;
583 final PersonIdent committerIdent = revCommit.getCommitterIdent();
584 final long when;
585 if (committerIdent == null) {
586 author = Author.UNKNOWN;
587 when = 0;
588 } else {
589 author = new Author(committerIdent.getName(), committerIdent.getEmailAddress());
590 when = committerIdent.getWhen().getTime();
591 }
592
593 try {
594 return CommitUtil.newCommit(author, when, revCommit.getFullMessage());
595 } catch (Exception e) {
596 throw new StorageException("failed to create a Commit", e);
597 }
598 }
599
600 @Override
601 public CompletableFuture<Map<String, Change<?>>> diff(Revision from, Revision to, String pathPattern,
602 DiffResultType diffResultType) {
603 final ServiceRequestContext ctx = context();
604 return CompletableFuture.supplyAsync(() -> {
605 requireNonNull(from, "from");
606 requireNonNull(to, "to");
607 requireNonNull(pathPattern, "pathPattern");
608
609 failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern);
610
611 final RevisionRange range = normalizeNow(from, to).toAscending();
612 readLock();
613 try (RevWalk rw = newRevWalk()) {
614 final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from()));
615 final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to()));
616
617
618
619 return toChangeMap(blockingCompareTreesUncached(
620 treeA, treeB, pathPatternFilterOrTreeFilter(pathPattern)), diffResultType);
621 } catch (StorageException e) {
622 throw e;
623 } catch (Exception e) {
624 throw new StorageException("failed to parse two trees: range=" + range, e);
625 } finally {
626 readUnlock();
627 }
628 }, repositoryWorker);
629 }
630
631 private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) {
632 if (pathPattern == null) {
633 return TreeFilter.ALL;
634 }
635
636 final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern);
637 return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter;
638 }
639
640 @Override
641 public CompletableFuture<Map<String, Change<?>>> previewDiff(Revision baseRevision,
642 Iterable<Change<?>> changes) {
643 final ServiceRequestContext ctx = context();
644 return CompletableFuture.supplyAsync(() -> {
645 failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision);
646 return blockingPreviewDiff(baseRevision, new DefaultChangesApplier(changes));
647 }, repositoryWorker);
648 }
649
650 Map<String, Change<?>> blockingPreviewDiff(Revision baseRevision, AbstractChangesApplier changesApplier) {
651 baseRevision = normalizeNow(baseRevision);
652
653 readLock();
654 try (ObjectReader reader = jGitRepository.newObjectReader();
655 RevWalk revWalk = newRevWalk(reader);
656 DiffFormatter diffFormatter = new DiffFormatter(null)) {
657
658 final ObjectId baseTreeId = toTree(revWalk, baseRevision);
659 final DirCache dirCache = DirCache.newInCore();
660 final int numEdits = changesApplier.apply(jGitRepository, baseRevision, baseTreeId, dirCache);
661 if (numEdits == 0) {
662 return Collections.emptyMap();
663 }
664
665 final CanonicalTreeParser p = new CanonicalTreeParser();
666 p.reset(reader, baseTreeId);
667 diffFormatter.setRepository(jGitRepository);
668 final List<DiffEntry> result = diffFormatter.scan(p, new DirCacheIterator(dirCache));
669 return toChangeMap(result, DiffResultType.NORMAL);
670 } catch (IOException e) {
671 throw new StorageException("failed to perform a dry-run diff", e);
672 } finally {
673 readUnlock();
674 }
675 }
676
677 private Map<String, Change<?>> toChangeMap(List<DiffEntry> diffEntryList, DiffResultType diffResultType) {
678 try (ObjectReader reader = jGitRepository.newObjectReader()) {
679 final Map<String, Change<?>> changeMap = new LinkedHashMap<>();
680
681 for (DiffEntry diffEntry : diffEntryList) {
682 final String oldPath = '/' + diffEntry.getOldPath();
683 final String newPath = '/' + diffEntry.getNewPath();
684
685 switch (diffEntry.getChangeType()) {
686 case MODIFY:
687 if (diffResultType == DiffResultType.PATCH_TO_TEXT_UPSERT) {
688 putTextDiff(changeMap, oldPath, newPath,
689 sanitizeText(new String(
690 reader.open(diffEntry.getOldId().toObjectId())
691 .getBytes(), UTF_8)),
692 sanitizeText(new String(
693 reader.open(diffEntry.getNewId().toObjectId())
694 .getBytes(), UTF_8)),
695 DiffResultType.PATCH_TO_TEXT_UPSERT);
696 break;
697 }
698 final EntryType oldEntryType = EntryType.guessFromPath(oldPath);
699 switch (oldEntryType) {
700 case JSON:
701 if (!oldPath.equals(newPath)) {
702 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
703 }
704
705 final byte[] oldJsonBytes =
706 reader.open(diffEntry.getOldId().toObjectId()).getBytes();
707 final byte[] newJsonBytes =
708 reader.open(diffEntry.getNewId().toObjectId()).getBytes();
709 final JsonNode oldJsonNode =
710 Jackson.readTree(oldPath, oldJsonBytes);
711 final JsonNode newJsonNode =
712 Jackson.readTree(newPath, newJsonBytes);
713 final JsonPatch patch =
714 JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE);
715
716 if (!patch.isEmpty()) {
717 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
718 putChange(changeMap, newPath,
719 Change.ofJsonUpsert(newPath, newJsonNode));
720 } else {
721 putChange(changeMap, newPath,
722 Change.ofJsonPatch(newPath, Jackson.valueToTree(patch)));
723 }
724 } else if (Json5.isJson5(newPath)) {
725
726
727 putTextDiff(changeMap, oldPath, newPath,
728 sanitizeText(new String(oldJsonBytes, UTF_8)),
729 sanitizeText(new String(newJsonBytes, UTF_8)),
730 diffResultType);
731 }
732 break;
733 case YAML:
734 final byte[] oldYamlBytes =
735 reader.open(diffEntry.getOldId().toObjectId()).getBytes();
736 final byte[] newYamlBytes =
737 reader.open(diffEntry.getNewId().toObjectId()).getBytes();
738 JsonNode oldYamlNode = null;
739 JsonNode newYamlNode = null;
740 try {
741 oldYamlNode = Jackson.readTree(oldPath, oldYamlBytes);
742 newYamlNode = Jackson.readTree(newPath, newYamlBytes);
743 } catch (JsonProcessingException e) {
744 logger.debug("Failed to parse YAML content for diff at {}; " +
745 "falling back to text diff", oldPath, e);
746 }
747
748 if (oldYamlNode != null && newYamlNode != null) {
749 if (!oldPath.equals(newPath)) {
750 putChange(changeMap, oldPath,
751 Change.ofRename(oldPath, newPath));
752 }
753 final JsonPatch yamlPatch =
754 JsonPatch.generate(oldYamlNode, newYamlNode,
755 ReplaceMode.SAFE);
756 if (!yamlPatch.isEmpty()) {
757 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
758 putChange(changeMap, newPath,
759 Change.ofYamlUpsert(newPath, newYamlNode));
760 } else {
761 putChange(changeMap, newPath,
762 Change.ofJsonPatch(newPath,
763 Jackson.valueToTree(yamlPatch)));
764 }
765 } else {
766
767
768 putTextDiff(changeMap, oldPath, newPath,
769 sanitizeText(new String(oldYamlBytes, UTF_8)),
770 sanitizeText(new String(newYamlBytes, UTF_8)),
771 diffResultType);
772 }
773 } else {
774
775 putTextDiff(changeMap, oldPath, newPath,
776 sanitizeText(new String(oldYamlBytes, UTF_8)),
777 sanitizeText(new String(newYamlBytes, UTF_8)),
778 diffResultType);
779 }
780 break;
781 case TEXT:
782 putTextDiff(changeMap, oldPath, newPath,
783 sanitizeText(new String(
784 reader.open(diffEntry.getOldId().toObjectId())
785 .getBytes(), UTF_8)),
786 sanitizeText(new String(
787 reader.open(diffEntry.getNewId().toObjectId())
788 .getBytes(), UTF_8)),
789 diffResultType);
790 break;
791 default:
792 throw new Error("unexpected old entry type: " + oldEntryType);
793 }
794 break;
795 case ADD:
796 if (diffResultType == DiffResultType.PATCH_TO_TEXT_UPSERT) {
797 final String addedText = sanitizeText(new String(
798 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
799 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, addedText));
800 break;
801 }
802 final EntryType newEntryType = EntryType.guessFromPath(newPath);
803 switch (newEntryType) {
804 case JSON: {
805 final JsonNode jsonNode = Jackson.readTree(newPath,
806 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
807
808 putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode));
809 break;
810 }
811 case YAML: {
812 final String text = sanitizeText(new String(
813 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
814
815 try {
816 putChange(changeMap, newPath,
817 Change.ofYamlUpsert(newPath, text));
818 } catch (ChangeFormatException e) {
819
820 logger.debug("Failed to parse YAML content at {}; " +
821 "falling back to text upsert", newPath, e);
822 putChange(changeMap, newPath,
823 Change.ofTextUpsert(newPath, text));
824 }
825 break;
826 }
827 case TEXT: {
828 final String text = sanitizeText(new String(
829 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
830
831 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text));
832 break;
833 }
834 default:
835 throw new Error("unexpected new entry type: " + newEntryType);
836 }
837 break;
838 case DELETE:
839 putChange(changeMap, oldPath, Change.ofRemoval(oldPath));
840 break;
841 default:
842 throw new Error();
843 }
844 }
845 return changeMap;
846 } catch (Exception e) {
847 throw new StorageException("failed to convert list of DiffEntry to Changes map", e);
848 }
849 }
850
851 private static void putTextDiff(Map<String, Change<?>> changeMap, String oldPath, String newPath,
852 String oldText, String newText, DiffResultType diffResultType) {
853 if (!oldPath.equals(newPath)) {
854 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
855 }
856 if (!oldText.equals(newText)) {
857 if (diffResultType == DiffResultType.PATCH_TO_UPSERT ||
858 diffResultType == DiffResultType.PATCH_TO_TEXT_UPSERT) {
859 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, newText));
860 } else {
861 putChange(changeMap, newPath, Change.ofTextPatch(newPath, oldText, newText));
862 }
863 }
864 }
865
866 private static void putChange(Map<String, Change<?>> changeMap, String path, Change<?> change) {
867 final Change<?> oldChange = changeMap.put(path, change);
868 assert oldChange == null;
869 }
870
871 @Override
872 public CompletableFuture<CommitResult> commit(
873 Revision baseRevision, long commitTimeMillis, Author author, String summary,
874 String detail, Markup markup, Iterable<Change<?>> changes, boolean directExecution) {
875 requireNonNull(baseRevision, "baseRevision");
876 requireNonNull(author, "author");
877 requireNonNull(summary, "summary");
878 requireNonNull(detail, "detail");
879 requireNonNull(markup, "markup");
880 requireNonNull(changes, "changes");
881 final CommitExecutor commitExecutor =
882 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
883 return commit(baseRevision, commitExecutor, normBaseRevision -> changes);
884 }
885
886 @Override
887 public CompletableFuture<CommitResult> commit(Revision baseRevision, long commitTimeMillis, Author author,
888 String summary, String detail, Markup markup,
889 ContentTransformer<?> transformer) {
890 requireNonNull(baseRevision, "baseRevision");
891 requireNonNull(author, "author");
892 requireNonNull(summary, "summary");
893 requireNonNull(detail, "detail");
894 requireNonNull(markup, "markup");
895 requireNonNull(transformer, "transformer");
896 final CommitExecutor commitExecutor =
897 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
898 return commit(baseRevision, commitExecutor,
899 normBaseRevision -> blockingPreviewDiff(
900 normBaseRevision, new TransformingChangesApplier(transformer)).values());
901 }
902
903 private CompletableFuture<CommitResult> commit(
904 Revision baseRevision,
905 CommitExecutor commitExecutor,
906 Function<Revision, Iterable<Change<?>>> applyingChangesProvider) {
907 final ServiceRequestContext ctx = context();
908 return CompletableFuture.supplyAsync(() -> {
909 failFastIfTimedOut(this, logger, ctx, "commit", baseRevision,
910 commitExecutor.author(), commitExecutor.summary());
911 return commitExecutor.execute(baseRevision, applyingChangesProvider);
912 }, repositoryWorker);
913 }
914
915
916
917
918 static String sanitizeText(String text) {
919 if (text.indexOf('\r') >= 0) {
920 text = CR.matcher(text).replaceAll("");
921 }
922 if (!text.isEmpty() && !text.endsWith("\n")) {
923 text += "\n";
924 }
925 return text;
926 }
927
928 static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
929 String ref, ObjectId commitId) throws IOException {
930 if (ref.startsWith(Constants.R_TAGS)) {
931 throw new StorageException("Using a tag is not allowed. ref: " + ref);
932 }
933
934 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
935 refUpdate.setNewObjectId(commitId);
936
937 final Result res = refUpdate.update(revWalk);
938 switch (res) {
939 case NEW:
940 case FAST_FORWARD:
941
942 break;
943 default:
944 throw new StorageException("unexpected refUpdate state: " + res);
945 }
946 }
947
948 @Override
949 public CompletableFuture<Revision> findLatestRevision(Revision lastKnownRevision, String pathPattern,
950 boolean errorOnEntryNotFound) {
951 requireNonNull(lastKnownRevision, "lastKnownRevision");
952 requireNonNull(pathPattern, "pathPattern");
953
954 final ServiceRequestContext ctx = context();
955 return CompletableFuture.supplyAsync(() -> {
956 failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern);
957 return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound);
958 }, repositoryWorker);
959 }
960
961 @Nullable
962 private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern,
963 boolean errorOnEntryNotFound) {
964 final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD);
965 if (range.from().equals(range.to())) {
966
967 if (!errorOnEntryNotFound) {
968 return null;
969 }
970
971 final Map<String, Entry<?>> entries =
972 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
973 if (!entries.isEmpty()) {
974
975 return null;
976 }
977 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
978 }
979
980 if (range.from().major() == 1) {
981
982 final Map<String, Entry<?>> entries =
983 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
984 if (entries.isEmpty()) {
985 if (!errorOnEntryNotFound) {
986 return null;
987 }
988 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
989 } else {
990 return range.to();
991 }
992 }
993
994
995 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
996
997 final List<DiffEntry> diffEntries;
998 readLock();
999 try (RevWalk revWalk = newRevWalk()) {
1000 final RevTree treeA = toTree(revWalk, range.from());
1001 final RevTree treeB = toTree(revWalk, range.to());
1002 diffEntries = blockingCompareTrees(treeA, treeB);
1003 } finally {
1004 readUnlock();
1005 }
1006
1007
1008 for (DiffEntry e : diffEntries) {
1009 final String path;
1010 switch (e.getChangeType()) {
1011 case ADD:
1012 path = e.getNewPath();
1013 break;
1014 case MODIFY:
1015 case DELETE:
1016 path = e.getOldPath();
1017 break;
1018 default:
1019 throw new Error();
1020 }
1021
1022 if (filter.matches(path)) {
1023 return range.to();
1024 }
1025 }
1026
1027 if (!errorOnEntryNotFound) {
1028 return null;
1029 }
1030 if (!blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) {
1031
1032
1033 return null;
1034 }
1035 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
1036 }
1037
1038
1039
1040
1041 private List<DiffEntry> blockingCompareTrees(RevTree treeA, RevTree treeB) {
1042 if (cache == null) {
1043 return blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL);
1044 }
1045
1046 final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB);
1047 return cache.get(key).join();
1048 }
1049
1050 List<DiffEntry> blockingCompareTreesUncached(@Nullable RevTree treeA,
1051 @Nullable RevTree treeB,
1052 TreeFilter filter) {
1053 readLock();
1054 try (DiffFormatter diffFormatter = new DiffFormatter(null)) {
1055 diffFormatter.setRepository(jGitRepository);
1056 diffFormatter.setPathFilter(filter);
1057 return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB));
1058 } catch (IOException e) {
1059 throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e);
1060 } finally {
1061 readUnlock();
1062 }
1063 }
1064
1065 @Override
1066 public CompletableFuture<Revision> watch(Revision lastKnownRevision, String pathPattern,
1067 boolean errorOnEntryNotFound) {
1068 requireNonNull(lastKnownRevision, "lastKnownRevision");
1069 requireNonNull(pathPattern, "pathPattern");
1070 final ServiceRequestContext ctx = context();
1071 final Revision normLastKnownRevision = normalizeNow(lastKnownRevision);
1072 final CompletableFuture<Revision> future = new CompletableFuture<>();
1073 CompletableFuture.runAsync(() -> {
1074 failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern);
1075 readLock();
1076 try {
1077
1078
1079 final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern,
1080 errorOnEntryNotFound);
1081 if (latestRevision != null) {
1082 future.complete(latestRevision);
1083 } else {
1084 commitWatchers.add(normLastKnownRevision, pathPattern, future, null);
1085 }
1086 } finally {
1087 readUnlock();
1088 }
1089 }, repositoryWorker).exceptionally(cause -> {
1090 future.completeExceptionally(cause);
1091 return null;
1092 });
1093
1094 return future;
1095 }
1096
1097 private void recursiveWatch(String pathPattern, WatchListener listener) {
1098 requireNonNull(pathPattern, "pathPattern");
1099 CompletableFuture.runAsync(() -> {
1100 final Revision headRevision = this.headRevision;
1101
1102 commitWatchers.add(headRevision, pathPattern, null, listener);
1103 listener.onUpdate(headRevision, null);
1104 }, repositoryWorker);
1105 }
1106
1107 @Override
1108 public <T> CompletableFuture<T> execute(CacheableCall<T> cacheableCall) {
1109
1110 requireNonNull(cacheableCall, "cacheableCall");
1111 final ServiceRequestContext ctx = context();
1112
1113 return CompletableFuture.supplyAsync(() -> {
1114 failFastIfTimedOut(this, logger, ctx, "execute", cacheableCall);
1115 return cacheableCall.execute();
1116 }, repositoryWorker).thenCompose(Function.identity());
1117 }
1118
1119 @Override
1120 public void addListener(RepositoryListener listener) {
1121 listeners.add(listener);
1122
1123 final String pathPattern = listener.pathPattern();
1124 recursiveWatch(pathPattern, (newRevision, cause) -> {
1125 if (shouldStopListening()) {
1126 return;
1127 }
1128
1129 if (cause != null) {
1130 cause = Exceptions.peel(cause);
1131 if (cause instanceof ShuttingDownException) {
1132 return;
1133 }
1134
1135 logger.warn("Failed to watch {} file in {}/{}.", pathPattern, parent.name(), name, cause);
1136 return;
1137 }
1138
1139 try {
1140 assert newRevision != null;
1141
1142 listener.onUpdate(blockingFind(headRevision, pathPattern, ImmutableMap.of()));
1143 } catch (Exception ex) {
1144 logger.warn("Unexpected exception while invoking {}.onUpdate(). listener: {}",
1145 RepositoryListener.class.getSimpleName(), listener, ex);
1146 }
1147 });
1148 }
1149
1150 private boolean shouldStopListening() {
1151 return closePending.get() != null;
1152 }
1153
1154 void notifyWatchers(Revision newRevision, List<DiffEntry> diffEntries) {
1155 for (DiffEntry entry : diffEntries) {
1156 switch (entry.getChangeType()) {
1157 case ADD:
1158 commitWatchers.notify(newRevision, entry.getNewPath());
1159 break;
1160 case MODIFY:
1161 case DELETE:
1162 commitWatchers.notify(newRevision, entry.getOldPath());
1163 break;
1164 default:
1165 throw new Error();
1166 }
1167 }
1168 }
1169
1170 Revision cachedHeadRevision() {
1171 return headRevision;
1172 }
1173
1174 void setHeadRevision(Revision headRevision) {
1175 this.headRevision = headRevision;
1176 }
1177
1178 private RevTree toTree(RevWalk revWalk, Revision revision) {
1179 return toTree(commitIdDatabase, revWalk, revision);
1180 }
1181
1182 static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) {
1183 final ObjectId commitId = commitIdDatabase.get(revision);
1184 try {
1185 return revWalk.parseCommit(commitId).getTree();
1186 } catch (IOException e) {
1187 throw new StorageException("failed to parse a commit: " + commitId, e);
1188 }
1189 }
1190
1191 private RevWalk newRevWalk() {
1192 final RevWalk revWalk = new RevWalk(jGitRepository);
1193 configureRevWalk(revWalk);
1194 return revWalk;
1195 }
1196
1197 static RevWalk newRevWalk(ObjectReader reader) {
1198 final RevWalk revWalk = new RevWalk(reader);
1199 configureRevWalk(revWalk);
1200 return revWalk;
1201 }
1202
1203 private static void configureRevWalk(RevWalk revWalk) {
1204
1205 revWalk.setRewriteParents(false);
1206 }
1207
1208 private void readLock() {
1209 rwLock.readLock().lock();
1210 if (closePending.get() != null) {
1211 rwLock.readLock().unlock();
1212 throw closePending.get().get();
1213 }
1214 }
1215
1216 private void readUnlock() {
1217 rwLock.readLock().unlock();
1218 }
1219
1220 void writeLock() {
1221 rwLock.writeLock().lock();
1222 if (closePending.get() != null) {
1223 writeUnLock();
1224 throw closePending.get().get();
1225 }
1226 }
1227
1228 void writeUnLock() {
1229 rwLock.writeLock().unlock();
1230 }
1231
1232 static void deleteCruft(File repoDir) {
1233 try {
1234 Util.deleteFileTree(repoDir);
1235 } catch (IOException e) {
1236 logger.error("Failed to delete a half-created repository at: {}", repoDir, e);
1237 }
1238 }
1239
1240 @Override
1241 public boolean isEncrypted() {
1242 return isEncrypted;
1243 }
1244
1245 @Override
1246 public String toString() {
1247 return MoreObjects.toStringHelper(this)
1248 .add("dir", jGitRepository.getDirectory())
1249 .toString();
1250 }
1251 }