1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage.repository.git;
18
19 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context;
20 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut;
21 import static java.nio.charset.StandardCharsets.UTF_8;
22 import static java.util.Objects.requireNonNull;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.lang.reflect.Field;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.LinkedHashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Properties;
34 import java.util.concurrent.CompletableFuture;
35 import java.util.concurrent.CopyOnWriteArrayList;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.atomic.AtomicReference;
38 import java.util.concurrent.locks.ReadWriteLock;
39 import java.util.concurrent.locks.ReentrantReadWriteLock;
40 import java.util.function.Function;
41 import java.util.function.Supplier;
42 import java.util.regex.Pattern;
43
44 import javax.annotation.Nullable;
45
46 import org.eclipse.jgit.diff.DiffEntry;
47 import org.eclipse.jgit.diff.DiffFormatter;
48 import org.eclipse.jgit.dircache.DirCache;
49 import org.eclipse.jgit.dircache.DirCacheIterator;
50 import org.eclipse.jgit.lib.Constants;
51 import org.eclipse.jgit.lib.ObjectId;
52 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
53 import org.eclipse.jgit.lib.ObjectReader;
54 import org.eclipse.jgit.lib.PersonIdent;
55 import org.eclipse.jgit.lib.RefUpdate;
56 import org.eclipse.jgit.lib.RefUpdate.Result;
57 import org.eclipse.jgit.revwalk.RevCommit;
58 import org.eclipse.jgit.revwalk.RevTree;
59 import org.eclipse.jgit.revwalk.RevWalk;
60 import org.eclipse.jgit.revwalk.TreeRevFilter;
61 import org.eclipse.jgit.revwalk.filter.RevFilter;
62 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
63 import org.eclipse.jgit.treewalk.TreeWalk;
64 import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
65 import org.eclipse.jgit.treewalk.filter.TreeFilter;
66 import org.eclipse.jgit.util.SystemReader;
67 import org.slf4j.Logger;
68 import org.slf4j.LoggerFactory;
69
70 import com.fasterxml.jackson.databind.JsonNode;
71 import com.google.common.annotations.VisibleForTesting;
72 import com.google.common.base.MoreObjects;
73 import com.google.common.collect.ImmutableList;
74 import com.google.common.collect.ImmutableMap;
75
76 import com.linecorp.armeria.common.util.Exceptions;
77 import com.linecorp.armeria.server.ServiceRequestContext;
78 import com.linecorp.centraldogma.common.Author;
79 import com.linecorp.centraldogma.common.CentralDogmaException;
80 import com.linecorp.centraldogma.common.Change;
81 import com.linecorp.centraldogma.common.Commit;
82 import com.linecorp.centraldogma.common.Entry;
83 import com.linecorp.centraldogma.common.EntryNotFoundException;
84 import com.linecorp.centraldogma.common.EntryType;
85 import com.linecorp.centraldogma.common.Markup;
86 import com.linecorp.centraldogma.common.Revision;
87 import com.linecorp.centraldogma.common.RevisionNotFoundException;
88 import com.linecorp.centraldogma.common.RevisionRange;
89 import com.linecorp.centraldogma.common.ShuttingDownException;
90 import com.linecorp.centraldogma.internal.Jackson;
91 import com.linecorp.centraldogma.internal.Util;
92 import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch;
93 import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode;
94 import com.linecorp.centraldogma.server.command.CommitResult;
95 import com.linecorp.centraldogma.server.command.ContentTransformer;
96 import com.linecorp.centraldogma.server.internal.IsolatedSystemReader;
97 import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
98 import com.linecorp.centraldogma.server.internal.storage.repository.git.Watch.WatchListener;
99 import com.linecorp.centraldogma.server.internal.storage.repository.git.rocksdb.RocksDbRepository;
100 import com.linecorp.centraldogma.server.storage.StorageException;
101 import com.linecorp.centraldogma.server.storage.project.Project;
102 import com.linecorp.centraldogma.server.storage.repository.CacheableCall;
103 import com.linecorp.centraldogma.server.storage.repository.DiffResultType;
104 import com.linecorp.centraldogma.server.storage.repository.FindOption;
105 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
106 import com.linecorp.centraldogma.server.storage.repository.Repository;
107 import com.linecorp.centraldogma.server.storage.repository.RepositoryListener;
108
109
110
111
112 class GitRepository implements Repository {
113
114 private static final Logger logger = LoggerFactory.getLogger(GitRepository.class);
115
116 static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER;
117
118 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
119
120 private static final Field revWalkObjectsField;
121
122 static {
123 final String jgitPomProperties = "META-INF/maven/org.eclipse.jgit/org.eclipse.jgit/pom.properties";
124 try (InputStream is = SystemReader.class.getClassLoader().getResourceAsStream(jgitPomProperties)) {
125 final Properties props = new Properties();
126 props.load(is);
127 final Object jgitVersion = props.get("version");
128 if (jgitVersion != null) {
129 logger.info("Using JGit: {}", jgitVersion);
130 }
131 } catch (IOException e) {
132 logger.debug("Failed to read JGit version", e);
133 }
134
135 IsolatedSystemReader.install();
136
137 Field field = null;
138 try {
139 field = RevWalk.class.getDeclaredField("objects");
140 if (field.getType() != ObjectIdOwnerMap.class) {
141 throw new IllegalStateException(
142 RevWalk.class.getSimpleName() + ".objects is not an " +
143 ObjectIdOwnerMap.class.getSimpleName() + '.');
144 }
145 field.setAccessible(true);
146 } catch (NoSuchFieldException e) {
147 throw new IllegalStateException(
148 RevWalk.class.getSimpleName() + ".objects does not exist.");
149 }
150
151 revWalkObjectsField = field;
152 }
153
154 private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
155 private final Project parent;
156 private final Executor repositoryWorker;
157 private final long creationTimeMillis;
158 private final Author author;
159 @VisibleForTesting
160 @Nullable
161 final RepositoryCache cache;
162 private final File repoDir;
163 private final String name;
164 private final org.eclipse.jgit.lib.Repository jGitRepository;
165 private final boolean isEncrypted;
166 private final CommitIdDatabase commitIdDatabase;
167 @VisibleForTesting
168 final CommitWatchers commitWatchers = new CommitWatchers();
169 private final AtomicReference<Supplier<CentralDogmaException>> closePending = new AtomicReference<>();
170 private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
171 private final List<RepositoryListener> listeners = new CopyOnWriteArrayList<>();
172
173
174
175
176 private volatile Revision headRevision;
177
178
179
180
181 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
182 long creationTimeMillis, Author author, @Nullable RepositoryCache cache,
183 org.eclipse.jgit.lib.Repository jGitRepository, CommitIdDatabase commitIdDatabase) {
184 this.parent = parent;
185 this.repoDir = repoDir;
186 name = repoDir.getName();
187 this.repositoryWorker = repositoryWorker;
188 this.creationTimeMillis = creationTimeMillis;
189 this.author = author;
190 this.cache = cache;
191 this.jGitRepository = jGitRepository;
192 isEncrypted = jGitRepository instanceof RocksDbRepository;
193 this.commitIdDatabase = commitIdDatabase;
194 new CommitExecutor(this, creationTimeMillis, author, "Create a new repository", "",
195 Markup.PLAINTEXT, true)
196 .executeInitialCommit();
197
198 headRevision = Revision.INIT;
199 }
200
201
202
203
204 GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache,
205 org.eclipse.jgit.lib.Repository jGitRepository, CommitIdDatabase commitIdDatabase,
206 Revision headRevision) {
207 this.parent = requireNonNull(parent, "parent");
208 this.repoDir = requireNonNull(repoDir, "repoDir");
209 name = requireNonNull(repoDir, "repoDir").getName();
210 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
211 this.cache = cache;
212 this.jGitRepository = requireNonNull(jGitRepository, "jGitRepository");
213 isEncrypted = jGitRepository instanceof RocksDbRepository;
214 this.commitIdDatabase = requireNonNull(commitIdDatabase, "commitIdDatabase");
215 this.headRevision = requireNonNull(headRevision, "headRevision");
216 final Commit initialCommit = blockingHistory(Revision.INIT, Revision.INIT, ALL_PATH, 1).get(0);
217 creationTimeMillis = initialCommit.when();
218 author = initialCommit.author();
219 }
220
221
222
223
224
225
226
227 void close(Supplier<CentralDogmaException> failureCauseSupplier) {
228 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
229 if (closePending.compareAndSet(null, failureCauseSupplier)) {
230 repositoryWorker.execute(() -> {
231
232 rwLock.writeLock().lock();
233 try {
234 closeRepository(commitIdDatabase, jGitRepository);
235 } finally {
236 try {
237 rwLock.writeLock().unlock();
238 } finally {
239 commitWatchers.close(failureCauseSupplier);
240 closeFuture.complete(null);
241 }
242 }
243 });
244 }
245
246 closeFuture.join();
247 }
248
249 static void closeRepository(@Nullable CommitIdDatabase commitIdDatabase,
250 @Nullable org.eclipse.jgit.lib.Repository jGitRepository) {
251 if (commitIdDatabase != null) {
252 try {
253 commitIdDatabase.close();
254 } catch (Exception e) {
255 logger.warn("Failed to close a commitId database:", e);
256 }
257 }
258
259 if (jGitRepository != null) {
260 try {
261 jGitRepository.close();
262 } catch (Exception e) {
263 logger.warn("Failed to close a Git repository: {}", jGitRepository.getDirectory(), e);
264 }
265 }
266 }
267
268 void internalClose() {
269 close(() -> new CentralDogmaException("should never reach here"));
270 }
271
272 CommitIdDatabase commitIdDatabase() {
273 return commitIdDatabase;
274 }
275
276 @Override
277 public org.eclipse.jgit.lib.Repository jGitRepository() {
278 return jGitRepository;
279 }
280
281 @Override
282 public Project parent() {
283 return parent;
284 }
285
286 @Override
287 public File repoDir() {
288 return repoDir;
289 }
290
291 @Override
292 public String name() {
293 return name;
294 }
295
296 @Override
297 public long creationTimeMillis() {
298 return creationTimeMillis;
299 }
300
301 @Override
302 public Author author() {
303 return author;
304 }
305
306 @Override
307 public Revision normalizeNow(Revision revision) {
308 return normalizeNow(revision, cachedHeadRevision().major());
309 }
310
311 private static Revision normalizeNow(Revision revision, int baseMajor) {
312 requireNonNull(revision, "revision");
313
314 int major = revision.major();
315
316 if (major >= 0) {
317 if (major > baseMajor) {
318 throw new RevisionNotFoundException(revision);
319 }
320 } else {
321 major = baseMajor + major + 1;
322 if (major <= 0) {
323 throw new RevisionNotFoundException(revision);
324 }
325 }
326
327
328 if (revision.major() == major) {
329 return revision;
330 } else {
331 return new Revision(major);
332 }
333 }
334
335 @Override
336 public RevisionRange normalizeNow(Revision from, Revision to) {
337 final int baseMajor = cachedHeadRevision().major();
338 return new RevisionRange(normalizeNow(from, baseMajor), normalizeNow(to, baseMajor));
339 }
340
341 @Override
342 public CompletableFuture<Map<String, Entry<?>>> find(
343 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
344 final ServiceRequestContext ctx = context();
345 return CompletableFuture.supplyAsync(() -> {
346 failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options);
347 return blockingFind(revision, pathPattern, options);
348 }, repositoryWorker);
349 }
350
351 private Map<String, Entry<?>> blockingFind(
352 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
353
354 requireNonNull(pathPattern, "pathPattern");
355 requireNonNull(revision, "revision");
356 requireNonNull(options, "options");
357
358 final Revision normRevision = normalizeNow(revision);
359 final boolean fetchContent = FindOption.FETCH_CONTENT.get(options);
360 final int maxEntries = FindOption.MAX_ENTRIES.get(options);
361
362 readLock();
363 try (ObjectReader reader = jGitRepository.newObjectReader();
364 TreeWalk treeWalk = new TreeWalk(reader);
365 RevWalk revWalk = newRevWalk(reader)) {
366
367
368 final Revision headRevision = cachedHeadRevision();
369 if (normRevision.compareTo(headRevision) > 0) {
370 return Collections.emptyMap();
371 }
372
373 if ("/".equals(pathPattern)) {
374 return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/"));
375 }
376
377 final Map<String, Entry<?>> result = new LinkedHashMap<>();
378 final ObjectId commitId = commitIdDatabase.get(normRevision);
379 final RevCommit revCommit = revWalk.parseCommit(commitId);
380 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
381
382 final RevTree revTree = revCommit.getTree();
383 treeWalk.addTree(revTree.getId());
384 while (treeWalk.next() && result.size() < maxEntries) {
385 final boolean matches = filter.matches(treeWalk);
386 final String path = '/' + treeWalk.getPathString();
387
388
389 if (treeWalk.isSubtree()) {
390 if (matches) {
391
392 result.put(path, Entry.ofDirectory(normRevision, path));
393 }
394
395 treeWalk.enterSubtree();
396 continue;
397 }
398
399 if (!matches) {
400 continue;
401 }
402
403
404 final Entry<?> entry;
405 final EntryType entryType = EntryType.guessFromPath(path);
406 if (fetchContent) {
407 final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes();
408 switch (entryType) {
409 case JSON:
410 final JsonNode jsonNode = Jackson.readTree(content);
411 entry = Entry.ofJson(normRevision, path, jsonNode);
412 break;
413 case TEXT:
414 final String strVal = sanitizeText(new String(content, UTF_8));
415 entry = Entry.ofText(normRevision, path, strVal);
416 break;
417 default:
418 throw new Error("unexpected entry type: " + entryType);
419 }
420 } else {
421 switch (entryType) {
422 case JSON:
423 entry = Entry.ofJson(normRevision, path, Jackson.nullNode);
424 break;
425 case TEXT:
426 entry = Entry.ofText(normRevision, path, "");
427 break;
428 default:
429 throw new Error("unexpected entry type: " + entryType);
430 }
431 }
432
433 result.put(path, entry);
434 }
435
436 return Util.unsafeCast(result);
437 } catch (CentralDogmaException | IllegalArgumentException e) {
438 throw e;
439 } catch (Exception e) {
440 throw new StorageException(
441 "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern +
442 " for " + revision, e);
443 } finally {
444 readUnlock();
445 }
446 }
447
448 @Override
449 public CompletableFuture<List<Commit>> history(
450 Revision from, Revision to, String pathPattern, int maxCommits) {
451
452 final ServiceRequestContext ctx = context();
453 return CompletableFuture.supplyAsync(() -> {
454 failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits);
455 return blockingHistory(from, to, pathPattern, maxCommits);
456 }, repositoryWorker);
457 }
458
459 @VisibleForTesting
460 List<Commit> blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) {
461 requireNonNull(pathPattern, "pathPattern");
462 requireNonNull(from, "from");
463 requireNonNull(to, "to");
464 if (maxCommits <= 0) {
465 throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)");
466 }
467
468 maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS);
469
470 final RevisionRange range = normalizeNow(from, to);
471 final RevisionRange descendingRange = range.toDescending();
472
473
474 readLock();
475 final RepositoryCache cache =
476
477 (descendingRange.from().major() < headRevision.major() - MAX_MAX_COMMITS * 3) ? null
478 : this.cache;
479 try (ObjectReader objectReader = jGitRepository.newObjectReader();
480 RevWalk revWalk = newRevWalk(new CachingTreeObjectReader(this, objectReader, cache))) {
481 final ObjectIdOwnerMap<?> revWalkInternalMap =
482 (ObjectIdOwnerMap<?>) revWalkObjectsField.get(revWalk);
483
484 final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from());
485 final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to());
486
487 revWalk.markStart(revWalk.parseCommit(fromCommitId));
488 revWalk.setRetainBody(false);
489
490
491
492
493
494
495 final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create(
496 TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern)));
497
498
499
500 final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS);
501
502 final List<Commit> commitList = new ArrayList<>();
503 int numProcessedCommits = 0;
504 for (RevCommit revCommit : revWalk) {
505 numProcessedCommits++;
506
507 if (filter.include(revWalk, revCommit)) {
508 revWalk.parseBody(revCommit);
509 commitList.add(toCommit(revCommit));
510 revCommit.disposeBody();
511 }
512
513 if (revCommit.getId().equals(toCommitId) ||
514 commitList.size() >= maxCommits ||
515
516 numProcessedCommits >= maxNumProcessedCommits) {
517 break;
518 }
519
520
521
522 if (numProcessedCommits % 16 == 0) {
523 revWalkInternalMap.clear();
524 }
525 }
526
527
528
529 if (commitList.size() < maxCommits &&
530 descendingRange.to().major() == 1 &&
531 pathPattern.contains(ALL_PATH)) {
532 try (RevWalk tmpRevWalk = newRevWalk()) {
533 final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId);
534 commitList.add(toCommit(lastRevCommit));
535 }
536 }
537
538 if (!descendingRange.equals(range)) {
539 Collections.reverse(commitList);
540 }
541
542 return commitList;
543 } catch (CentralDogmaException e) {
544 throw e;
545 } catch (Exception e) {
546 throw new StorageException(
547 "failed to retrieve the history: " + parent.name() + '/' + name +
548 " (" + pathPattern + ", " + from + ".." + to + ')', e);
549 } finally {
550 readUnlock();
551 }
552 }
553
554 private static Commit toCommit(RevCommit revCommit) {
555 final Author author;
556 final PersonIdent committerIdent = revCommit.getCommitterIdent();
557 final long when;
558 if (committerIdent == null) {
559 author = Author.UNKNOWN;
560 when = 0;
561 } else {
562 author = new Author(committerIdent.getName(), committerIdent.getEmailAddress());
563 when = committerIdent.getWhen().getTime();
564 }
565
566 try {
567 return CommitUtil.newCommit(author, when, revCommit.getFullMessage());
568 } catch (Exception e) {
569 throw new StorageException("failed to create a Commit", e);
570 }
571 }
572
573 @Override
574 public CompletableFuture<Map<String, Change<?>>> diff(Revision from, Revision to, String pathPattern,
575 DiffResultType diffResultType) {
576 final ServiceRequestContext ctx = context();
577 return CompletableFuture.supplyAsync(() -> {
578 requireNonNull(from, "from");
579 requireNonNull(to, "to");
580 requireNonNull(pathPattern, "pathPattern");
581
582 failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern);
583
584 final RevisionRange range = normalizeNow(from, to).toAscending();
585 readLock();
586 try (RevWalk rw = newRevWalk()) {
587 final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from()));
588 final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to()));
589
590
591
592 return toChangeMap(blockingCompareTreesUncached(
593 treeA, treeB, pathPatternFilterOrTreeFilter(pathPattern)), diffResultType);
594 } catch (StorageException e) {
595 throw e;
596 } catch (Exception e) {
597 throw new StorageException("failed to parse two trees: range=" + range, e);
598 } finally {
599 readUnlock();
600 }
601 }, repositoryWorker);
602 }
603
604 private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) {
605 if (pathPattern == null) {
606 return TreeFilter.ALL;
607 }
608
609 final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern);
610 return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter;
611 }
612
613 @Override
614 public CompletableFuture<Map<String, Change<?>>> previewDiff(Revision baseRevision,
615 Iterable<Change<?>> changes) {
616 final ServiceRequestContext ctx = context();
617 return CompletableFuture.supplyAsync(() -> {
618 failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision);
619 return blockingPreviewDiff(baseRevision, new DefaultChangesApplier(changes));
620 }, repositoryWorker);
621 }
622
623 Map<String, Change<?>> blockingPreviewDiff(Revision baseRevision, AbstractChangesApplier changesApplier) {
624 baseRevision = normalizeNow(baseRevision);
625
626 readLock();
627 try (ObjectReader reader = jGitRepository.newObjectReader();
628 RevWalk revWalk = newRevWalk(reader);
629 DiffFormatter diffFormatter = new DiffFormatter(null)) {
630
631 final ObjectId baseTreeId = toTree(revWalk, baseRevision);
632 final DirCache dirCache = DirCache.newInCore();
633 final int numEdits = changesApplier.apply(jGitRepository, baseRevision, baseTreeId, dirCache);
634 if (numEdits == 0) {
635 return Collections.emptyMap();
636 }
637
638 final CanonicalTreeParser p = new CanonicalTreeParser();
639 p.reset(reader, baseTreeId);
640 diffFormatter.setRepository(jGitRepository);
641 final List<DiffEntry> result = diffFormatter.scan(p, new DirCacheIterator(dirCache));
642 return toChangeMap(result, DiffResultType.NORMAL);
643 } catch (IOException e) {
644 throw new StorageException("failed to perform a dry-run diff", e);
645 } finally {
646 readUnlock();
647 }
648 }
649
650 private Map<String, Change<?>> toChangeMap(List<DiffEntry> diffEntryList, DiffResultType diffResultType) {
651 try (ObjectReader reader = jGitRepository.newObjectReader()) {
652 final Map<String, Change<?>> changeMap = new LinkedHashMap<>();
653
654 for (DiffEntry diffEntry : diffEntryList) {
655 final String oldPath = '/' + diffEntry.getOldPath();
656 final String newPath = '/' + diffEntry.getNewPath();
657
658 switch (diffEntry.getChangeType()) {
659 case MODIFY:
660 final EntryType oldEntryType = EntryType.guessFromPath(oldPath);
661 switch (oldEntryType) {
662 case JSON:
663 if (!oldPath.equals(newPath)) {
664 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
665 }
666
667 final JsonNode oldJsonNode =
668 Jackson.readTree(
669 reader.open(diffEntry.getOldId().toObjectId()).getBytes());
670 final JsonNode newJsonNode =
671 Jackson.readTree(
672 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
673 final JsonPatch patch =
674 JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE);
675
676 if (!patch.isEmpty()) {
677 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
678 putChange(changeMap, newPath,
679 Change.ofJsonUpsert(newPath, newJsonNode));
680 } else {
681 putChange(changeMap, newPath,
682 Change.ofJsonPatch(newPath, Jackson.valueToTree(patch)));
683 }
684 }
685 break;
686 case TEXT:
687 final String oldText = sanitizeText(new String(
688 reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8));
689
690 final String newText = sanitizeText(new String(
691 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
692
693 if (!oldPath.equals(newPath)) {
694 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
695 }
696
697 if (!oldText.equals(newText)) {
698 if (diffResultType == DiffResultType.PATCH_TO_UPSERT) {
699 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, newText));
700 } else {
701 putChange(changeMap, newPath,
702 Change.ofTextPatch(newPath, oldText, newText));
703 }
704 }
705 break;
706 default:
707 throw new Error("unexpected old entry type: " + oldEntryType);
708 }
709 break;
710 case ADD:
711 final EntryType newEntryType = EntryType.guessFromPath(newPath);
712 switch (newEntryType) {
713 case JSON: {
714 final JsonNode jsonNode = Jackson.readTree(
715 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
716
717 putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode));
718 break;
719 }
720 case TEXT: {
721 final String text = sanitizeText(new String(
722 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
723
724 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text));
725 break;
726 }
727 default:
728 throw new Error("unexpected new entry type: " + newEntryType);
729 }
730 break;
731 case DELETE:
732 putChange(changeMap, oldPath, Change.ofRemoval(oldPath));
733 break;
734 default:
735 throw new Error();
736 }
737 }
738 return changeMap;
739 } catch (Exception e) {
740 throw new StorageException("failed to convert list of DiffEntry to Changes map", e);
741 }
742 }
743
744 private static void putChange(Map<String, Change<?>> changeMap, String path, Change<?> change) {
745 final Change<?> oldChange = changeMap.put(path, change);
746 assert oldChange == null;
747 }
748
749 @Override
750 public CompletableFuture<CommitResult> commit(
751 Revision baseRevision, long commitTimeMillis, Author author, String summary,
752 String detail, Markup markup, Iterable<Change<?>> changes, boolean directExecution) {
753 requireNonNull(baseRevision, "baseRevision");
754 requireNonNull(author, "author");
755 requireNonNull(summary, "summary");
756 requireNonNull(detail, "detail");
757 requireNonNull(markup, "markup");
758 requireNonNull(changes, "changes");
759 final CommitExecutor commitExecutor =
760 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
761 return commit(baseRevision, commitExecutor, normBaseRevision -> {
762 if (!directExecution) {
763 return changes;
764 }
765 return blockingPreviewDiff(normBaseRevision, new DefaultChangesApplier(changes)).values();
766 });
767 }
768
769 @Override
770 public CompletableFuture<CommitResult> commit(Revision baseRevision, long commitTimeMillis, Author author,
771 String summary, String detail, Markup markup,
772 ContentTransformer<?> transformer) {
773 requireNonNull(baseRevision, "baseRevision");
774 requireNonNull(author, "author");
775 requireNonNull(summary, "summary");
776 requireNonNull(detail, "detail");
777 requireNonNull(markup, "markup");
778 requireNonNull(transformer, "transformer");
779 final CommitExecutor commitExecutor =
780 new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false);
781 return commit(baseRevision, commitExecutor,
782 normBaseRevision -> blockingPreviewDiff(
783 normBaseRevision, new TransformingChangesApplier(transformer)).values());
784 }
785
786 private CompletableFuture<CommitResult> commit(
787 Revision baseRevision,
788 CommitExecutor commitExecutor,
789 Function<Revision, Iterable<Change<?>>> applyingChangesProvider) {
790 final ServiceRequestContext ctx = context();
791 return CompletableFuture.supplyAsync(() -> {
792 failFastIfTimedOut(this, logger, ctx, "commit", baseRevision,
793 commitExecutor.author(), commitExecutor.summary());
794 return commitExecutor.execute(baseRevision, applyingChangesProvider);
795 }, repositoryWorker);
796 }
797
798
799
800
801 static String sanitizeText(String text) {
802 if (text.indexOf('\r') >= 0) {
803 text = CR.matcher(text).replaceAll("");
804 }
805 if (!text.isEmpty() && !text.endsWith("\n")) {
806 text += "\n";
807 }
808 return text;
809 }
810
811 static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
812 String ref, ObjectId commitId) throws IOException {
813 if (ref.startsWith(Constants.R_TAGS)) {
814 throw new StorageException("Using a tag is not allowed. ref: " + ref);
815 }
816
817 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
818 refUpdate.setNewObjectId(commitId);
819
820 final Result res = refUpdate.update(revWalk);
821 switch (res) {
822 case NEW:
823 case FAST_FORWARD:
824
825 break;
826 default:
827 throw new StorageException("unexpected refUpdate state: " + res);
828 }
829 }
830
831 @Override
832 public CompletableFuture<Revision> findLatestRevision(Revision lastKnownRevision, String pathPattern,
833 boolean errorOnEntryNotFound) {
834 requireNonNull(lastKnownRevision, "lastKnownRevision");
835 requireNonNull(pathPattern, "pathPattern");
836
837 final ServiceRequestContext ctx = context();
838 return CompletableFuture.supplyAsync(() -> {
839 failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern);
840 return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound);
841 }, repositoryWorker);
842 }
843
844 @Nullable
845 private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern,
846 boolean errorOnEntryNotFound) {
847 final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD);
848 if (range.from().equals(range.to())) {
849
850 if (!errorOnEntryNotFound) {
851 return null;
852 }
853
854 final Map<String, Entry<?>> entries =
855 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
856 if (!entries.isEmpty()) {
857
858 return null;
859 }
860 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
861 }
862
863 if (range.from().major() == 1) {
864
865 final Map<String, Entry<?>> entries =
866 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
867 if (entries.isEmpty()) {
868 if (!errorOnEntryNotFound) {
869 return null;
870 }
871 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
872 } else {
873 return range.to();
874 }
875 }
876
877
878 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
879
880 final List<DiffEntry> diffEntries;
881 readLock();
882 try (RevWalk revWalk = newRevWalk()) {
883 final RevTree treeA = toTree(revWalk, range.from());
884 final RevTree treeB = toTree(revWalk, range.to());
885 diffEntries = blockingCompareTrees(treeA, treeB);
886 } finally {
887 readUnlock();
888 }
889
890
891 for (DiffEntry e : diffEntries) {
892 final String path;
893 switch (e.getChangeType()) {
894 case ADD:
895 path = e.getNewPath();
896 break;
897 case MODIFY:
898 case DELETE:
899 path = e.getOldPath();
900 break;
901 default:
902 throw new Error();
903 }
904
905 if (filter.matches(path)) {
906 return range.to();
907 }
908 }
909
910 if (!errorOnEntryNotFound) {
911 return null;
912 }
913 if (!blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) {
914
915
916 return null;
917 }
918 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
919 }
920
921
922
923
924 private List<DiffEntry> blockingCompareTrees(RevTree treeA, RevTree treeB) {
925 if (cache == null) {
926 return blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL);
927 }
928
929 final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB);
930 return cache.get(key).join();
931 }
932
933 List<DiffEntry> blockingCompareTreesUncached(@Nullable RevTree treeA,
934 @Nullable RevTree treeB,
935 TreeFilter filter) {
936 readLock();
937 try (DiffFormatter diffFormatter = new DiffFormatter(null)) {
938 diffFormatter.setRepository(jGitRepository);
939 diffFormatter.setPathFilter(filter);
940 return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB));
941 } catch (IOException e) {
942 throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e);
943 } finally {
944 readUnlock();
945 }
946 }
947
948 @Override
949 public CompletableFuture<Revision> watch(Revision lastKnownRevision, String pathPattern,
950 boolean errorOnEntryNotFound) {
951 requireNonNull(lastKnownRevision, "lastKnownRevision");
952 requireNonNull(pathPattern, "pathPattern");
953 final ServiceRequestContext ctx = context();
954 final Revision normLastKnownRevision = normalizeNow(lastKnownRevision);
955 final CompletableFuture<Revision> future = new CompletableFuture<>();
956 CompletableFuture.runAsync(() -> {
957 failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern);
958 readLock();
959 try {
960
961
962 final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern,
963 errorOnEntryNotFound);
964 if (latestRevision != null) {
965 future.complete(latestRevision);
966 } else {
967 commitWatchers.add(normLastKnownRevision, pathPattern, future, null);
968 }
969 } finally {
970 readUnlock();
971 }
972 }, repositoryWorker).exceptionally(cause -> {
973 future.completeExceptionally(cause);
974 return null;
975 });
976
977 return future;
978 }
979
980 private void recursiveWatch(String pathPattern, WatchListener listener) {
981 requireNonNull(pathPattern, "pathPattern");
982 CompletableFuture.runAsync(() -> {
983 final Revision headRevision = this.headRevision;
984
985 commitWatchers.add(headRevision, pathPattern, null, listener);
986 listener.onUpdate(headRevision, null);
987 }, repositoryWorker);
988 }
989
990 @Override
991 public <T> CompletableFuture<T> execute(CacheableCall<T> cacheableCall) {
992
993 requireNonNull(cacheableCall, "cacheableCall");
994 final ServiceRequestContext ctx = context();
995
996 return CompletableFuture.supplyAsync(() -> {
997 failFastIfTimedOut(this, logger, ctx, "execute", cacheableCall);
998 return cacheableCall.execute();
999 }, repositoryWorker).thenCompose(Function.identity());
1000 }
1001
1002 @Override
1003 public void addListener(RepositoryListener listener) {
1004 listeners.add(listener);
1005
1006 final String pathPattern = listener.pathPattern();
1007 recursiveWatch(pathPattern, (newRevision, cause) -> {
1008 if (shouldStopListening()) {
1009 return;
1010 }
1011
1012 if (cause != null) {
1013 cause = Exceptions.peel(cause);
1014 if (cause instanceof ShuttingDownException) {
1015 return;
1016 }
1017
1018 logger.warn("Failed to watch {} file in {}/{}.", pathPattern, parent.name(), name, cause);
1019 return;
1020 }
1021
1022 try {
1023 assert newRevision != null;
1024
1025 listener.onUpdate(blockingFind(headRevision, pathPattern, ImmutableMap.of()));
1026 } catch (Exception ex) {
1027 logger.warn("Unexpected exception while invoking {}.onUpdate(). listener: {}",
1028 RepositoryListener.class.getSimpleName(), listener, ex);
1029 }
1030 });
1031 }
1032
1033 private boolean shouldStopListening() {
1034 return closePending.get() != null;
1035 }
1036
1037 void notifyWatchers(Revision newRevision, List<DiffEntry> diffEntries) {
1038 for (DiffEntry entry : diffEntries) {
1039 switch (entry.getChangeType()) {
1040 case ADD:
1041 commitWatchers.notify(newRevision, entry.getNewPath());
1042 break;
1043 case MODIFY:
1044 case DELETE:
1045 commitWatchers.notify(newRevision, entry.getOldPath());
1046 break;
1047 default:
1048 throw new Error();
1049 }
1050 }
1051 }
1052
1053 Revision cachedHeadRevision() {
1054 return headRevision;
1055 }
1056
1057 void setHeadRevision(Revision headRevision) {
1058 this.headRevision = headRevision;
1059 }
1060
1061 private RevTree toTree(RevWalk revWalk, Revision revision) {
1062 return toTree(commitIdDatabase, revWalk, revision);
1063 }
1064
1065 static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) {
1066 final ObjectId commitId = commitIdDatabase.get(revision);
1067 try {
1068 return revWalk.parseCommit(commitId).getTree();
1069 } catch (IOException e) {
1070 throw new StorageException("failed to parse a commit: " + commitId, e);
1071 }
1072 }
1073
1074 private RevWalk newRevWalk() {
1075 final RevWalk revWalk = new RevWalk(jGitRepository);
1076 configureRevWalk(revWalk);
1077 return revWalk;
1078 }
1079
1080 static RevWalk newRevWalk(ObjectReader reader) {
1081 final RevWalk revWalk = new RevWalk(reader);
1082 configureRevWalk(revWalk);
1083 return revWalk;
1084 }
1085
1086 private static void configureRevWalk(RevWalk revWalk) {
1087
1088 revWalk.setRewriteParents(false);
1089 }
1090
1091 private void readLock() {
1092 rwLock.readLock().lock();
1093 if (closePending.get() != null) {
1094 rwLock.readLock().unlock();
1095 throw closePending.get().get();
1096 }
1097 }
1098
1099 private void readUnlock() {
1100 rwLock.readLock().unlock();
1101 }
1102
1103 void writeLock() {
1104 rwLock.writeLock().lock();
1105 if (closePending.get() != null) {
1106 writeUnLock();
1107 throw closePending.get().get();
1108 }
1109 }
1110
1111 void writeUnLock() {
1112 rwLock.writeLock().unlock();
1113 }
1114
1115 static void deleteCruft(File repoDir) {
1116 try {
1117 Util.deleteFileTree(repoDir);
1118 } catch (IOException e) {
1119 logger.error("Failed to delete a half-created repository at: {}", repoDir, e);
1120 }
1121 }
1122
1123 @Override
1124 public boolean isEncrypted() {
1125 return isEncrypted;
1126 }
1127
1128 @Override
1129 public String toString() {
1130 return MoreObjects.toStringHelper(this)
1131 .add("dir", jGitRepository.getDirectory())
1132 .toString();
1133 }
1134 }