1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage.repository.git;
18
19 import static com.google.common.base.MoreObjects.firstNonNull;
20 import static com.google.common.base.Preconditions.checkState;
21 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context;
22 import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut;
23 import static java.nio.charset.StandardCharsets.UTF_8;
24 import static java.util.Objects.requireNonNull;
25 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
26 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION;
27
28 import java.io.File;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.lang.reflect.Field;
32 import java.util.ArrayList;
33 import java.util.Collection;
34 import java.util.Collections;
35 import java.util.LinkedHashMap;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Objects;
39 import java.util.Properties;
40 import java.util.StringJoiner;
41 import java.util.concurrent.CompletableFuture;
42 import java.util.concurrent.Executor;
43 import java.util.concurrent.atomic.AtomicReference;
44 import java.util.concurrent.locks.Lock;
45 import java.util.concurrent.locks.ReadWriteLock;
46 import java.util.concurrent.locks.ReentrantReadWriteLock;
47 import java.util.function.BiConsumer;
48 import java.util.function.Supplier;
49 import java.util.regex.Pattern;
50
51 import javax.annotation.Nullable;
52
53 import org.eclipse.jgit.diff.DiffEntry;
54 import org.eclipse.jgit.diff.DiffFormatter;
55 import org.eclipse.jgit.dircache.DirCache;
56 import org.eclipse.jgit.dircache.DirCacheBuilder;
57 import org.eclipse.jgit.dircache.DirCacheEditor;
58 import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath;
59 import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree;
60 import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit;
61 import org.eclipse.jgit.dircache.DirCacheEntry;
62 import org.eclipse.jgit.dircache.DirCacheIterator;
63 import org.eclipse.jgit.lib.CommitBuilder;
64 import org.eclipse.jgit.lib.Constants;
65 import org.eclipse.jgit.lib.FileMode;
66 import org.eclipse.jgit.lib.ObjectId;
67 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
68 import org.eclipse.jgit.lib.ObjectInserter;
69 import org.eclipse.jgit.lib.ObjectReader;
70 import org.eclipse.jgit.lib.PersonIdent;
71 import org.eclipse.jgit.lib.Ref;
72 import org.eclipse.jgit.lib.RefUpdate;
73 import org.eclipse.jgit.lib.RefUpdate.Result;
74 import org.eclipse.jgit.lib.RepositoryBuilder;
75 import org.eclipse.jgit.revwalk.RevCommit;
76 import org.eclipse.jgit.revwalk.RevTree;
77 import org.eclipse.jgit.revwalk.RevWalk;
78 import org.eclipse.jgit.revwalk.TreeRevFilter;
79 import org.eclipse.jgit.revwalk.filter.RevFilter;
80 import org.eclipse.jgit.storage.file.FileBasedConfig;
81 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
82 import org.eclipse.jgit.treewalk.TreeWalk;
83 import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
84 import org.eclipse.jgit.treewalk.filter.TreeFilter;
85 import org.eclipse.jgit.util.SystemReader;
86 import org.slf4j.Logger;
87 import org.slf4j.LoggerFactory;
88
89 import com.fasterxml.jackson.databind.JsonNode;
90 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
91 import com.google.common.annotations.VisibleForTesting;
92 import com.google.common.base.MoreObjects;
93 import com.google.common.collect.ImmutableList;
94
95 import com.linecorp.armeria.server.ServiceRequestContext;
96 import com.linecorp.centraldogma.common.Author;
97 import com.linecorp.centraldogma.common.CentralDogmaException;
98 import com.linecorp.centraldogma.common.Change;
99 import com.linecorp.centraldogma.common.ChangeConflictException;
100 import com.linecorp.centraldogma.common.Commit;
101 import com.linecorp.centraldogma.common.Entry;
102 import com.linecorp.centraldogma.common.EntryNotFoundException;
103 import com.linecorp.centraldogma.common.EntryType;
104 import com.linecorp.centraldogma.common.Markup;
105 import com.linecorp.centraldogma.common.RedundantChangeException;
106 import com.linecorp.centraldogma.common.RepositoryNotFoundException;
107 import com.linecorp.centraldogma.common.Revision;
108 import com.linecorp.centraldogma.common.RevisionNotFoundException;
109 import com.linecorp.centraldogma.common.RevisionRange;
110 import com.linecorp.centraldogma.internal.Jackson;
111 import com.linecorp.centraldogma.internal.Util;
112 import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch;
113 import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode;
114 import com.linecorp.centraldogma.server.command.CommitResult;
115 import com.linecorp.centraldogma.server.internal.IsolatedSystemReader;
116 import com.linecorp.centraldogma.server.internal.JGitUtil;
117 import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
118 import com.linecorp.centraldogma.server.storage.StorageException;
119 import com.linecorp.centraldogma.server.storage.project.Project;
120 import com.linecorp.centraldogma.server.storage.repository.FindOption;
121 import com.linecorp.centraldogma.server.storage.repository.FindOptions;
122 import com.linecorp.centraldogma.server.storage.repository.Repository;
123
124 import difflib.DiffUtils;
125 import difflib.Patch;
126
127
128
129
130 class GitRepository implements Repository {
131
132 private static final Logger logger = LoggerFactory.getLogger(GitRepository.class);
133
134 static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER;
135
136 private static final byte[] EMPTY_BYTE = new byte[0];
137 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
138
139 private static final Field revWalkObjectsField;
140
141 static {
142 final String jgitPomProperties = "META-INF/maven/org.eclipse.jgit/org.eclipse.jgit/pom.properties";
143 try (InputStream is = SystemReader.class.getClassLoader().getResourceAsStream(jgitPomProperties)) {
144 final Properties props = new Properties();
145 props.load(is);
146 final Object jgitVersion = props.get("version");
147 if (jgitVersion != null) {
148 logger.info("Using JGit: {}", jgitVersion);
149 }
150 } catch (IOException e) {
151 logger.debug("Failed to read JGit version", e);
152 }
153
154 IsolatedSystemReader.install();
155
156 Field field = null;
157 try {
158 field = RevWalk.class.getDeclaredField("objects");
159 if (field.getType() != ObjectIdOwnerMap.class) {
160 throw new IllegalStateException(
161 RevWalk.class.getSimpleName() + ".objects is not an " +
162 ObjectIdOwnerMap.class.getSimpleName() + '.');
163 }
164 field.setAccessible(true);
165 } catch (NoSuchFieldException e) {
166 throw new IllegalStateException(
167 RevWalk.class.getSimpleName() + ".objects does not exist.");
168 }
169
170 revWalkObjectsField = field;
171 }
172
173 private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
174 private final Project parent;
175 private final Executor repositoryWorker;
176 private final long creationTimeMillis;
177 private final Author author;
178 @VisibleForTesting
179 @Nullable
180 final RepositoryCache cache;
181 private final String name;
182 private final org.eclipse.jgit.lib.Repository jGitRepository;
183 private final CommitIdDatabase commitIdDatabase;
184 @VisibleForTesting
185 final CommitWatchers commitWatchers = new CommitWatchers();
186 private final AtomicReference<Supplier<CentralDogmaException>> closePending = new AtomicReference<>();
187 private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
188
189
190
191
192 private volatile Revision headRevision;
193
194
195
196
197
198
199
200
201
202
203
204 @VisibleForTesting
205 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
206 long creationTimeMillis, Author author) {
207 this(parent, repoDir, repositoryWorker, creationTimeMillis, author, null);
208 }
209
210
211
212
213
214
215
216
217
218
219
220 GitRepository(Project parent, File repoDir, Executor repositoryWorker,
221 long creationTimeMillis, Author author, @Nullable RepositoryCache cache) {
222
223 this.parent = requireNonNull(parent, "parent");
224 name = requireNonNull(repoDir, "repoDir").getName();
225 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
226 this.creationTimeMillis = creationTimeMillis;
227 this.author = requireNonNull(author, "author");
228 this.cache = cache;
229
230 final RepositoryBuilder repositoryBuilder = new RepositoryBuilder().setGitDir(repoDir).setBare();
231 boolean success = false;
232 try {
233
234 try (org.eclipse.jgit.lib.Repository initRepo = repositoryBuilder.build()) {
235 if (exist(repoDir)) {
236 throw new StorageException(
237 "failed to create a repository at: " + repoDir + " (exists already)");
238 }
239 initRepo.create(true);
240
241
242 JGitUtil.applyDefaultsAndSave(initRepo.getConfig());
243 }
244
245
246 jGitRepository = new RepositoryBuilder().setGitDir(repoDir).build();
247
248
249 final RefUpdate head = jGitRepository.updateRef(Constants.HEAD);
250 head.disableRefLog();
251 head.link(Constants.R_HEADS + Constants.MASTER);
252
253
254 commitIdDatabase = new CommitIdDatabase(jGitRepository);
255
256
257 commit0(null, Revision.INIT, creationTimeMillis, author,
258 "Create a new repository", "", Markup.PLAINTEXT,
259 Collections.emptyList(), true);
260
261 headRevision = Revision.INIT;
262 success = true;
263 } catch (IOException e) {
264 throw new StorageException("failed to create a repository at: " + repoDir, e);
265 } finally {
266 if (!success) {
267 internalClose();
268
269 deleteCruft(repoDir);
270 }
271 }
272 }
273
274
275
276
277
278
279
280
281
282 GitRepository(Project parent, File repoDir, Executor repositoryWorker, @Nullable RepositoryCache cache) {
283 this.parent = requireNonNull(parent, "parent");
284 name = requireNonNull(repoDir, "repoDir").getName();
285 this.repositoryWorker = requireNonNull(repositoryWorker, "repositoryWorker");
286 this.cache = cache;
287
288 final RepositoryBuilder repositoryBuilder = new RepositoryBuilder().setGitDir(repoDir).setBare();
289 try {
290 jGitRepository = repositoryBuilder.build();
291 if (!exist(repoDir)) {
292 throw new RepositoryNotFoundException(repoDir.toString());
293 }
294
295
296 final int formatVersion = jGitRepository.getConfig().getInt(
297 CONFIG_CORE_SECTION, null, CONFIG_KEY_REPO_FORMAT_VERSION, 0);
298 if (formatVersion != JGitUtil.REPO_FORMAT_VERSION) {
299 throw new StorageException("unsupported repository format version: " + formatVersion +
300 " (expected: " + JGitUtil.REPO_FORMAT_VERSION + ')');
301 }
302
303
304 JGitUtil.applyDefaultsAndSave(jGitRepository.getConfig());
305 } catch (IOException e) {
306 throw new StorageException("failed to open a repository at: " + repoDir, e);
307 }
308
309 boolean success = false;
310 try {
311 headRevision = uncachedHeadRevision();
312 commitIdDatabase = new CommitIdDatabase(jGitRepository);
313 if (!headRevision.equals(commitIdDatabase.headRevision())) {
314 commitIdDatabase.rebuild(jGitRepository);
315 assert headRevision.equals(commitIdDatabase.headRevision());
316 }
317 final Commit initialCommit = blockingHistory(Revision.INIT, Revision.INIT, ALL_PATH, 1).get(0);
318 creationTimeMillis = initialCommit.when();
319 author = initialCommit.author();
320 success = true;
321 } finally {
322 if (!success) {
323 internalClose();
324 }
325 }
326 }
327
328 private static boolean exist(File repoDir) {
329 try {
330 final RepositoryBuilder repositoryBuilder = new RepositoryBuilder().setGitDir(repoDir);
331 final org.eclipse.jgit.lib.Repository repository = repositoryBuilder.build();
332 if (repository.getConfig() instanceof FileBasedConfig) {
333 return ((FileBasedConfig) repository.getConfig()).getFile().exists();
334 }
335 return repository.getDirectory().exists();
336 } catch (IOException e) {
337 throw new StorageException("failed to check if repository exists at " + repoDir, e);
338 }
339 }
340
341
342
343
344
345
346
347 void close(Supplier<CentralDogmaException> failureCauseSupplier) {
348 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
349 if (closePending.compareAndSet(null, failureCauseSupplier)) {
350 repositoryWorker.execute(() -> {
351
352 rwLock.writeLock().lock();
353 try {
354 if (commitIdDatabase != null) {
355 try {
356 commitIdDatabase.close();
357 } catch (Exception e) {
358 logger.warn("Failed to close a commitId database:", e);
359 }
360 }
361
362 if (jGitRepository != null) {
363 try {
364 jGitRepository.close();
365 } catch (Exception e) {
366 logger.warn("Failed to close a Git repository: {}",
367 jGitRepository.getDirectory(), e);
368 }
369 }
370 } finally {
371 try {
372 rwLock.writeLock().unlock();
373 } finally {
374 commitWatchers.close(failureCauseSupplier);
375 closeFuture.complete(null);
376 }
377 }
378 });
379 }
380
381 closeFuture.join();
382 }
383
384 void internalClose() {
385 close(() -> new CentralDogmaException("should never reach here"));
386 }
387
388 @Override
389 public Project parent() {
390 return parent;
391 }
392
393 @Override
394 public String name() {
395 return name;
396 }
397
398 @Override
399 public long creationTimeMillis() {
400 return creationTimeMillis;
401 }
402
403 @Override
404 public Author author() {
405 return author;
406 }
407
408 @Override
409 public Revision normalizeNow(Revision revision) {
410 return normalizeNow(revision, cachedHeadRevision().major());
411 }
412
413 private static Revision normalizeNow(Revision revision, int baseMajor) {
414 requireNonNull(revision, "revision");
415
416 int major = revision.major();
417
418 if (major >= 0) {
419 if (major > baseMajor) {
420 throw new RevisionNotFoundException(revision);
421 }
422 } else {
423 major = baseMajor + major + 1;
424 if (major <= 0) {
425 throw new RevisionNotFoundException(revision);
426 }
427 }
428
429
430 if (revision.major() == major) {
431 return revision;
432 } else {
433 return new Revision(major);
434 }
435 }
436
437 @Override
438 public RevisionRange normalizeNow(Revision from, Revision to) {
439 final int baseMajor = cachedHeadRevision().major();
440 return new RevisionRange(normalizeNow(from, baseMajor), normalizeNow(to, baseMajor));
441 }
442
443 @Override
444 public CompletableFuture<Map<String, Entry<?>>> find(
445 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
446 final ServiceRequestContext ctx = context();
447 return CompletableFuture.supplyAsync(() -> {
448 failFastIfTimedOut(this, logger, ctx, "find", revision, pathPattern, options);
449 return blockingFind(revision, pathPattern, options);
450 }, repositoryWorker);
451 }
452
453 private Map<String, Entry<?>> blockingFind(
454 Revision revision, String pathPattern, Map<FindOption<?>, ?> options) {
455
456 requireNonNull(pathPattern, "pathPattern");
457 requireNonNull(revision, "revision");
458 requireNonNull(options, "options");
459
460 final Revision normRevision = normalizeNow(revision);
461 final boolean fetchContent = FindOption.FETCH_CONTENT.get(options);
462 final int maxEntries = FindOption.MAX_ENTRIES.get(options);
463
464 readLock();
465 try (ObjectReader reader = jGitRepository.newObjectReader();
466 TreeWalk treeWalk = new TreeWalk(reader);
467 RevWalk revWalk = newRevWalk(reader)) {
468
469
470 final Revision headRevision = cachedHeadRevision();
471 if (normRevision.compareTo(headRevision) > 0) {
472 return Collections.emptyMap();
473 }
474
475 if ("/".equals(pathPattern)) {
476 return Collections.singletonMap(pathPattern, Entry.ofDirectory(normRevision, "/"));
477 }
478
479 final Map<String, Entry<?>> result = new LinkedHashMap<>();
480 final ObjectId commitId = commitIdDatabase.get(normRevision);
481 final RevCommit revCommit = revWalk.parseCommit(commitId);
482 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
483
484 final RevTree revTree = revCommit.getTree();
485 treeWalk.addTree(revTree.getId());
486 while (treeWalk.next() && result.size() < maxEntries) {
487 final boolean matches = filter.matches(treeWalk);
488 final String path = '/' + treeWalk.getPathString();
489
490
491 if (treeWalk.isSubtree()) {
492 if (matches) {
493
494 result.put(path, Entry.ofDirectory(normRevision, path));
495 }
496
497 treeWalk.enterSubtree();
498 continue;
499 }
500
501 if (!matches) {
502 continue;
503 }
504
505
506 final Entry<?> entry;
507 final EntryType entryType = EntryType.guessFromPath(path);
508 if (fetchContent) {
509 final byte[] content = reader.open(treeWalk.getObjectId(0)).getBytes();
510 switch (entryType) {
511 case JSON:
512 final JsonNode jsonNode = Jackson.readTree(content);
513 entry = Entry.ofJson(normRevision, path, jsonNode);
514 break;
515 case TEXT:
516 final String strVal = sanitizeText(new String(content, UTF_8));
517 entry = Entry.ofText(normRevision, path, strVal);
518 break;
519 default:
520 throw new Error("unexpected entry type: " + entryType);
521 }
522 } else {
523 switch (entryType) {
524 case JSON:
525 entry = Entry.ofJson(normRevision, path, Jackson.nullNode);
526 break;
527 case TEXT:
528 entry = Entry.ofText(normRevision, path, "");
529 break;
530 default:
531 throw new Error("unexpected entry type: " + entryType);
532 }
533 }
534
535 result.put(path, entry);
536 }
537
538 return Util.unsafeCast(result);
539 } catch (CentralDogmaException | IllegalArgumentException e) {
540 throw e;
541 } catch (Exception e) {
542 throw new StorageException(
543 "failed to get data from '" + parent.name() + '/' + name + "' at " + pathPattern +
544 " for " + revision, e);
545 } finally {
546 readUnlock();
547 }
548 }
549
550 @Override
551 public CompletableFuture<List<Commit>> history(
552 Revision from, Revision to, String pathPattern, int maxCommits) {
553
554 final ServiceRequestContext ctx = context();
555 return CompletableFuture.supplyAsync(() -> {
556 failFastIfTimedOut(this, logger, ctx, "history", from, to, pathPattern, maxCommits);
557 return blockingHistory(from, to, pathPattern, maxCommits);
558 }, repositoryWorker);
559 }
560
561 private List<Commit> blockingHistory(Revision from, Revision to, String pathPattern, int maxCommits) {
562 requireNonNull(pathPattern, "pathPattern");
563 requireNonNull(from, "from");
564 requireNonNull(to, "to");
565 if (maxCommits <= 0) {
566 throw new IllegalArgumentException("maxCommits: " + maxCommits + " (expected: > 0)");
567 }
568
569 maxCommits = Math.min(maxCommits, MAX_MAX_COMMITS);
570
571 final RevisionRange range = normalizeNow(from, to);
572 final RevisionRange descendingRange = range.toDescending();
573
574
575 readLock();
576 try (RevWalk revWalk = newRevWalk()) {
577 final ObjectIdOwnerMap<?> revWalkInternalMap =
578 (ObjectIdOwnerMap<?>) revWalkObjectsField.get(revWalk);
579
580 final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from());
581 final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to());
582
583 revWalk.markStart(revWalk.parseCommit(fromCommitId));
584 revWalk.setRetainBody(false);
585
586
587
588
589
590
591 final RevFilter filter = new TreeRevFilter(revWalk, AndTreeFilter.create(
592 TreeFilter.ANY_DIFF, PathPatternFilter.of(pathPattern)));
593
594
595
596 final int maxNumProcessedCommits = Math.max(maxCommits * 10, MAX_MAX_COMMITS);
597
598 final List<Commit> commitList = new ArrayList<>();
599 int numProcessedCommits = 0;
600 for (RevCommit revCommit : revWalk) {
601 numProcessedCommits++;
602
603 if (filter.include(revWalk, revCommit)) {
604 revWalk.parseBody(revCommit);
605 commitList.add(toCommit(revCommit));
606 revCommit.disposeBody();
607 }
608
609 if (revCommit.getId().equals(toCommitId) ||
610 commitList.size() >= maxCommits ||
611
612 numProcessedCommits >= maxNumProcessedCommits) {
613 break;
614 }
615
616
617
618 if (numProcessedCommits % 16 == 0) {
619 revWalkInternalMap.clear();
620 }
621 }
622
623
624
625 if (commitList.size() < maxCommits &&
626 descendingRange.to().major() == 1 &&
627 pathPattern.contains(ALL_PATH)) {
628 try (RevWalk tmpRevWalk = newRevWalk()) {
629 final RevCommit lastRevCommit = tmpRevWalk.parseCommit(toCommitId);
630 commitList.add(toCommit(lastRevCommit));
631 }
632 }
633
634 if (!descendingRange.equals(range)) {
635 Collections.reverse(commitList);
636 }
637
638 return commitList;
639 } catch (CentralDogmaException e) {
640 throw e;
641 } catch (Exception e) {
642 throw new StorageException(
643 "failed to retrieve the history: " + parent.name() + '/' + name +
644 " (" + pathPattern + ", " + from + ".." + to + ')', e);
645 } finally {
646 readUnlock();
647 }
648 }
649
650 private static Commit toCommit(RevCommit revCommit) {
651 final Author author;
652 final PersonIdent committerIdent = revCommit.getCommitterIdent();
653 final long when;
654 if (committerIdent == null) {
655 author = Author.UNKNOWN;
656 when = 0;
657 } else {
658 author = new Author(committerIdent.getName(), committerIdent.getEmailAddress());
659 when = committerIdent.getWhen().getTime();
660 }
661
662 try {
663 return CommitUtil.newCommit(author, when, revCommit.getFullMessage());
664 } catch (Exception e) {
665 throw new StorageException("failed to create a Commit", e);
666 }
667 }
668
669
670
671
672
673
674
675
676
677
678 @Override
679 public CompletableFuture<Map<String, Change<?>>> diff(Revision from, Revision to, String pathPattern) {
680 final ServiceRequestContext ctx = context();
681 return CompletableFuture.supplyAsync(() -> {
682 requireNonNull(from, "from");
683 requireNonNull(to, "to");
684 requireNonNull(pathPattern, "pathPattern");
685
686 failFastIfTimedOut(this, logger, ctx, "diff", from, to, pathPattern);
687
688 final RevisionRange range = normalizeNow(from, to).toAscending();
689 readLock();
690 try (RevWalk rw = newRevWalk()) {
691 final RevTree treeA = rw.parseTree(commitIdDatabase.get(range.from()));
692 final RevTree treeB = rw.parseTree(commitIdDatabase.get(range.to()));
693
694
695
696 return toChangeMap(blockingCompareTreesUncached(treeA, treeB,
697 pathPatternFilterOrTreeFilter(pathPattern)));
698 } catch (StorageException e) {
699 throw e;
700 } catch (Exception e) {
701 throw new StorageException("failed to parse two trees: range=" + range, e);
702 } finally {
703 readUnlock();
704 }
705 }, repositoryWorker);
706 }
707
708 private static TreeFilter pathPatternFilterOrTreeFilter(@Nullable String pathPattern) {
709 if (pathPattern == null) {
710 return TreeFilter.ALL;
711 }
712
713 final PathPatternFilter pathPatternFilter = PathPatternFilter.of(pathPattern);
714 return pathPatternFilter.matchesAll() ? TreeFilter.ALL : pathPatternFilter;
715 }
716
717 @Override
718 public CompletableFuture<Map<String, Change<?>>> previewDiff(Revision baseRevision,
719 Iterable<Change<?>> changes) {
720 final ServiceRequestContext ctx = context();
721 return CompletableFuture.supplyAsync(() -> {
722 failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision);
723 return blockingPreviewDiff(baseRevision, changes);
724 }, repositoryWorker);
725 }
726
727 private Map<String, Change<?>> blockingPreviewDiff(Revision baseRevision, Iterable<Change<?>> changes) {
728 requireNonNull(baseRevision, "baseRevision");
729 requireNonNull(changes, "changes");
730 baseRevision = normalizeNow(baseRevision);
731
732 readLock();
733 try (ObjectReader reader = jGitRepository.newObjectReader();
734 RevWalk revWalk = newRevWalk(reader);
735 DiffFormatter diffFormatter = new DiffFormatter(null)) {
736
737 final ObjectId baseTreeId = toTree(revWalk, baseRevision);
738 final DirCache dirCache = DirCache.newInCore();
739 final int numEdits = applyChanges(baseRevision, baseTreeId, dirCache, changes);
740 if (numEdits == 0) {
741 return Collections.emptyMap();
742 }
743
744 final CanonicalTreeParser p = new CanonicalTreeParser();
745 p.reset(reader, baseTreeId);
746 diffFormatter.setRepository(jGitRepository);
747 final List<DiffEntry> result = diffFormatter.scan(p, new DirCacheIterator(dirCache));
748 return toChangeMap(result);
749 } catch (IOException e) {
750 throw new StorageException("failed to perform a dry-run diff", e);
751 } finally {
752 readUnlock();
753 }
754 }
755
756 private Map<String, Change<?>> toChangeMap(List<DiffEntry> diffEntryList) {
757 try (ObjectReader reader = jGitRepository.newObjectReader()) {
758 final Map<String, Change<?>> changeMap = new LinkedHashMap<>();
759
760 for (DiffEntry diffEntry : diffEntryList) {
761 final String oldPath = '/' + diffEntry.getOldPath();
762 final String newPath = '/' + diffEntry.getNewPath();
763
764 switch (diffEntry.getChangeType()) {
765 case MODIFY:
766 final EntryType oldEntryType = EntryType.guessFromPath(oldPath);
767 switch (oldEntryType) {
768 case JSON:
769 if (!oldPath.equals(newPath)) {
770 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
771 }
772
773 final JsonNode oldJsonNode =
774 Jackson.readTree(
775 reader.open(diffEntry.getOldId().toObjectId()).getBytes());
776 final JsonNode newJsonNode =
777 Jackson.readTree(
778 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
779 final JsonPatch patch =
780 JsonPatch.generate(oldJsonNode, newJsonNode, ReplaceMode.SAFE);
781
782 if (!patch.isEmpty()) {
783 putChange(changeMap, newPath,
784 Change.ofJsonPatch(newPath, Jackson.valueToTree(patch)));
785 }
786 break;
787 case TEXT:
788 final String oldText = sanitizeText(new String(
789 reader.open(diffEntry.getOldId().toObjectId()).getBytes(), UTF_8));
790
791 final String newText = sanitizeText(new String(
792 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
793
794 if (!oldPath.equals(newPath)) {
795 putChange(changeMap, oldPath, Change.ofRename(oldPath, newPath));
796 }
797
798 if (!oldText.equals(newText)) {
799 putChange(changeMap, newPath,
800 Change.ofTextPatch(newPath, oldText, newText));
801 }
802 break;
803 default:
804 throw new Error("unexpected old entry type: " + oldEntryType);
805 }
806 break;
807 case ADD:
808 final EntryType newEntryType = EntryType.guessFromPath(newPath);
809 switch (newEntryType) {
810 case JSON: {
811 final JsonNode jsonNode = Jackson.readTree(
812 reader.open(diffEntry.getNewId().toObjectId()).getBytes());
813
814 putChange(changeMap, newPath, Change.ofJsonUpsert(newPath, jsonNode));
815 break;
816 }
817 case TEXT: {
818 final String text = sanitizeText(new String(
819 reader.open(diffEntry.getNewId().toObjectId()).getBytes(), UTF_8));
820
821 putChange(changeMap, newPath, Change.ofTextUpsert(newPath, text));
822 break;
823 }
824 default:
825 throw new Error("unexpected new entry type: " + newEntryType);
826 }
827 break;
828 case DELETE:
829 putChange(changeMap, oldPath, Change.ofRemoval(oldPath));
830 break;
831 default:
832 throw new Error();
833 }
834 }
835 return changeMap;
836 } catch (Exception e) {
837 throw new StorageException("failed to convert list of DiffEntry to Changes map", e);
838 }
839 }
840
841 private static void putChange(Map<String, Change<?>> changeMap, String path, Change<?> change) {
842 final Change<?> oldChange = changeMap.put(path, change);
843 assert oldChange == null;
844 }
845
846 @Override
847 public CompletableFuture<CommitResult> commit(
848 Revision baseRevision, long commitTimeMillis, Author author, String summary,
849 String detail, Markup markup, Iterable<Change<?>> changes, boolean directExecution) {
850 requireNonNull(baseRevision, "baseRevision");
851 requireNonNull(author, "author");
852 requireNonNull(summary, "summary");
853 requireNonNull(detail, "detail");
854 requireNonNull(markup, "markup");
855 requireNonNull(changes, "changes");
856
857 final ServiceRequestContext ctx = context();
858 return CompletableFuture.supplyAsync(() -> {
859 failFastIfTimedOut(this, logger, ctx, "commit", baseRevision, author, summary);
860 return blockingCommit(baseRevision, commitTimeMillis,
861 author, summary, detail, markup, changes, false, directExecution);
862 }, repositoryWorker);
863 }
864
865 private CommitResult blockingCommit(
866 Revision baseRevision, long commitTimeMillis, Author author, String summary,
867 String detail, Markup markup, Iterable<Change<?>> changes, boolean allowEmptyCommit,
868 boolean directExecution) {
869
870 requireNonNull(baseRevision, "baseRevision");
871
872 final RevisionAndEntries res;
873 final Iterable<Change<?>> applyingChanges;
874 writeLock();
875 try {
876 final Revision normBaseRevision = normalizeNow(baseRevision);
877 final Revision headRevision = cachedHeadRevision();
878 if (headRevision.major() != normBaseRevision.major()) {
879 throw new ChangeConflictException(
880 "invalid baseRevision: " + baseRevision + " (expected: " + headRevision +
881 " or equivalent)");
882 }
883
884 if (directExecution) {
885 applyingChanges = blockingPreviewDiff(normBaseRevision, changes).values();
886 } else {
887 applyingChanges = changes;
888 }
889 res = commit0(headRevision, headRevision.forward(1), commitTimeMillis,
890 author, summary, detail, markup, applyingChanges, allowEmptyCommit);
891
892 this.headRevision = res.revision;
893 } finally {
894 writeUnLock();
895 }
896
897
898 notifyWatchers(res.revision, res.diffEntries);
899 return CommitResult.of(res.revision, applyingChanges);
900 }
901
902 private RevisionAndEntries commit0(@Nullable Revision prevRevision, Revision nextRevision,
903 long commitTimeMillis, Author author, String summary,
904 String detail, Markup markup,
905 Iterable<Change<?>> changes, boolean allowEmpty) {
906
907 requireNonNull(author, "author");
908 requireNonNull(summary, "summary");
909 requireNonNull(changes, "changes");
910 requireNonNull(detail, "detail");
911 requireNonNull(markup, "markup");
912
913 assert prevRevision == null || prevRevision.major() > 0;
914 assert nextRevision.major() > 0;
915
916 try (ObjectInserter inserter = jGitRepository.newObjectInserter();
917 ObjectReader reader = jGitRepository.newObjectReader();
918 RevWalk revWalk = newRevWalk(reader)) {
919
920 final ObjectId prevTreeId = prevRevision != null ? toTree(revWalk, prevRevision) : null;
921
922
923
924
925
926 final DirCache dirCache = DirCache.newInCore();
927
928
929 final int numEdits = applyChanges(prevRevision, prevTreeId, dirCache, changes);
930
931
932 final List<DiffEntry> diffEntries;
933 boolean isEmpty = numEdits == 0;
934 if (!isEmpty) {
935
936 final CanonicalTreeParser p = new CanonicalTreeParser();
937 p.reset(reader, prevTreeId);
938 final DiffFormatter diffFormatter = new DiffFormatter(null);
939 diffFormatter.setRepository(jGitRepository);
940 diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache));
941 isEmpty = diffEntries.isEmpty();
942 } else {
943 diffEntries = ImmutableList.of();
944 }
945
946 if (!allowEmpty && isEmpty) {
947 throw new RedundantChangeException(
948 "changes did not change anything in " + parent().name() + '/' + name() +
949 " at revision " + (prevRevision != null ? prevRevision.major() : 0) +
950 ": " + changes);
951 }
952
953
954 final ObjectId nextTreeId = dirCache.writeTree(inserter);
955
956
957 final PersonIdent personIdent = new PersonIdent(author.name(), author.email(),
958 commitTimeMillis / 1000L * 1000L, 0);
959
960 final CommitBuilder commitBuilder = new CommitBuilder();
961
962 commitBuilder.setAuthor(personIdent);
963 commitBuilder.setCommitter(personIdent);
964 commitBuilder.setTreeId(nextTreeId);
965 commitBuilder.setEncoding(UTF_8);
966
967
968 commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision));
969
970
971 if (prevRevision != null) {
972 commitBuilder.setParentId(commitIdDatabase.get(prevRevision));
973 }
974
975 final ObjectId nextCommitId = inserter.insert(commitBuilder);
976 inserter.flush();
977
978
979 commitIdDatabase.put(nextRevision, nextCommitId);
980 doRefUpdate(revWalk, R_HEADS_MASTER, nextCommitId);
981
982 return new RevisionAndEntries(nextRevision, diffEntries);
983 } catch (CentralDogmaException | IllegalArgumentException e) {
984 throw e;
985 } catch (Exception e) {
986 throw new StorageException("failed to push at '" + parent.name() + '/' + name + '\'', e);
987 }
988 }
989
990 private int applyChanges(@Nullable Revision baseRevision, @Nullable ObjectId baseTreeId, DirCache dirCache,
991 Iterable<Change<?>> changes) {
992
993 int numEdits = 0;
994
995 try (ObjectInserter inserter = jGitRepository.newObjectInserter();
996 ObjectReader reader = jGitRepository.newObjectReader()) {
997
998 if (baseTreeId != null) {
999
1000 final DirCacheBuilder builder = dirCache.builder();
1001
1002
1003 builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId);
1004 builder.finish();
1005 }
1006
1007
1008 for (Change<?> change : changes) {
1009 final String changePath = change.path().substring(1);
1010 final DirCacheEntry oldEntry = dirCache.getEntry(changePath);
1011 final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes()
1012 : null;
1013
1014 switch (change.type()) {
1015 case UPSERT_JSON: {
1016 final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null;
1017 final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(),
1018 JsonNodeFactory.instance.nullNode());
1019
1020
1021 if (!Objects.equals(newJsonNode, oldJsonNode)) {
1022 applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode));
1023 numEdits++;
1024 }
1025 break;
1026 }
1027 case UPSERT_TEXT: {
1028 final String sanitizedOldText;
1029 if (oldContent != null) {
1030 sanitizedOldText = sanitizeText(new String(oldContent, UTF_8));
1031 } else {
1032 sanitizedOldText = null;
1033 }
1034
1035 final String sanitizedNewText = sanitizeText(change.contentAsText());
1036
1037
1038 if (!sanitizedNewText.equals(sanitizedOldText)) {
1039 applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText));
1040 numEdits++;
1041 }
1042 break;
1043 }
1044 case REMOVE:
1045 if (oldEntry != null) {
1046 applyPathEdit(dirCache, new DeletePath(changePath));
1047 numEdits++;
1048 break;
1049 }
1050
1051
1052 if (applyDirectoryEdits(dirCache, changePath, null, change)) {
1053 numEdits++;
1054 } else {
1055
1056 reportNonExistentEntry(change);
1057 break;
1058 }
1059 break;
1060 case RENAME: {
1061 final String newPath =
1062 ((String) change.content()).substring(1);
1063
1064 if (dirCache.getEntry(newPath) != null) {
1065 throw new ChangeConflictException("a file exists at the target path: " + change);
1066 }
1067
1068 if (oldEntry != null) {
1069 if (changePath.equals(newPath)) {
1070
1071 break;
1072 }
1073
1074 final DirCacheEditor editor = dirCache.editor();
1075 editor.add(new DeletePath(changePath));
1076 editor.add(new CopyOldEntry(newPath, oldEntry));
1077 editor.finish();
1078 numEdits++;
1079 break;
1080 }
1081
1082
1083 if (applyDirectoryEdits(dirCache, changePath, newPath, change)) {
1084 numEdits++;
1085 } else {
1086
1087 reportNonExistentEntry(change);
1088 }
1089 break;
1090 }
1091 case APPLY_JSON_PATCH: {
1092 final JsonNode oldJsonNode;
1093 if (oldContent != null) {
1094 oldJsonNode = Jackson.readTree(oldContent);
1095 } else {
1096 oldJsonNode = Jackson.nullNode;
1097 }
1098
1099 final JsonNode newJsonNode;
1100 try {
1101 newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode);
1102 } catch (Exception e) {
1103 throw new ChangeConflictException("failed to apply JSON patch: " + change, e);
1104 }
1105
1106
1107 if (!newJsonNode.equals(oldJsonNode)) {
1108 applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode));
1109 numEdits++;
1110 }
1111 break;
1112 }
1113 case APPLY_TEXT_PATCH:
1114 final Patch<String> patch = DiffUtils.parseUnifiedDiff(
1115 Util.stringToLines(sanitizeText((String) change.content())));
1116
1117 final String sanitizedOldText;
1118 final List<String> sanitizedOldTextLines;
1119 if (oldContent != null) {
1120 sanitizedOldText = sanitizeText(new String(oldContent, UTF_8));
1121 sanitizedOldTextLines = Util.stringToLines(sanitizedOldText);
1122 } else {
1123 sanitizedOldText = null;
1124 sanitizedOldTextLines = Collections.emptyList();
1125 }
1126
1127 final String newText;
1128 try {
1129 final List<String> newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch);
1130 if (newTextLines.isEmpty()) {
1131 newText = "";
1132 } else {
1133 final StringJoiner joiner = new StringJoiner("\n", "", "\n");
1134 for (String line : newTextLines) {
1135 joiner.add(line);
1136 }
1137 newText = joiner.toString();
1138 }
1139 } catch (Exception e) {
1140 throw new ChangeConflictException("failed to apply text patch: " + change, e);
1141 }
1142
1143
1144 if (!newText.equals(sanitizedOldText)) {
1145 applyPathEdit(dirCache, new InsertText(changePath, inserter, newText));
1146 numEdits++;
1147 }
1148 break;
1149 }
1150 }
1151 } catch (CentralDogmaException | IllegalArgumentException e) {
1152 throw e;
1153 } catch (Exception e) {
1154 throw new StorageException("failed to apply changes on revision " + baseRevision, e);
1155 }
1156 return numEdits;
1157 }
1158
1159
1160
1161
1162 private static String sanitizeText(String text) {
1163 if (text.indexOf('\r') >= 0) {
1164 text = CR.matcher(text).replaceAll("");
1165 }
1166 if (!text.isEmpty() && !text.endsWith("\n")) {
1167 text += "\n";
1168 }
1169 return text;
1170 }
1171
1172 private static void reportNonExistentEntry(Change<?> change) {
1173 throw new ChangeConflictException("non-existent file/directory: " + change);
1174 }
1175
1176 private static void applyPathEdit(DirCache dirCache, PathEdit edit) {
1177 final DirCacheEditor e = dirCache.editor();
1178 e.add(edit);
1179 e.finish();
1180 }
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190 private static boolean applyDirectoryEdits(DirCache dirCache,
1191 String oldDir, @Nullable String newDir, Change<?> change) {
1192
1193 if (!oldDir.endsWith("/")) {
1194 oldDir += '/';
1195 }
1196 if (newDir != null && !newDir.endsWith("/")) {
1197 newDir += '/';
1198 }
1199
1200 final byte[] rawOldDir = Constants.encode(oldDir);
1201 final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null;
1202 final int numEntries = dirCache.getEntryCount();
1203 DirCacheEditor editor = null;
1204
1205 loop:
1206 for (int i = 0; i < numEntries; i++) {
1207 final DirCacheEntry e = dirCache.getEntry(i);
1208 final byte[] rawPath = e.getRawPath();
1209
1210
1211 if (rawNewDir != null) {
1212 boolean conflict = true;
1213 if (rawPath.length > rawNewDir.length) {
1214
1215 for (int j = 0; j < rawNewDir.length; j++) {
1216 if (rawNewDir[j] != rawPath[j]) {
1217 conflict = false;
1218 break;
1219 }
1220 }
1221 } else if (rawPath.length == rawNewDir.length - 1) {
1222
1223 for (int j = 0; j < rawNewDir.length - 1; j++) {
1224 if (rawNewDir[j] != rawPath[j]) {
1225 conflict = false;
1226 break;
1227 }
1228 }
1229 } else {
1230 conflict = false;
1231 }
1232
1233 if (conflict) {
1234 throw new ChangeConflictException("target directory exists already: " + change);
1235 }
1236 }
1237
1238
1239 if (rawPath.length <= rawOldDir.length) {
1240 continue;
1241 }
1242 for (int j = 0; j < rawOldDir.length; j++) {
1243 if (rawOldDir[j] != rawPath[j]) {
1244 continue loop;
1245 }
1246 }
1247
1248
1249
1250 if (editor == null) {
1251 editor = dirCache.editor();
1252 editor.add(new DeleteTree(oldDir));
1253 if (newDir == null) {
1254
1255 break;
1256 }
1257 }
1258
1259 assert newDir != null;
1260
1261 final String oldPath = e.getPathString();
1262 final String newPath = newDir + oldPath.substring(oldDir.length());
1263 editor.add(new CopyOldEntry(newPath, e));
1264 }
1265
1266 if (editor != null) {
1267 editor.finish();
1268 return true;
1269 } else {
1270 return false;
1271 }
1272 }
1273
1274 private void doRefUpdate(RevWalk revWalk, String ref, ObjectId commitId) throws IOException {
1275 doRefUpdate(jGitRepository, revWalk, ref, commitId);
1276 }
1277
1278 @VisibleForTesting
1279 static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
1280 String ref, ObjectId commitId) throws IOException {
1281
1282 if (ref.startsWith(Constants.R_TAGS)) {
1283 final Ref oldRef = jGitRepository.exactRef(ref);
1284 if (oldRef != null) {
1285 throw new StorageException("tag ref exists already: " + ref);
1286 }
1287 }
1288
1289 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
1290 refUpdate.setNewObjectId(commitId);
1291
1292 final Result res = refUpdate.update(revWalk);
1293 switch (res) {
1294 case NEW:
1295 case FAST_FORWARD:
1296
1297 break;
1298 default:
1299 throw new StorageException("unexpected refUpdate state: " + res);
1300 }
1301 }
1302
1303 @Override
1304 public CompletableFuture<Revision> findLatestRevision(Revision lastKnownRevision, String pathPattern,
1305 boolean errorOnEntryNotFound) {
1306 requireNonNull(lastKnownRevision, "lastKnownRevision");
1307 requireNonNull(pathPattern, "pathPattern");
1308
1309 final ServiceRequestContext ctx = context();
1310 return CompletableFuture.supplyAsync(() -> {
1311 failFastIfTimedOut(this, logger, ctx, "findLatestRevision", lastKnownRevision, pathPattern);
1312 return blockingFindLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound);
1313 }, repositoryWorker);
1314 }
1315
1316 @Nullable
1317 private Revision blockingFindLatestRevision(Revision lastKnownRevision, String pathPattern,
1318 boolean errorOnEntryNotFound) {
1319 final RevisionRange range = normalizeNow(lastKnownRevision, Revision.HEAD);
1320 if (range.from().equals(range.to())) {
1321
1322 if (!errorOnEntryNotFound) {
1323 return null;
1324 }
1325
1326 final Map<String, Entry<?>> entries =
1327 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
1328 if (!entries.isEmpty()) {
1329
1330 return null;
1331 }
1332 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
1333 }
1334
1335 if (range.from().major() == 1) {
1336
1337 final Map<String, Entry<?>> entries =
1338 blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT);
1339 if (entries.isEmpty()) {
1340 if (!errorOnEntryNotFound) {
1341 return null;
1342 }
1343 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
1344 } else {
1345 return range.to();
1346 }
1347 }
1348
1349
1350 final PathPatternFilter filter = PathPatternFilter.of(pathPattern);
1351
1352 final List<DiffEntry> diffEntries;
1353 readLock();
1354 try (RevWalk revWalk = newRevWalk()) {
1355 final RevTree treeA = toTree(revWalk, range.from());
1356 final RevTree treeB = toTree(revWalk, range.to());
1357 diffEntries = blockingCompareTrees(treeA, treeB);
1358 } finally {
1359 readUnlock();
1360 }
1361
1362
1363 for (DiffEntry e : diffEntries) {
1364 final String path;
1365 switch (e.getChangeType()) {
1366 case ADD:
1367 path = e.getNewPath();
1368 break;
1369 case MODIFY:
1370 case DELETE:
1371 path = e.getOldPath();
1372 break;
1373 default:
1374 throw new Error();
1375 }
1376
1377 if (filter.matches(path)) {
1378 return range.to();
1379 }
1380 }
1381
1382 if (!errorOnEntryNotFound) {
1383 return null;
1384 }
1385 if (!blockingFind(range.to(), pathPattern, FindOptions.FIND_ONE_WITHOUT_CONTENT).isEmpty()) {
1386
1387
1388 return null;
1389 }
1390 throw new EntryNotFoundException(lastKnownRevision, pathPattern);
1391 }
1392
1393
1394
1395
1396 private List<DiffEntry> blockingCompareTrees(RevTree treeA, RevTree treeB) {
1397 if (cache == null) {
1398 return blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL);
1399 }
1400
1401 final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB);
1402 CompletableFuture<List<DiffEntry>> existingFuture = cache.getIfPresent(key);
1403 if (existingFuture != null) {
1404 final List<DiffEntry> existingDiffEntries = existingFuture.getNow(null);
1405 if (existingDiffEntries != null) {
1406
1407 return existingDiffEntries;
1408 }
1409 }
1410
1411
1412 final List<DiffEntry> newDiffEntries;
1413 final Lock lock = key.coarseGrainedLock();
1414 lock.lock();
1415 try {
1416 existingFuture = cache.getIfPresent(key);
1417 if (existingFuture != null) {
1418 final List<DiffEntry> existingDiffEntries = existingFuture.getNow(null);
1419 if (existingDiffEntries != null) {
1420
1421 return existingDiffEntries;
1422 }
1423 }
1424
1425 newDiffEntries = blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL);
1426 cache.put(key, newDiffEntries);
1427 } finally {
1428 lock.unlock();
1429 }
1430
1431 logger.debug("Cache miss: {}", key);
1432 return newDiffEntries;
1433 }
1434
1435 private List<DiffEntry> blockingCompareTreesUncached(@Nullable RevTree treeA,
1436 @Nullable RevTree treeB,
1437 TreeFilter filter) {
1438 readLock();
1439 try (DiffFormatter diffFormatter = new DiffFormatter(null)) {
1440 diffFormatter.setRepository(jGitRepository);
1441 diffFormatter.setPathFilter(filter);
1442 return ImmutableList.copyOf(diffFormatter.scan(treeA, treeB));
1443 } catch (IOException e) {
1444 throw new StorageException("failed to compare two trees: " + treeA + " vs. " + treeB, e);
1445 } finally {
1446 readUnlock();
1447 }
1448 }
1449
1450 @Override
1451 public CompletableFuture<Revision> watch(Revision lastKnownRevision, String pathPattern,
1452 boolean errorOnEntryNotFound) {
1453 requireNonNull(lastKnownRevision, "lastKnownRevision");
1454 requireNonNull(pathPattern, "pathPattern");
1455
1456 final ServiceRequestContext ctx = context();
1457 final Revision normLastKnownRevision = normalizeNow(lastKnownRevision);
1458 final CompletableFuture<Revision> future = new CompletableFuture<>();
1459 CompletableFuture.runAsync(() -> {
1460 failFastIfTimedOut(this, logger, ctx, "watch", lastKnownRevision, pathPattern);
1461 readLock();
1462 try {
1463
1464
1465 final Revision latestRevision = blockingFindLatestRevision(normLastKnownRevision, pathPattern,
1466 errorOnEntryNotFound);
1467 if (latestRevision != null) {
1468 future.complete(latestRevision);
1469 } else {
1470 commitWatchers.add(normLastKnownRevision, pathPattern, future);
1471 }
1472 } finally {
1473 readUnlock();
1474 }
1475 }, repositoryWorker).exceptionally(cause -> {
1476 future.completeExceptionally(cause);
1477 return null;
1478 });
1479
1480 return future;
1481 }
1482
1483 private void notifyWatchers(Revision newRevision, List<DiffEntry> diffEntries) {
1484 for (DiffEntry entry : diffEntries) {
1485 switch (entry.getChangeType()) {
1486 case ADD:
1487 commitWatchers.notify(newRevision, entry.getNewPath());
1488 break;
1489 case MODIFY:
1490 case DELETE:
1491 commitWatchers.notify(newRevision, entry.getOldPath());
1492 break;
1493 default:
1494 throw new Error();
1495 }
1496 }
1497 }
1498
1499 private Revision cachedHeadRevision() {
1500 return headRevision;
1501 }
1502
1503
1504
1505
1506 private Revision uncachedHeadRevision() {
1507 try (RevWalk revWalk = newRevWalk()) {
1508 final ObjectId headRevisionId = jGitRepository.resolve(R_HEADS_MASTER);
1509 if (headRevisionId != null) {
1510 final RevCommit revCommit = revWalk.parseCommit(headRevisionId);
1511 return CommitUtil.extractRevision(revCommit.getFullMessage());
1512 }
1513 } catch (CentralDogmaException e) {
1514 throw e;
1515 } catch (Exception e) {
1516 throw new StorageException("failed to get the current revision", e);
1517 }
1518
1519 throw new StorageException("failed to determine the HEAD: " + parent.name() + '/' + name);
1520 }
1521
1522 private RevTree toTree(RevWalk revWalk, Revision revision) {
1523 final ObjectId commitId = commitIdDatabase.get(revision);
1524 try {
1525 return revWalk.parseCommit(commitId).getTree();
1526 } catch (IOException e) {
1527 throw new StorageException("failed to parse a commit: " + commitId, e);
1528 }
1529 }
1530
1531 private RevWalk newRevWalk() {
1532 final RevWalk revWalk = new RevWalk(jGitRepository);
1533 configureRevWalk(revWalk);
1534 return revWalk;
1535 }
1536
1537 private static RevWalk newRevWalk(ObjectReader reader) {
1538 final RevWalk revWalk = new RevWalk(reader);
1539 configureRevWalk(revWalk);
1540 return revWalk;
1541 }
1542
1543 private static void configureRevWalk(RevWalk revWalk) {
1544
1545 revWalk.setRewriteParents(false);
1546 }
1547
1548 private void readLock() {
1549 rwLock.readLock().lock();
1550 if (closePending.get() != null) {
1551 rwLock.readLock().unlock();
1552 throw closePending.get().get();
1553 }
1554 }
1555
1556 private void readUnlock() {
1557 rwLock.readLock().unlock();
1558 }
1559
1560 private void writeLock() {
1561 rwLock.writeLock().lock();
1562 if (closePending.get() != null) {
1563 writeUnLock();
1564 throw closePending.get().get();
1565 }
1566 }
1567
1568 private void writeUnLock() {
1569 rwLock.writeLock().unlock();
1570 }
1571
1572
1573
1574
1575 public void cloneTo(File newRepoDir) {
1576 cloneTo(newRepoDir, (current, total) -> { });
1577 }
1578
1579
1580
1581
1582 public void cloneTo(File newRepoDir, BiConsumer<Integer, Integer> progressListener) {
1583 requireNonNull(newRepoDir, "newRepoDir");
1584 requireNonNull(progressListener, "progressListener");
1585
1586 final Revision endRevision = normalizeNow(Revision.HEAD);
1587 final GitRepository newRepo = new GitRepository(parent, newRepoDir, repositoryWorker,
1588 creationTimeMillis(), author(), cache);
1589
1590 progressListener.accept(1, endRevision.major());
1591 boolean success = false;
1592 try {
1593
1594 Revision previousNonEmptyRevision = null;
1595 for (int i = 2; i <= endRevision.major();) {
1596
1597 final int batch = 16;
1598 final List<Commit> commits = blockingHistory(
1599 new Revision(i), new Revision(Math.min(endRevision.major(), i + batch - 1)),
1600 Repository.ALL_PATH, batch);
1601 checkState(!commits.isEmpty(), "empty commits");
1602
1603 if (previousNonEmptyRevision == null) {
1604 previousNonEmptyRevision = commits.get(0).revision().backward(1);
1605 }
1606 for (Commit c : commits) {
1607 final Revision revision = c.revision();
1608 checkState(revision.major() == i,
1609 "mismatching revision: %s (expected: %s)", revision.major(), i);
1610
1611 final Revision baseRevision = revision.backward(1);
1612 final Collection<Change<?>> changes =
1613 diff(previousNonEmptyRevision, revision, Repository.ALL_PATH).join().values();
1614
1615 try {
1616 newRepo.blockingCommit(
1617 baseRevision, c.when(), c.author(), c.summary(), c.detail(), c.markup(),
1618 changes, false, false);
1619 previousNonEmptyRevision = revision;
1620 } catch (RedundantChangeException e) {
1621
1622
1623 newRepo.blockingCommit(
1624 baseRevision, c.when(), c.author(), c.summary(), c.detail(), c.markup(),
1625 changes, true, false);
1626 }
1627
1628 progressListener.accept(i, endRevision.major());
1629 i++;
1630 }
1631 }
1632
1633 success = true;
1634 } finally {
1635 newRepo.internalClose();
1636 if (!success) {
1637 deleteCruft(newRepoDir);
1638 }
1639 }
1640 }
1641
1642 private static void deleteCruft(File repoDir) {
1643 try {
1644 Util.deleteFileTree(repoDir);
1645 } catch (IOException e) {
1646 logger.error("Failed to delete a half-created repository at: {}", repoDir, e);
1647 }
1648 }
1649
1650 @Override
1651 public String toString() {
1652 return MoreObjects.toStringHelper(this)
1653 .add("dir", jGitRepository.getDirectory())
1654 .toString();
1655 }
1656
1657 private static final class RevisionAndEntries {
1658 final Revision revision;
1659 final List<DiffEntry> diffEntries;
1660
1661 RevisionAndEntries(Revision revision, List<DiffEntry> diffEntries) {
1662 this.revision = revision;
1663 this.diffEntries = diffEntries;
1664 }
1665 }
1666
1667
1668
1669 private static final class InsertText extends PathEdit {
1670 private final ObjectInserter inserter;
1671 private final String text;
1672
1673 InsertText(String entryPath, ObjectInserter inserter, String text) {
1674 super(entryPath);
1675 this.inserter = inserter;
1676 this.text = text;
1677 }
1678
1679 @Override
1680 public void apply(DirCacheEntry ent) {
1681 try {
1682 ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8)));
1683 ent.setFileMode(FileMode.REGULAR_FILE);
1684 } catch (IOException e) {
1685 throw new StorageException("failed to create a new text blob", e);
1686 }
1687 }
1688 }
1689
1690 private static final class InsertJson extends PathEdit {
1691 private final ObjectInserter inserter;
1692 private final JsonNode jsonNode;
1693
1694 InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) {
1695 super(entryPath);
1696 this.inserter = inserter;
1697 this.jsonNode = jsonNode;
1698 }
1699
1700 @Override
1701 public void apply(DirCacheEntry ent) {
1702 try {
1703 ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode)));
1704 ent.setFileMode(FileMode.REGULAR_FILE);
1705 } catch (IOException e) {
1706 throw new StorageException("failed to create a new JSON blob", e);
1707 }
1708 }
1709 }
1710
1711 private static final class CopyOldEntry extends PathEdit {
1712 private final DirCacheEntry oldEntry;
1713
1714 CopyOldEntry(String entryPath, DirCacheEntry oldEntry) {
1715 super(entryPath);
1716 this.oldEntry = oldEntry;
1717 }
1718
1719 @Override
1720 public void apply(DirCacheEntry ent) {
1721 ent.setFileMode(oldEntry.getFileMode());
1722 ent.setObjectId(oldEntry.getObjectId());
1723 }
1724 }
1725 }