1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.mirror;
18
19 import static com.linecorp.centraldogma.server.storage.repository.FindOptions.FIND_ALL_WITHOUT_CONTENT;
20 import static java.nio.charset.StandardCharsets.UTF_8;
21 import static org.eclipse.jgit.lib.Constants.OBJECT_ID_ABBREV_STRING_LENGTH;
22
23 import java.io.ByteArrayInputStream;
24 import java.io.File;
25 import java.io.IOException;
26 import java.net.URI;
27 import java.time.Instant;
28 import java.util.Collection;
29 import java.util.HashMap;
30 import java.util.LinkedHashMap;
31 import java.util.Map;
32 import java.util.Objects;
33 import java.util.Optional;
34 import java.util.concurrent.CompletionException;
35 import java.util.function.Consumer;
36 import java.util.regex.Pattern;
37 import java.util.stream.Collectors;
38 import java.util.stream.Stream;
39
40 import javax.annotation.Nullable;
41
42 import org.eclipse.jgit.api.RemoteSetUrlCommand;
43 import org.eclipse.jgit.api.RemoteSetUrlCommand.UriType;
44 import org.eclipse.jgit.api.TransportCommand;
45 import org.eclipse.jgit.api.errors.GitAPIException;
46 import org.eclipse.jgit.dircache.DirCache;
47 import org.eclipse.jgit.dircache.DirCacheBuilder;
48 import org.eclipse.jgit.dircache.DirCacheEditor;
49 import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath;
50 import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit;
51 import org.eclipse.jgit.dircache.DirCacheEntry;
52 import org.eclipse.jgit.ignore.IgnoreNode;
53 import org.eclipse.jgit.ignore.IgnoreNode.MatchResult;
54 import org.eclipse.jgit.lib.CommitBuilder;
55 import org.eclipse.jgit.lib.Constants;
56 import org.eclipse.jgit.lib.FileMode;
57 import org.eclipse.jgit.lib.ObjectId;
58 import org.eclipse.jgit.lib.ObjectInserter;
59 import org.eclipse.jgit.lib.ObjectReader;
60 import org.eclipse.jgit.lib.PersonIdent;
61 import org.eclipse.jgit.lib.Ref;
62 import org.eclipse.jgit.lib.RefUpdate;
63 import org.eclipse.jgit.lib.RefUpdate.Result;
64 import org.eclipse.jgit.revwalk.RevCommit;
65 import org.eclipse.jgit.revwalk.RevWalk;
66 import org.eclipse.jgit.transport.FetchResult;
67 import org.eclipse.jgit.transport.RefSpec;
68 import org.eclipse.jgit.transport.TagOpt;
69 import org.eclipse.jgit.transport.URIish;
70 import org.eclipse.jgit.treewalk.TreeWalk;
71 import org.slf4j.Logger;
72 import org.slf4j.LoggerFactory;
73
74 import com.cronutils.model.Cron;
75 import com.fasterxml.jackson.core.JsonParseException;
76 import com.fasterxml.jackson.core.JsonProcessingException;
77 import com.fasterxml.jackson.core.TreeNode;
78 import com.fasterxml.jackson.databind.JsonMappingException;
79 import com.fasterxml.jackson.databind.JsonNode;
80
81 import com.linecorp.centraldogma.common.Change;
82 import com.linecorp.centraldogma.common.Entry;
83 import com.linecorp.centraldogma.common.EntryType;
84 import com.linecorp.centraldogma.common.Markup;
85 import com.linecorp.centraldogma.common.MirrorException;
86 import com.linecorp.centraldogma.common.RedundantChangeException;
87 import com.linecorp.centraldogma.common.Revision;
88 import com.linecorp.centraldogma.internal.Jackson;
89 import com.linecorp.centraldogma.internal.Util;
90 import com.linecorp.centraldogma.server.command.Command;
91 import com.linecorp.centraldogma.server.command.CommandExecutor;
92 import com.linecorp.centraldogma.server.command.CommitResult;
93 import com.linecorp.centraldogma.server.credential.Credential;
94 import com.linecorp.centraldogma.server.mirror.MirrorDirection;
95 import com.linecorp.centraldogma.server.mirror.MirrorResult;
96 import com.linecorp.centraldogma.server.mirror.MirrorStatus;
97 import com.linecorp.centraldogma.server.mirror.git.GitMirrorException;
98 import com.linecorp.centraldogma.server.storage.StorageException;
99 import com.linecorp.centraldogma.server.storage.repository.Repository;
100
101 abstract class AbstractGitMirror extends AbstractMirror {
102
103 private static final Logger logger = LoggerFactory.getLogger(AbstractGitMirror.class);
104
105
106 private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json";
107
108
109 private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = '.' + MIRROR_STATE_FILE_NAME;
110
111 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
112
113 private static final byte[] EMPTY_BYTE = new byte[0];
114
115 private static final Pattern DISALLOWED_CHARS = Pattern.compile("[^-_a-zA-Z]");
116 private static final Pattern CONSECUTIVE_UNDERSCORES = Pattern.compile("_+");
117
118 private static final int GIT_TIMEOUT_SECS = 60;
119
120 private static final String HEAD_REF_MASTER = Constants.R_HEADS + Constants.MASTER;
121
122 @Nullable
123 private IgnoreNode ignoreNode;
124
125 AbstractGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction,
126 Credential credential, Repository localRepo, String localPath,
127 URI remoteRepoUri, String remotePath, String remoteBranch,
128 @Nullable String gitignore, @Nullable String zone) {
129 super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath,
130 remoteBranch, gitignore, zone);
131
132 if (gitignore != null) {
133 ignoreNode = new IgnoreNode();
134 try {
135 ignoreNode.parse(new ByteArrayInputStream(gitignore.getBytes()));
136 } catch (IOException e) {
137 throw new IllegalArgumentException("Failed to read gitignore: " + gitignore, e);
138 }
139 }
140 }
141
142 GitWithAuth openGit(File workDir,
143 URIish remoteUri,
144 Consumer<TransportCommand<?, ?>> configurator) throws IOException, GitAPIException {
145
146 final File repoDir = new File(
147 workDir,
148 CONSECUTIVE_UNDERSCORES.matcher(DISALLOWED_CHARS.matcher(
149 remoteRepoUri().toASCIIString()).replaceAll("_")).replaceAll("_"));
150 final GitWithAuth git = new GitWithAuth(this, repoDir, remoteUri, configurator);
151 boolean success = false;
152 try {
153
154 final RemoteSetUrlCommand remoteSetUrl = git.remoteSetUrl();
155 remoteSetUrl.setRemoteName(Constants.DEFAULT_REMOTE_NAME);
156 remoteSetUrl.setRemoteUri(remoteUri);
157
158 remoteSetUrl.setUriType(UriType.FETCH);
159 remoteSetUrl.call();
160
161 remoteSetUrl.setUriType(UriType.PUSH);
162 remoteSetUrl.call();
163
164
165
166
167
168
169
170 success = true;
171 return git;
172 } finally {
173 if (!success) {
174 git.close();
175 }
176 }
177 }
178
179 MirrorResult mirrorLocalToRemote(
180 GitWithAuth git, int maxNumFiles, long maxNumBytes, Instant triggeredTime)
181 throws GitAPIException, IOException {
182
183 final Ref headBranchRef = getHeadBranchRef(git);
184 final String headBranchRefName = headBranchRef.getName();
185 final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRefName);
186
187 final org.eclipse.jgit.lib.Repository gitRepository = git.getRepository();
188 final String description;
189 try (ObjectReader reader = gitRepository.newObjectReader();
190 TreeWalk treeWalk = new TreeWalk(reader);
191 RevWalk revWalk = new RevWalk(reader)) {
192
193
194 final ObjectId headTreeId = revWalk.parseTree(headCommitId).getId();
195 treeWalk.reset(headTreeId);
196
197 final String mirrorStatePath = remotePath() + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME;
198 final Revision localHead = localRepo().normalizeNow(Revision.HEAD);
199 final Revision remoteCurrentRevision = remoteCurrentRevision(reader, treeWalk, mirrorStatePath);
200 if (localHead.equals(remoteCurrentRevision)) {
201
202 description = String.format(
203 "The remote repository '%s#%s' already at %s. Local repository: '%s/%s'",
204 remoteRepoUri(), remoteBranch(), localHead,
205 localRepo().parent().name(), localRepo().name());
206 logger.debug(description);
207 return newMirrorResult(MirrorStatus.UP_TO_DATE, description, triggeredTime);
208 }
209
210
211 treeWalk.reset(headTreeId);
212
213
214
215
216 final DirCache dirCache = DirCache.newInCore();
217 final DirCacheBuilder builder = dirCache.builder();
218 builder.addTree(EMPTY_BYTE, 0, reader, headTreeId);
219 builder.finish();
220
221 try (ObjectInserter inserter = gitRepository.newObjectInserter()) {
222 addModifiedEntryToCache(localHead, dirCache, reader, inserter,
223 treeWalk, maxNumFiles, maxNumBytes);
224
225 final MirrorState mirrorState = new MirrorState(localHead.text());
226 applyPathEdit(
227 dirCache, new InsertText(mirrorStatePath.substring(1),
228 inserter,
229 Jackson.writeValueAsPrettyString(mirrorState) + '\n'));
230 }
231
232 final String summary = "Mirror '" + localRepo().name() + "' at " + localHead +
233 " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n";
234 description = summary;
235 final ObjectId nextCommitId =
236 commit(gitRepository, dirCache, headCommitId, summary);
237 logger.info(summary);
238 updateRef(gitRepository, revWalk, headBranchRefName, nextCommitId);
239 }
240
241 git.push()
242 .setRefSpecs(new RefSpec(headBranchRefName))
243 .setAtomic(true)
244 .setTimeout(GIT_TIMEOUT_SECS)
245 .call();
246 return newMirrorResult(MirrorStatus.SUCCESS, description, triggeredTime);
247 }
248
249 MirrorResult mirrorRemoteToLocal(
250 GitWithAuth git, CommandExecutor executor, int maxNumFiles, long maxNumBytes, Instant triggeredTime)
251 throws Exception {
252 final String summary;
253 final String detail;
254 final Ref headBranchRef;
255 final ObjectId headCommitId;
256 final Map<String, Change<?>> changes = new HashMap<>();
257 final Revision localRev = localRepo().normalizeNow(Revision.HEAD);
258 final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME;
259 try {
260 headBranchRef = getHeadBranchRef(git);
261 if (!needsFetch(headBranchRef, mirrorStatePath, localRev)) {
262 return newMirrorResultForUpToDate(headBranchRef, triggeredTime);
263 }
264
265
266
267 headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRef.getName());
268 } catch (Exception e) {
269 String message = "Failed to fetch the remote repository '" + git.remoteUri() +
270 "' to the local repository '" + localPath() + "'.";
271 if (e.getMessage() != null) {
272 message += " (reason: " + e.getMessage();
273 }
274 throw new GitMirrorException(message, e);
275 }
276
277 try (ObjectReader reader = git.getRepository().newObjectReader();
278 TreeWalk treeWalk = new TreeWalk(reader);
279 RevWalk revWalk = new RevWalk(reader)) {
280
281
282 treeWalk.addTree(revWalk.parseTree(headCommitId).getId());
283 final String abbrId = reader.abbreviate(headCommitId).name();
284
285
286 changes.put(mirrorStatePath, Change.ofJsonUpsert(
287 mirrorStatePath, "{ \"sourceRevision\": \"" + headCommitId.name() + "\" }"));
288
289 final String branchName = getRemoteBranchName(headBranchRef);
290 summary = "Mirror " + abbrId + ", '" + remoteRepoUri() + '#' + branchName +
291 "' to the repository '" + localRepo().name() + '\'';
292 final RevCommit headCommit = revWalk.parseCommit(headCommitId);
293 detail = generateCommitDetail(headCommit);
294 logger.info(summary);
295 long numFiles = 0;
296 long numBytes = 0;
297 while (treeWalk.next()) {
298 final FileMode fileMode = treeWalk.getFileMode();
299 final String path = '/' + treeWalk.getPathString();
300
301 if (ignoreNode != null && path.startsWith(remotePath())) {
302 assert ignoreNode != null;
303 if (ignoreNode.isIgnored('/' + path.substring(remotePath().length()),
304 fileMode == FileMode.TREE) == MatchResult.IGNORED) {
305 continue;
306 }
307 }
308
309 if (fileMode == FileMode.TREE) {
310 maybeEnterSubtree(treeWalk, remotePath(), path);
311 continue;
312 }
313
314 if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) {
315
316 continue;
317 }
318
319
320 if (!path.startsWith(remotePath())) {
321 continue;
322 }
323
324 final String localPath = localPath() + path.substring(remotePath().length());
325
326
327 if (!Util.isValidFilePath(localPath)) {
328 continue;
329 }
330
331 if (++numFiles > maxNumFiles) {
332 throwMirrorException(maxNumFiles, "files");
333 }
334
335 final ObjectId objectId = treeWalk.getObjectId(0);
336 final long contentLength = reader.getObjectSize(objectId, ObjectReader.OBJ_ANY);
337 if (numBytes > maxNumBytes - contentLength) {
338 throwMirrorException(maxNumBytes, "bytes");
339 }
340 numBytes += contentLength;
341
342 final byte[] content = reader.open(objectId).getBytes();
343 switch (EntryType.guessFromPath(localPath)) {
344 case JSON:
345 final JsonNode jsonNode = Jackson.readTree(content);
346 changes.putIfAbsent(localPath, Change.ofJsonUpsert(localPath, jsonNode));
347 break;
348 case TEXT:
349 final String strVal = new String(content, UTF_8);
350 changes.putIfAbsent(localPath, Change.ofTextUpsert(localPath, strVal));
351 break;
352 }
353 }
354 }
355
356 final Map<String, Entry<?>> oldEntries = localRepo().find(
357 localRev, localPath() + "**", FIND_ALL_WITHOUT_CONTENT).join();
358 oldEntries.keySet().removeAll(changes.keySet());
359
360
361 oldEntries.forEach((path, entry) -> {
362 if (entry.type() != EntryType.DIRECTORY && !changes.containsKey(path)) {
363 changes.put(path, Change.ofRemoval(path));
364 }
365 });
366
367 try {
368 final CommitResult commitResult = executor.execute(Command.push(
369 MIRROR_AUTHOR, localRepo().parent().name(), localRepo().name(),
370 Revision.HEAD, summary, detail, Markup.PLAINTEXT, changes.values())).join();
371 final String description = summary + ", revision: " + commitResult.revision().text();
372 return newMirrorResult(MirrorStatus.SUCCESS, description, triggeredTime);
373 } catch (CompletionException e) {
374 if (e.getCause() instanceof RedundantChangeException) {
375 return newMirrorResultForUpToDate(headBranchRef, triggeredTime);
376 }
377 throw e;
378 }
379 }
380
381 private MirrorResult newMirrorResultForUpToDate(Ref headBranchRef, Instant triggeredTime) {
382 final String abbrId = headBranchRef.getObjectId().abbreviate(OBJECT_ID_ABBREV_STRING_LENGTH).name();
383 final String message = String.format("Repository '%s/%s' already at %s, %s#%s",
384 localRepo().parent().name(), localRepo().name(), abbrId,
385 remoteRepoUri(), remoteBranch());
386
387 logger.debug(message);
388 return newMirrorResult(MirrorStatus.UP_TO_DATE, message, triggeredTime);
389 }
390
391 private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision localRev)
392 throws JsonParseException, JsonMappingException {
393 final Entry<?> mirrorState = localRepo().getOrNull(localRev, mirrorStatePath).join();
394 final String localSourceRevision;
395 if (mirrorState == null || mirrorState.type() != EntryType.JSON) {
396 localSourceRevision = null;
397 } else {
398 localSourceRevision = Jackson.treeToValue((TreeNode) mirrorState.content(),
399 MirrorState.class).sourceRevision();
400 }
401
402 final ObjectId headCommitId = headBranchRef.getObjectId();
403 return !headCommitId.name().equals(localSourceRevision);
404 }
405
406 private Ref getHeadBranchRef(GitWithAuth git) throws GitAPIException {
407 if (!remoteBranch().isEmpty()) {
408 final String headBranchRefName = Constants.R_HEADS + remoteBranch();
409 final Collection<Ref> refs = lsRemote(git, true);
410 return findHeadBranchRef(git, headBranchRefName, refs);
411 }
412
413
414
415 final Collection<Ref> refs = lsRemote(git, false);
416
417
418 final Optional<String> headRefNameOptional = refs.stream()
419 .filter(ref -> Constants.HEAD.equals(ref.getName()))
420 .map(ref -> ref.getTarget().getName())
421 .findFirst();
422 final String headBranchRefName;
423 if (headRefNameOptional.isPresent()) {
424 headBranchRefName = headRefNameOptional.get();
425 } else {
426
427 headBranchRefName = HEAD_REF_MASTER;
428 }
429 return findHeadBranchRef(git, headBranchRefName, refs);
430 }
431
432 private static Collection<Ref> lsRemote(GitWithAuth git,
433 boolean setHeads) throws GitAPIException {
434 return git.lsRemote()
435 .setTags(false)
436 .setTimeout(GIT_TIMEOUT_SECS)
437 .setHeads(setHeads)
438 .call();
439 }
440
441 private static Ref findHeadBranchRef(GitWithAuth git, String headBranchRefName, Collection<Ref> refs) {
442 final Optional<Ref> headBranchRef = refs.stream()
443 .filter(ref -> headBranchRefName.equals(ref.getName()))
444 .findFirst();
445 if (headBranchRef.isPresent()) {
446 return headBranchRef.get();
447 }
448 throw new GitMirrorException("Remote does not have " + headBranchRefName + " branch. remote: " +
449 git.remoteUri());
450 }
451
452 private static String generateCommitDetail(RevCommit headCommit) {
453 final PersonIdent authorIdent = headCommit.getAuthorIdent();
454 return "Remote commit:\n" +
455 "- SHA: " + headCommit.name() + '\n' +
456 "- Subject: " + headCommit.getShortMessage() + '\n' +
457 "- Author: " + authorIdent.getName() + " <" +
458 authorIdent.getEmailAddress() + "> \n" +
459 "- Date: " + authorIdent.getWhen() + "\n\n" +
460 headCommit.getFullMessage();
461 }
462
463 @Nullable
464 private Revision remoteCurrentRevision(
465 ObjectReader reader, TreeWalk treeWalk, String mirrorStatePath) {
466 try {
467 while (treeWalk.next()) {
468 final FileMode fileMode = treeWalk.getFileMode();
469 final String path = '/' + treeWalk.getPathString();
470
471
472 if (fileMode == FileMode.TREE) {
473 if (remotePath().startsWith(path + '/')) {
474 treeWalk.enterSubtree();
475 }
476 continue;
477 }
478
479 if (!path.equals(mirrorStatePath)) {
480 continue;
481 }
482
483 final byte[] content = currentEntryContent(reader, treeWalk);
484 final MirrorState mirrorState = Jackson.readValue(content, MirrorState.class);
485 return new Revision(mirrorState.sourceRevision());
486 }
487
488 return null;
489 } catch (Exception e) {
490 logger.warn("Unexpected exception while retrieving the remote source revision", e);
491 return null;
492 }
493 }
494
495 private static ObjectId fetchRemoteHeadAndGetCommitId(
496 GitWithAuth git, String headBranchRefName) throws GitAPIException, IOException {
497 final FetchResult fetchResult = git.fetch()
498 .setDepth(1)
499 .setRefSpecs(new RefSpec(headBranchRefName))
500 .setRemoveDeletedRefs(true)
501 .setTagOpt(TagOpt.NO_TAGS)
502 .setTimeout(GIT_TIMEOUT_SECS)
503 .call();
504 final ObjectId commitId = fetchResult.getAdvertisedRef(headBranchRefName).getObjectId();
505 final RefUpdate refUpdate = git.getRepository().updateRef(headBranchRefName);
506 refUpdate.setNewObjectId(commitId);
507 refUpdate.setForceUpdate(true);
508 refUpdate.update();
509 return commitId;
510 }
511
512 private Map<String, Entry<?>> localHeadEntries(Revision localHead) {
513 final Map<String, Entry<?>> localRawHeadEntries = localRepo().find(localHead, localPath() + "**")
514 .join();
515
516 final Stream<Map.Entry<String, Entry<?>>> entryStream =
517 localRawHeadEntries.entrySet()
518 .stream();
519 if (ignoreNode == null) {
520
521 return entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
522 }
523
524 final Map<String, Entry<?>> sortedMap =
525 entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
526 (v1, v2) -> v1, LinkedHashMap::new));
527
528 final HashMap<String, Entry<?>> result = new HashMap<>(sortedMap.size());
529 String lastIgnoredDirectory = null;
530 for (Map.Entry<String, ? extends Entry<?>> entry : sortedMap.entrySet()) {
531 final String path = entry.getKey();
532 final boolean isDirectory = entry.getValue().type() == EntryType.DIRECTORY;
533 assert ignoreNode != null;
534 final MatchResult ignoreResult = ignoreNode.isIgnored(
535 path.substring(localPath().length()), isDirectory);
536 if (ignoreResult == MatchResult.IGNORED) {
537 if (isDirectory) {
538 lastIgnoredDirectory = path;
539 }
540 continue;
541 }
542 if (ignoreResult == MatchResult.CHECK_PARENT) {
543 if (lastIgnoredDirectory != null && path.startsWith(lastIgnoredDirectory)) {
544 continue;
545 }
546 }
547 result.put(path, entry.getValue());
548 }
549
550 return result;
551 }
552
553 private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, ObjectReader reader,
554 ObjectInserter inserter, TreeWalk treeWalk,
555 int maxNumFiles, long maxNumBytes) throws IOException {
556 final Map<String, Entry<?>> localHeadEntries = localHeadEntries(localHead);
557 long numFiles = 0;
558 long numBytes = 0;
559 while (treeWalk.next()) {
560 final FileMode fileMode = treeWalk.getFileMode();
561 final String pathString = treeWalk.getPathString();
562 final String remoteFilePath = '/' + pathString;
563
564
565 if (fileMode == FileMode.TREE) {
566 maybeEnterSubtree(treeWalk, remotePath(), remoteFilePath);
567 continue;
568 }
569
570 if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) {
571
572 continue;
573 }
574
575
576 if (!remoteFilePath.startsWith(remotePath())) {
577 continue;
578 }
579
580 final String localFilePath = localPath() + remoteFilePath.substring(remotePath().length());
581
582
583 if (!Util.isValidFilePath(localFilePath)) {
584 continue;
585 }
586
587 final Entry<?> entry = localHeadEntries.remove(localFilePath);
588 if (entry == null) {
589
590 applyPathEdit(dirCache, new DeletePath(pathString));
591 continue;
592 }
593
594 if (++numFiles > maxNumFiles) {
595 throwMirrorException(maxNumFiles, "files");
596 return;
597 }
598
599 final byte[] oldContent = currentEntryContent(reader, treeWalk);
600 final long contentLength = applyPathEdit(dirCache, inserter, pathString, entry, oldContent);
601 numBytes += contentLength;
602 if (numBytes > maxNumBytes) {
603 throwMirrorException(maxNumBytes, "bytes");
604 return;
605 }
606 }
607
608
609 for (Map.Entry<String, Entry<?>> entry : localHeadEntries.entrySet()) {
610 final Entry<?> value = entry.getValue();
611 if (value.type() == EntryType.DIRECTORY) {
612 continue;
613 }
614 if (entry.getKey().endsWith(MIRROR_STATE_FILE_NAME)) {
615 continue;
616 }
617
618 if (++numFiles > maxNumFiles) {
619 throwMirrorException(maxNumFiles, "files");
620 return;
621 }
622
623 final String convertedPath = remotePath().substring(1) +
624 entry.getKey().substring(localPath().length());
625 final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, value, null);
626 numBytes += contentLength;
627 if (numBytes > maxNumBytes) {
628 throwMirrorException(maxNumBytes, "bytes");
629 }
630 }
631 }
632
633 private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, String pathString,
634 Entry<?> entry, @Nullable byte[] oldContent)
635 throws JsonProcessingException {
636 switch (EntryType.guessFromPath(pathString)) {
637 case JSON:
638 final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null;
639 final JsonNode newJsonNode = (JsonNode) entry.content();
640
641
642 if (!Objects.equals(newJsonNode, oldJsonNode)) {
643
644 final String newContent = newJsonNode.toPrettyString() + '\n';
645 applyPathEdit(dirCache, new InsertText(pathString, inserter, newContent));
646 return newContent.length();
647 }
648 break;
649 case TEXT:
650 final String sanitizedOldText = oldContent != null ?
651 sanitizeText(new String(oldContent, UTF_8)) : null;
652 final String sanitizedNewText = entry.contentAsText();
653
654 if (!sanitizedNewText.equals(sanitizedOldText)) {
655 applyPathEdit(dirCache, new InsertText(pathString, inserter, sanitizedNewText));
656 return sanitizedNewText.length();
657 }
658 break;
659 }
660 return 0;
661 }
662
663 private static void applyPathEdit(DirCache dirCache, PathEdit edit) {
664 final DirCacheEditor e = dirCache.editor();
665 e.add(edit);
666 e.finish();
667 }
668
669 private static byte[] currentEntryContent(ObjectReader reader, TreeWalk treeWalk) throws IOException {
670 final ObjectId objectId = treeWalk.getObjectId(0);
671 return reader.open(objectId).getBytes();
672 }
673
674 private static void maybeEnterSubtree(
675 TreeWalk treeWalk, String remotePath, String path) throws IOException {
676
677
678
679
680 if (path.startsWith(remotePath)) {
681 treeWalk.enterSubtree();
682 return;
683 }
684
685
686
687
688
689 final int pathLen = path.length() + 1;
690 if (pathLen == remotePath.length() && remotePath.startsWith(path)) {
691 treeWalk.enterSubtree();
692 return;
693 }
694
695
696
697
698
699 if (pathLen < remotePath.length() && remotePath.startsWith(path + '/')) {
700 treeWalk.enterSubtree();
701 }
702 }
703
704
705
706
707 private static String sanitizeText(String text) {
708 if (text.indexOf('\r') >= 0) {
709 text = CR.matcher(text).replaceAll("");
710 }
711 if (!text.isEmpty() && !text.endsWith("\n")) {
712 text += "\n";
713 }
714 return text;
715 }
716
717 private static ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache,
718 ObjectId headCommitId, String message) throws IOException {
719 try (ObjectInserter inserter = gitRepository.newObjectInserter()) {
720
721 final ObjectId nextTreeId = dirCache.writeTree(inserter);
722
723 final PersonIdent personIdent =
724 new PersonIdent(MIRROR_AUTHOR.name(), MIRROR_AUTHOR.email(),
725 System.currentTimeMillis() / 1000L * 1000L,
726 0);
727
728 final CommitBuilder commitBuilder = new CommitBuilder();
729 commitBuilder.setAuthor(personIdent);
730 commitBuilder.setCommitter(personIdent);
731 commitBuilder.setTreeId(nextTreeId);
732 commitBuilder.setEncoding(UTF_8);
733 commitBuilder.setParentId(headCommitId);
734 commitBuilder.setMessage(message);
735
736 final ObjectId nextCommitId = inserter.insert(commitBuilder);
737 inserter.flush();
738 return nextCommitId;
739 }
740 }
741
742 private <T> T throwMirrorException(long number, String filesOrBytes) {
743 throw new MirrorException("mirror (" + remoteRepoUri() + '#' + remoteBranch() +
744 ") contains more than " + number + ' ' + filesOrBytes);
745 }
746
747 static void updateRef(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
748 String ref, ObjectId commitId) throws IOException {
749 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
750 refUpdate.setNewObjectId(commitId);
751
752 final Result res = refUpdate.update(revWalk);
753 switch (res) {
754 case NEW:
755 case FAST_FORWARD:
756
757 break;
758 default:
759 throw new StorageException("unexpected refUpdate state: " + res);
760 }
761 }
762
763 private String getRemoteBranchName(Ref headBranchRef) {
764 final String remoteBranch = remoteBranch();
765 if (remoteBranch != null && !remoteBranch.isEmpty()) {
766 return remoteBranch;
767 }
768 final String headBranchName = headBranchRef.getName();
769 if (headBranchName.startsWith(Constants.R_HEADS)) {
770 return headBranchName.substring(Constants.R_HEADS.length());
771 }
772 return headBranchName;
773 }
774
775 private static final class InsertText extends PathEdit {
776 private final ObjectInserter inserter;
777 private final String text;
778
779 InsertText(String entryPath, ObjectInserter inserter, String text) {
780 super(entryPath);
781 this.inserter = inserter;
782 this.text = text;
783 }
784
785 @Override
786 public void apply(DirCacheEntry ent) {
787 try {
788 ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8)));
789 ent.setFileMode(FileMode.REGULAR_FILE);
790 } catch (IOException e) {
791 throw new StorageException("failed to create a new text blob", e);
792 }
793 }
794 }
795 }