1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.mirror;
18
19 import static com.linecorp.centraldogma.server.storage.repository.FindOptions.FIND_ALL_WITHOUT_CONTENT;
20 import static java.nio.charset.StandardCharsets.UTF_8;
21 import static org.eclipse.jgit.lib.Constants.OBJECT_ID_ABBREV_STRING_LENGTH;
22
23 import java.io.ByteArrayInputStream;
24 import java.io.File;
25 import java.io.IOException;
26 import java.net.URI;
27 import java.util.Collection;
28 import java.util.HashMap;
29 import java.util.LinkedHashMap;
30 import java.util.Map;
31 import java.util.Objects;
32 import java.util.Optional;
33 import java.util.function.Consumer;
34 import java.util.regex.Pattern;
35 import java.util.stream.Collectors;
36 import java.util.stream.Stream;
37
38 import javax.annotation.Nullable;
39
40 import org.eclipse.jgit.api.RemoteSetUrlCommand;
41 import org.eclipse.jgit.api.RemoteSetUrlCommand.UriType;
42 import org.eclipse.jgit.api.TransportCommand;
43 import org.eclipse.jgit.api.errors.GitAPIException;
44 import org.eclipse.jgit.dircache.DirCache;
45 import org.eclipse.jgit.dircache.DirCacheBuilder;
46 import org.eclipse.jgit.dircache.DirCacheEditor;
47 import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath;
48 import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit;
49 import org.eclipse.jgit.dircache.DirCacheEntry;
50 import org.eclipse.jgit.ignore.IgnoreNode;
51 import org.eclipse.jgit.ignore.IgnoreNode.MatchResult;
52 import org.eclipse.jgit.lib.CommitBuilder;
53 import org.eclipse.jgit.lib.Constants;
54 import org.eclipse.jgit.lib.FileMode;
55 import org.eclipse.jgit.lib.ObjectId;
56 import org.eclipse.jgit.lib.ObjectInserter;
57 import org.eclipse.jgit.lib.ObjectReader;
58 import org.eclipse.jgit.lib.PersonIdent;
59 import org.eclipse.jgit.lib.Ref;
60 import org.eclipse.jgit.lib.RefUpdate;
61 import org.eclipse.jgit.lib.RefUpdate.Result;
62 import org.eclipse.jgit.revwalk.RevCommit;
63 import org.eclipse.jgit.revwalk.RevWalk;
64 import org.eclipse.jgit.transport.FetchResult;
65 import org.eclipse.jgit.transport.RefSpec;
66 import org.eclipse.jgit.transport.TagOpt;
67 import org.eclipse.jgit.transport.URIish;
68 import org.eclipse.jgit.treewalk.TreeWalk;
69 import org.slf4j.Logger;
70 import org.slf4j.LoggerFactory;
71
72 import com.cronutils.model.Cron;
73 import com.fasterxml.jackson.core.JsonParseException;
74 import com.fasterxml.jackson.core.JsonProcessingException;
75 import com.fasterxml.jackson.core.TreeNode;
76 import com.fasterxml.jackson.databind.JsonMappingException;
77 import com.fasterxml.jackson.databind.JsonNode;
78
79 import com.linecorp.centraldogma.common.Change;
80 import com.linecorp.centraldogma.common.Entry;
81 import com.linecorp.centraldogma.common.EntryType;
82 import com.linecorp.centraldogma.common.Markup;
83 import com.linecorp.centraldogma.common.Revision;
84 import com.linecorp.centraldogma.internal.Jackson;
85 import com.linecorp.centraldogma.internal.Util;
86 import com.linecorp.centraldogma.server.MirrorException;
87 import com.linecorp.centraldogma.server.command.Command;
88 import com.linecorp.centraldogma.server.command.CommandExecutor;
89 import com.linecorp.centraldogma.server.mirror.MirrorCredential;
90 import com.linecorp.centraldogma.server.mirror.MirrorDirection;
91 import com.linecorp.centraldogma.server.storage.StorageException;
92 import com.linecorp.centraldogma.server.storage.repository.Repository;
93
94 abstract class AbstractGitMirror extends AbstractMirror {
95
96 private static final Logger logger = LoggerFactory.getLogger(AbstractGitMirror.class);
97
98
99 private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json";
100
101
102 private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = '.' + MIRROR_STATE_FILE_NAME;
103
104 private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL);
105
106 private static final byte[] EMPTY_BYTE = new byte[0];
107
108 private static final Pattern DISALLOWED_CHARS = Pattern.compile("[^-_a-zA-Z]");
109 private static final Pattern CONSECUTIVE_UNDERSCORES = Pattern.compile("_+");
110
111 private static final int GIT_TIMEOUT_SECS = 60;
112
113 private static final String HEAD_REF_MASTER = Constants.R_HEADS + Constants.MASTER;
114
115 @Nullable
116 private IgnoreNode ignoreNode;
117
118 AbstractGitMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential,
119 Repository localRepo, String localPath,
120 URI remoteRepoUri, String remotePath, @Nullable String remoteBranch,
121 @Nullable String gitignore) {
122 super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, remoteBranch,
123 gitignore);
124
125 if (gitignore != null) {
126 ignoreNode = new IgnoreNode();
127 try {
128 ignoreNode.parse(new ByteArrayInputStream(gitignore.getBytes()));
129 } catch (IOException e) {
130 throw new IllegalArgumentException("Failed to read gitignore: " + gitignore, e);
131 }
132 }
133 }
134
135 GitWithAuth openGit(File workDir,
136 URIish remoteUri,
137 Consumer<TransportCommand<?, ?>> configurator) throws IOException, GitAPIException {
138
139 final File repoDir = new File(
140 workDir,
141 CONSECUTIVE_UNDERSCORES.matcher(DISALLOWED_CHARS.matcher(
142 remoteRepoUri().toASCIIString()).replaceAll("_")).replaceAll("_"));
143 final GitWithAuth git = new GitWithAuth(this, repoDir, remoteUri, configurator);
144 boolean success = false;
145 try {
146
147 final RemoteSetUrlCommand remoteSetUrl = git.remoteSetUrl();
148 remoteSetUrl.setRemoteName(Constants.DEFAULT_REMOTE_NAME);
149 remoteSetUrl.setRemoteUri(remoteUri);
150
151 remoteSetUrl.setUriType(UriType.FETCH);
152 remoteSetUrl.call();
153
154 remoteSetUrl.setUriType(UriType.PUSH);
155 remoteSetUrl.call();
156
157
158
159
160
161
162
163 success = true;
164 return git;
165 } finally {
166 if (!success) {
167 git.close();
168 }
169 }
170 }
171
172 void mirrorLocalToRemote(
173 GitWithAuth git, int maxNumFiles, long maxNumBytes) throws GitAPIException, IOException {
174
175 final Ref headBranchRef = getHeadBranchRef(git);
176 final String headBranchRefName = headBranchRef.getName();
177 final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRefName);
178
179 final org.eclipse.jgit.lib.Repository gitRepository = git.getRepository();
180 try (ObjectReader reader = gitRepository.newObjectReader();
181 TreeWalk treeWalk = new TreeWalk(reader);
182 RevWalk revWalk = new RevWalk(reader)) {
183
184
185 final ObjectId headTreeId = revWalk.parseTree(headCommitId).getId();
186 treeWalk.reset(headTreeId);
187
188 final String mirrorStatePath = remotePath() + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME;
189 final Revision localHead = localRepo().normalizeNow(Revision.HEAD);
190 final Revision remoteCurrentRevision = remoteCurrentRevision(reader, treeWalk, mirrorStatePath);
191 if (localHead.equals(remoteCurrentRevision)) {
192
193 logger.debug("The remote repository '{}#{}' already at {}. Local repository: '{}'",
194 remoteRepoUri(), remoteBranch(), localHead, localRepo().name());
195 return;
196 }
197
198
199 treeWalk.reset(headTreeId);
200
201
202
203
204 final DirCache dirCache = DirCache.newInCore();
205 final DirCacheBuilder builder = dirCache.builder();
206 builder.addTree(EMPTY_BYTE, 0, reader, headTreeId);
207 builder.finish();
208
209 try (ObjectInserter inserter = gitRepository.newObjectInserter()) {
210 addModifiedEntryToCache(localHead, dirCache, reader, inserter,
211 treeWalk, maxNumFiles, maxNumBytes);
212
213 final MirrorState mirrorState = new MirrorState(localHead.text());
214 applyPathEdit(
215 dirCache, new InsertText(mirrorStatePath.substring(1),
216 inserter,
217 Jackson.writeValueAsPrettyString(mirrorState) + '\n'));
218 }
219
220 final ObjectId nextCommitId =
221 commit(gitRepository, dirCache, headCommitId, localHead);
222 updateRef(gitRepository, revWalk, headBranchRefName, nextCommitId);
223 }
224
225 git.push()
226 .setRefSpecs(new RefSpec(headBranchRefName))
227 .setAtomic(true)
228 .setTimeout(GIT_TIMEOUT_SECS)
229 .call();
230 }
231
232 void mirrorRemoteToLocal(
233 GitWithAuth git, CommandExecutor executor, int maxNumFiles, long maxNumBytes) throws Exception {
234 final String summary;
235 final String detail;
236 final Map<String, Change<?>> changes = new HashMap<>();
237 final Ref headBranchRef = getHeadBranchRef(git);
238
239 final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME;
240 final Revision localRev = localRepo().normalizeNow(Revision.HEAD);
241 if (!needsFetch(headBranchRef, mirrorStatePath, localRev)) {
242 return;
243 }
244
245
246
247 final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRef.getName());
248 try (ObjectReader reader = git.getRepository().newObjectReader();
249 TreeWalk treeWalk = new TreeWalk(reader);
250 RevWalk revWalk = new RevWalk(reader)) {
251
252
253 treeWalk.addTree(revWalk.parseTree(headCommitId).getId());
254 final String abbrId = reader.abbreviate(headCommitId).name();
255
256
257 changes.put(mirrorStatePath, Change.ofJsonUpsert(
258 mirrorStatePath, "{ \"sourceRevision\": \"" + headCommitId.name() + "\" }"));
259
260 summary = "Mirror " + abbrId + ", " + remoteRepoUri() + '#' + remoteBranch() +
261 " to the repository '" + localRepo().name() + '\'';
262 final RevCommit headCommit = revWalk.parseCommit(headCommitId);
263 detail = generateCommitDetail(headCommit);
264 logger.info(summary);
265 long numFiles = 0;
266 long numBytes = 0;
267 while (treeWalk.next()) {
268 final FileMode fileMode = treeWalk.getFileMode();
269 final String path = '/' + treeWalk.getPathString();
270
271 if (ignoreNode != null && path.startsWith(remotePath())) {
272 assert ignoreNode != null;
273 if (ignoreNode.isIgnored('/' + path.substring(remotePath().length()),
274 fileMode == FileMode.TREE) == MatchResult.IGNORED) {
275 continue;
276 }
277 }
278
279 if (fileMode == FileMode.TREE) {
280 maybeEnterSubtree(treeWalk, remotePath(), path);
281 continue;
282 }
283
284 if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) {
285
286 continue;
287 }
288
289
290 if (!path.startsWith(remotePath())) {
291 continue;
292 }
293
294 final String localPath = localPath() + path.substring(remotePath().length());
295
296
297 if (!Util.isValidFilePath(localPath)) {
298 continue;
299 }
300
301 if (++numFiles > maxNumFiles) {
302 throwMirrorException(maxNumFiles, "files");
303 return;
304 }
305
306 final ObjectId objectId = treeWalk.getObjectId(0);
307 final long contentLength = reader.getObjectSize(objectId, ObjectReader.OBJ_ANY);
308 if (numBytes > maxNumBytes - contentLength) {
309 throwMirrorException(maxNumBytes, "bytes");
310 return;
311 }
312 numBytes += contentLength;
313
314 final byte[] content = reader.open(objectId).getBytes();
315 switch (EntryType.guessFromPath(localPath)) {
316 case JSON:
317 final JsonNode jsonNode = Jackson.readTree(content);
318 changes.putIfAbsent(localPath, Change.ofJsonUpsert(localPath, jsonNode));
319 break;
320 case TEXT:
321 final String strVal = new String(content, UTF_8);
322 changes.putIfAbsent(localPath, Change.ofTextUpsert(localPath, strVal));
323 break;
324 }
325 }
326 }
327
328 final Map<String, Entry<?>> oldEntries = localRepo().find(
329 localRev, localPath() + "**", FIND_ALL_WITHOUT_CONTENT).join();
330 oldEntries.keySet().removeAll(changes.keySet());
331
332
333 oldEntries.forEach((path, entry) -> {
334 if (entry.type() != EntryType.DIRECTORY && !changes.containsKey(path)) {
335 changes.put(path, Change.ofRemoval(path));
336 }
337 });
338
339 executor.execute(Command.push(
340 MIRROR_AUTHOR, localRepo().parent().name(), localRepo().name(),
341 Revision.HEAD, summary, detail, Markup.PLAINTEXT, changes.values())).join();
342 }
343
344 private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision localRev)
345 throws JsonParseException, JsonMappingException {
346 final Entry<?> mirrorState = localRepo().getOrNull(localRev, mirrorStatePath).join();
347 final String localSourceRevision;
348 if (mirrorState == null || mirrorState.type() != EntryType.JSON) {
349 localSourceRevision = null;
350 } else {
351 localSourceRevision = Jackson.treeToValue((TreeNode) mirrorState.content(),
352 MirrorState.class).sourceRevision();
353 }
354
355 final ObjectId headCommitId = headBranchRef.getObjectId();
356 if (headCommitId.name().equals(localSourceRevision)) {
357 final String abbrId = headCommitId.abbreviate(OBJECT_ID_ABBREV_STRING_LENGTH).name();
358 logger.info("Repository '{}' already at {}, {}#{}", localRepo().name(), abbrId,
359 remoteRepoUri(), remoteBranch());
360 return false;
361 }
362 return true;
363 }
364
365 private Ref getHeadBranchRef(GitWithAuth git) throws GitAPIException {
366 if (remoteBranch() != null) {
367 final String headBranchRefName = Constants.R_HEADS + remoteBranch();
368 final Collection<Ref> refs = lsRemote(git, true);
369 return findHeadBranchRef(git, headBranchRefName, refs);
370 }
371
372
373
374 final Collection<Ref> refs = lsRemote(git, false);
375
376
377 final Optional<String> headRefNameOptional = refs.stream()
378 .filter(ref -> Constants.HEAD.equals(ref.getName()))
379 .map(ref -> ref.getTarget().getName())
380 .findFirst();
381 final String headBranchRefName;
382 if (headRefNameOptional.isPresent()) {
383 headBranchRefName = headRefNameOptional.get();
384 } else {
385
386 headBranchRefName = HEAD_REF_MASTER;
387 }
388 return findHeadBranchRef(git, headBranchRefName, refs);
389 }
390
391 private static Collection<Ref> lsRemote(GitWithAuth git,
392 boolean setHeads) throws GitAPIException {
393 return git.lsRemote()
394 .setTags(false)
395 .setTimeout(GIT_TIMEOUT_SECS)
396 .setHeads(setHeads)
397 .call();
398 }
399
400 private static Ref findHeadBranchRef(GitWithAuth git, String headBranchRefName, Collection<Ref> refs) {
401 final Optional<Ref> headBranchRef = refs.stream()
402 .filter(ref -> headBranchRefName.equals(ref.getName()))
403 .findFirst();
404 if (headBranchRef.isPresent()) {
405 return headBranchRef.get();
406 }
407 throw new MirrorException("Remote does not have " + headBranchRefName + " branch. remote: " +
408 git.remoteUri());
409 }
410
411 private static String generateCommitDetail(RevCommit headCommit) {
412 final PersonIdent authorIdent = headCommit.getAuthorIdent();
413 return "Remote commit:\n" +
414 "- SHA: " + headCommit.name() + '\n' +
415 "- Subject: " + headCommit.getShortMessage() + '\n' +
416 "- Author: " + authorIdent.getName() + " <" +
417 authorIdent.getEmailAddress() + "> \n" +
418 "- Date: " + authorIdent.getWhen() + "\n\n" +
419 headCommit.getFullMessage();
420 }
421
422 @Nullable
423 private Revision remoteCurrentRevision(
424 ObjectReader reader, TreeWalk treeWalk, String mirrorStatePath) {
425 try {
426 while (treeWalk.next()) {
427 final FileMode fileMode = treeWalk.getFileMode();
428 final String path = '/' + treeWalk.getPathString();
429
430
431 if (fileMode == FileMode.TREE) {
432 if (remotePath().startsWith(path + '/')) {
433 treeWalk.enterSubtree();
434 }
435 continue;
436 }
437
438 if (!path.equals(mirrorStatePath)) {
439 continue;
440 }
441
442 final byte[] content = currentEntryContent(reader, treeWalk);
443 final MirrorState mirrorState = Jackson.readValue(content, MirrorState.class);
444 return new Revision(mirrorState.sourceRevision());
445 }
446
447 return null;
448 } catch (Exception e) {
449 logger.warn("Unexpected exception while retrieving the remote source revision", e);
450 return null;
451 }
452 }
453
454 private static ObjectId fetchRemoteHeadAndGetCommitId(
455 GitWithAuth git, String headBranchRefName) throws GitAPIException, IOException {
456 final FetchResult fetchResult = git.fetch()
457 .setDepth(1)
458 .setRefSpecs(new RefSpec(headBranchRefName))
459 .setRemoveDeletedRefs(true)
460 .setTagOpt(TagOpt.NO_TAGS)
461 .setTimeout(GIT_TIMEOUT_SECS)
462 .call();
463 final ObjectId commitId = fetchResult.getAdvertisedRef(headBranchRefName).getObjectId();
464 final RefUpdate refUpdate = git.getRepository().updateRef(headBranchRefName);
465 refUpdate.setNewObjectId(commitId);
466 refUpdate.setForceUpdate(true);
467 refUpdate.update();
468 return commitId;
469 }
470
471 private Map<String, Entry<?>> localHeadEntries(Revision localHead) {
472 final Map<String, Entry<?>> localRawHeadEntries = localRepo().find(localHead, localPath() + "**")
473 .join();
474
475 final Stream<Map.Entry<String, Entry<?>>> entryStream =
476 localRawHeadEntries.entrySet()
477 .stream();
478 if (ignoreNode == null) {
479
480 return entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
481 }
482
483 final Map<String, Entry<?>> sortedMap =
484 entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
485 (v1, v2) -> v1, LinkedHashMap::new));
486
487 final HashMap<String, Entry<?>> result = new HashMap<>(sortedMap.size());
488 String lastIgnoredDirectory = null;
489 for (Map.Entry<String, ? extends Entry<?>> entry : sortedMap.entrySet()) {
490 final String path = entry.getKey();
491 final boolean isDirectory = entry.getValue().type() == EntryType.DIRECTORY;
492 assert ignoreNode != null;
493 final MatchResult ignoreResult = ignoreNode.isIgnored(
494 path.substring(localPath().length()), isDirectory);
495 if (ignoreResult == MatchResult.IGNORED) {
496 if (isDirectory) {
497 lastIgnoredDirectory = path;
498 }
499 continue;
500 }
501 if (ignoreResult == MatchResult.CHECK_PARENT) {
502 if (lastIgnoredDirectory != null && path.startsWith(lastIgnoredDirectory)) {
503 continue;
504 }
505 }
506 result.put(path, entry.getValue());
507 }
508
509 return result;
510 }
511
512 private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, ObjectReader reader,
513 ObjectInserter inserter, TreeWalk treeWalk,
514 int maxNumFiles, long maxNumBytes) throws IOException {
515 final Map<String, Entry<?>> localHeadEntries = localHeadEntries(localHead);
516 long numFiles = 0;
517 long numBytes = 0;
518 while (treeWalk.next()) {
519 final FileMode fileMode = treeWalk.getFileMode();
520 final String pathString = treeWalk.getPathString();
521 final String remoteFilePath = '/' + pathString;
522
523
524 if (fileMode == FileMode.TREE) {
525 maybeEnterSubtree(treeWalk, remotePath(), remoteFilePath);
526 continue;
527 }
528
529 if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) {
530
531 continue;
532 }
533
534
535 if (!remoteFilePath.startsWith(remotePath())) {
536 continue;
537 }
538
539 final String localFilePath = localPath() + remoteFilePath.substring(remotePath().length());
540
541
542 if (!Util.isValidFilePath(localFilePath)) {
543 continue;
544 }
545
546 final Entry<?> entry = localHeadEntries.remove(localFilePath);
547 if (entry == null) {
548
549 applyPathEdit(dirCache, new DeletePath(pathString));
550 continue;
551 }
552
553 if (++numFiles > maxNumFiles) {
554 throwMirrorException(maxNumFiles, "files");
555 return;
556 }
557
558 final byte[] oldContent = currentEntryContent(reader, treeWalk);
559 final long contentLength = applyPathEdit(dirCache, inserter, pathString, entry, oldContent);
560 numBytes += contentLength;
561 if (numBytes > maxNumBytes) {
562 throwMirrorException(maxNumBytes, "bytes");
563 return;
564 }
565 }
566
567
568 for (Map.Entry<String, Entry<?>> entry : localHeadEntries.entrySet()) {
569 final Entry<?> value = entry.getValue();
570 if (value.type() == EntryType.DIRECTORY) {
571 continue;
572 }
573 if (entry.getKey().endsWith(MIRROR_STATE_FILE_NAME)) {
574 continue;
575 }
576
577 if (++numFiles > maxNumFiles) {
578 throwMirrorException(maxNumFiles, "files");
579 return;
580 }
581
582 final String convertedPath = remotePath().substring(1) +
583 entry.getKey().substring(localPath().length());
584 final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, value, null);
585 numBytes += contentLength;
586 if (numBytes > maxNumBytes) {
587 throwMirrorException(maxNumBytes, "bytes");
588 }
589 }
590 }
591
592 private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, String pathString,
593 Entry<?> entry, @Nullable byte[] oldContent)
594 throws JsonProcessingException {
595 switch (EntryType.guessFromPath(pathString)) {
596 case JSON:
597 final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null;
598 final JsonNode newJsonNode = (JsonNode) entry.content();
599
600
601 if (!Objects.equals(newJsonNode, oldJsonNode)) {
602
603 final String newContent = newJsonNode.toPrettyString() + '\n';
604 applyPathEdit(dirCache, new InsertText(pathString, inserter, newContent));
605 return newContent.length();
606 }
607 break;
608 case TEXT:
609 final String sanitizedOldText = oldContent != null ?
610 sanitizeText(new String(oldContent, UTF_8)) : null;
611 final String sanitizedNewText = entry.contentAsText();
612
613 if (!sanitizedNewText.equals(sanitizedOldText)) {
614 applyPathEdit(dirCache, new InsertText(pathString, inserter, sanitizedNewText));
615 return sanitizedNewText.length();
616 }
617 break;
618 }
619 return 0;
620 }
621
622 private static void applyPathEdit(DirCache dirCache, PathEdit edit) {
623 final DirCacheEditor e = dirCache.editor();
624 e.add(edit);
625 e.finish();
626 }
627
628 private static byte[] currentEntryContent(ObjectReader reader, TreeWalk treeWalk) throws IOException {
629 final ObjectId objectId = treeWalk.getObjectId(0);
630 return reader.open(objectId).getBytes();
631 }
632
633 private static void maybeEnterSubtree(
634 TreeWalk treeWalk, String remotePath, String path) throws IOException {
635
636
637
638
639 if (path.startsWith(remotePath)) {
640 treeWalk.enterSubtree();
641 return;
642 }
643
644
645
646
647
648 final int pathLen = path.length() + 1;
649 if (pathLen == remotePath.length() && remotePath.startsWith(path)) {
650 treeWalk.enterSubtree();
651 return;
652 }
653
654
655
656
657
658 if (pathLen < remotePath.length() && remotePath.startsWith(path + '/')) {
659 treeWalk.enterSubtree();
660 }
661 }
662
663
664
665
666 private static String sanitizeText(String text) {
667 if (text.indexOf('\r') >= 0) {
668 text = CR.matcher(text).replaceAll("");
669 }
670 if (!text.isEmpty() && !text.endsWith("\n")) {
671 text += "\n";
672 }
673 return text;
674 }
675
676 private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache,
677 ObjectId headCommitId, Revision localHead) throws IOException {
678 try (ObjectInserter inserter = gitRepository.newObjectInserter()) {
679
680 final ObjectId nextTreeId = dirCache.writeTree(inserter);
681
682 final PersonIdent personIdent =
683 new PersonIdent(MIRROR_AUTHOR.name(), MIRROR_AUTHOR.email(),
684 System.currentTimeMillis() / 1000L * 1000L,
685 0);
686
687 final CommitBuilder commitBuilder = new CommitBuilder();
688 commitBuilder.setAuthor(personIdent);
689 commitBuilder.setCommitter(personIdent);
690 commitBuilder.setTreeId(nextTreeId);
691 commitBuilder.setEncoding(UTF_8);
692 commitBuilder.setParentId(headCommitId);
693
694 final String summary = "Mirror '" + localRepo().name() + "' at " + localHead +
695 " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n";
696 logger.info(summary);
697 commitBuilder.setMessage(summary);
698
699 final ObjectId nextCommitId = inserter.insert(commitBuilder);
700 inserter.flush();
701 return nextCommitId;
702 }
703 }
704
705 private <T> T throwMirrorException(long number, String filesOrBytes) {
706 throw new MirrorException("mirror (" + remoteRepoUri() + '#' + remoteBranch() +
707 ") contains more than " + number + ' ' + filesOrBytes);
708 }
709
710 static void updateRef(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk,
711 String ref, ObjectId commitId) throws IOException {
712 final RefUpdate refUpdate = jGitRepository.updateRef(ref);
713 refUpdate.setNewObjectId(commitId);
714
715 final Result res = refUpdate.update(revWalk);
716 switch (res) {
717 case NEW:
718 case FAST_FORWARD:
719
720 break;
721 default:
722 throw new StorageException("unexpected refUpdate state: " + res);
723 }
724 }
725
726 private static final class InsertText extends PathEdit {
727 private final ObjectInserter inserter;
728 private final String text;
729
730 InsertText(String entryPath, ObjectInserter inserter, String text) {
731 super(entryPath);
732 this.inserter = inserter;
733 this.text = text;
734 }
735
736 @Override
737 public void apply(DirCacheEntry ent) {
738 try {
739 ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8)));
740 ent.setFileMode(FileMode.REGULAR_FILE);
741 } catch (IOException e) {
742 throw new StorageException("failed to create a new text blob", e);
743 }
744 }
745 }
746 }