1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage.repository;
18
19 import static com.google.common.base.Preconditions.checkArgument;
20 import static com.google.common.collect.ImmutableList.toImmutableList;
21 import static com.linecorp.centraldogma.internal.CredentialUtil.credentialFile;
22 import static com.linecorp.centraldogma.server.internal.storage.repository.MirrorConverter.convertToMirror;
23 import static com.linecorp.centraldogma.server.internal.storage.repository.MirrorConverter.converterToMirrorConfig;
24
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Objects;
28 import java.util.concurrent.CompletableFuture;
29 import java.util.regex.Pattern;
30
31 import javax.annotation.Nullable;
32
33 import com.cronutils.model.Cron;
34 import com.cronutils.model.field.CronField;
35 import com.cronutils.model.field.CronFieldName;
36 import com.fasterxml.jackson.core.JsonProcessingException;
37 import com.fasterxml.jackson.databind.JsonNode;
38 import com.google.common.base.Strings;
39 import com.google.common.collect.ImmutableList;
40 import com.google.common.collect.ImmutableMap;
41
42 import com.linecorp.armeria.common.util.Exceptions;
43 import com.linecorp.armeria.common.util.UnmodifiableFuture;
44 import com.linecorp.centraldogma.common.Author;
45 import com.linecorp.centraldogma.common.Change;
46 import com.linecorp.centraldogma.common.Entry;
47 import com.linecorp.centraldogma.common.EntryNotFoundException;
48 import com.linecorp.centraldogma.common.Markup;
49 import com.linecorp.centraldogma.common.Revision;
50 import com.linecorp.centraldogma.internal.Jackson;
51 import com.linecorp.centraldogma.internal.api.v1.MirrorRequest;
52 import com.linecorp.centraldogma.server.ZoneConfig;
53 import com.linecorp.centraldogma.server.command.Command;
54 import com.linecorp.centraldogma.server.command.CommitResult;
55 import com.linecorp.centraldogma.server.credential.Credential;
56 import com.linecorp.centraldogma.server.mirror.Mirror;
57 import com.linecorp.centraldogma.server.mirror.MirrorDirection;
58 import com.linecorp.centraldogma.server.storage.repository.MetaRepository;
59 import com.linecorp.centraldogma.server.storage.repository.Repository;
60
61 public final class DefaultMetaRepository extends RepositoryWrapper implements MetaRepository {
62
63 private static final Pattern MIRROR_PATH_PATTERN = Pattern.compile("/repos/[^/]+/mirrors/[^/]+\\.json");
64
65 private static final Pattern REPO_CREDENTIAL_PATH_PATTERN =
66 Pattern.compile("/repos/[^/]+/credentials/[^/]+\\.json");
67
68 private static final Pattern PROJECT_CREDENTIAL_PATH_PATTERN =
69 Pattern.compile("/credentials/[^/]+\\.json");
70
71 public static final String CREDENTIALS = "/credentials/";
72
73 public static final String LEGACY_MIRRORS_PATH = "/mirrors/";
74
75 public static final String ALL_MIRRORS = "/repos/*/mirrors/*.json";
76
77 public static boolean isMetaFile(String path) {
78 return "/mirrors.json".equals(path) || "/credentials.json".equals(path) ||
79 (path.endsWith(".json") &&
80 (path.startsWith(CREDENTIALS) || path.startsWith(LEGACY_MIRRORS_PATH))) ||
81 isMirrorOrCredentialFile(path);
82 }
83
84 public static boolean isMirrorOrCredentialFile(String path) {
85 return MIRROR_PATH_PATTERN.matcher(path).matches() ||
86 REPO_CREDENTIAL_PATH_PATTERN.matcher(path).matches() ||
87 PROJECT_CREDENTIAL_PATH_PATTERN.matcher(path).matches();
88 }
89
90 public static String mirrorFile(String repoName, String mirrorId) {
91 return "/repos/" + repoName + "/mirrors/" + mirrorId + ".json";
92 }
93
94 public DefaultMetaRepository(Repository repo) {
95 super(repo);
96 }
97
98 @Override
99 public org.eclipse.jgit.lib.Repository jGitRepository() {
100 return unwrap().jGitRepository();
101 }
102
103 @Override
104 public CompletableFuture<List<Mirror>> mirrors(boolean includeDisabled) {
105 final CompletableFuture<List<Mirror>> future = allMirrors();
106 return maybeFilter(future, includeDisabled);
107 }
108
109 @Override
110 public CompletableFuture<List<Mirror>> mirrors(String repoName, boolean includeDisabled) {
111 final CompletableFuture<List<Mirror>> future = allMirrors(repoName);
112 return maybeFilter(future, includeDisabled);
113 }
114
115 private static CompletableFuture<List<Mirror>> maybeFilter(CompletableFuture<List<Mirror>> future,
116 boolean includeDisabled) {
117 if (includeDisabled) {
118 return future;
119 }
120 return future.thenApply(mirrors -> mirrors.stream().filter(Mirror::enabled).collect(toImmutableList()));
121 }
122
123 @Override
124 public CompletableFuture<Mirror> mirror(String repoName, String id, Revision revision) {
125 final String mirrorFile = mirrorFile(repoName, id);
126 return find(revision, mirrorFile).thenCompose(entries -> {
127 @SuppressWarnings("unchecked")
128 final Entry<JsonNode> entry = (Entry<JsonNode>) entries.get(mirrorFile);
129 if (entry == null) {
130 throw new EntryNotFoundException(
131 "failed to find mirror '" + mirrorFile + "' in " + parent().name() + '/' + name() +
132 " (revision: " + revision + ')');
133 }
134
135 final JsonNode mirrorJson = entry.content();
136 if (!mirrorJson.isObject()) {
137 throw newInvalidJsonTypeException(mirrorFile, mirrorJson);
138 }
139 final MirrorConfig c;
140 try {
141 c = Jackson.treeToValue(mirrorJson, MirrorConfig.class);
142 } catch (JsonProcessingException e) {
143 throw new RepositoryMetadataException("failed to load the mirror configuration", e);
144 }
145
146 if (c.credentialName().isEmpty()) {
147 if (!parent().repos().exists(repoName)) {
148 throw mirrorNotFound(revision, mirrorFile);
149 }
150 return CompletableFuture.completedFuture(convertToMirror(c, parent(), Credential.NONE));
151 }
152
153 final CompletableFuture<Credential> future = credential(c.credentialName());
154 return future.thenApply(credential -> convertToMirror(c, parent(), credential));
155 });
156 }
157
158 private EntryNotFoundException mirrorNotFound(Revision revision, String mirrorFile) {
159 return new EntryNotFoundException(
160 "failed to find a mirror config for '" + mirrorFile + "' in " +
161 parent().name() + '/' + name() + " (revision: " + revision + ')');
162 }
163
164 private CompletableFuture<List<Mirror>> allMirrors() {
165 return find(ALL_MIRRORS).thenCompose(this::handleAllMirrors);
166 }
167
168 private CompletableFuture<List<Mirror>> allMirrors(String repoName) {
169 return find("/repos/" + repoName + "/mirrors/*.json").thenCompose(this::handleAllMirrors);
170 }
171
172 private CompletableFuture<List<Mirror>> handleAllMirrors(Map<String, Entry<?>> entries) {
173 if (entries.isEmpty()) {
174 return UnmodifiableFuture.completedFuture(ImmutableList.of());
175 }
176
177 final CompletableFuture<List<Credential>> future = allCredentials();
178 return future.thenApply(credentials -> {
179 final List<MirrorConfig> mirrorConfigs = toMirrorConfigs(entries);
180 return mirrorConfigs.stream()
181 .map(mirrorConfig -> convertToMirror(
182 mirrorConfig, parent(), credentials))
183 .filter(Objects::nonNull)
184 .collect(toImmutableList());
185 });
186 }
187
188 private CompletableFuture<List<Credential>> allCredentials() {
189
190 return find("/credentials/*.json,/repos/*/credentials/*.json").thenApply(
191 entries -> credentials(entries, null));
192 }
193
194 private List<MirrorConfig> toMirrorConfigs(Map<String, Entry<?>> entries) {
195 return entries.entrySet().stream().map(entry -> {
196 final JsonNode mirrorJson = (JsonNode) entry.getValue().content();
197 if (!mirrorJson.isObject()) {
198 throw newInvalidJsonTypeException(entry.getKey(), mirrorJson);
199 }
200 try {
201 return Jackson.treeToValue(mirrorJson, MirrorConfig.class);
202 } catch (JsonProcessingException e) {
203 return Exceptions.throwUnsafely(e);
204 }
205 })
206 .collect(toImmutableList());
207 }
208
209 @Override
210 public CompletableFuture<List<Credential>> projectCredentials() {
211 return find(CREDENTIALS + "*.json").thenApply(
212 entries -> credentials(entries, null));
213 }
214
215 @Override
216 public CompletableFuture<List<Credential>> repoCredentials(String repoName) {
217 return find("/repos/" + repoName + CREDENTIALS + "*.json")
218 .thenApply(entries -> credentials(entries, repoName));
219 }
220
221 private List<Credential> credentials(Map<String, Entry<?>> entries, @Nullable String repoName) {
222 if (entries.isEmpty()) {
223 return ImmutableList.of();
224 }
225 try {
226 return parseCredentials(entries);
227 } catch (Exception e) {
228 String message = "failed to load the credential configuration";
229 if (repoName != null) {
230 message += " for " + repoName;
231 }
232 throw new RepositoryMetadataException(message, e);
233 }
234 }
235
236 @Override
237 public CompletableFuture<Credential> credential(String credentialName) {
238 final String credentialFile = credentialFile(credentialName);
239 return credential0(credentialFile);
240 }
241
242 private CompletableFuture<Credential> credential0(String credentialFile) {
243 return find(credentialFile).thenApply(entries -> {
244 @SuppressWarnings("unchecked")
245 final Entry<JsonNode> entry = (Entry<JsonNode>) entries.get(credentialFile);
246 if (entry == null) {
247 throw new EntryNotFoundException("failed to find credential file '" + credentialFile + "' in " +
248 parent().name() + '/' + name());
249 }
250
251 try {
252 return parseCredential(credentialFile, entry);
253 } catch (Exception e) {
254 throw new RepositoryMetadataException(
255 "failed to load the credential configuration. credential file: " + credentialFile, e);
256 }
257 });
258 }
259
260 private List<Credential> parseCredentials(Map<String, Entry<?>> entries)
261 throws JsonProcessingException {
262 return entries.entrySet().stream()
263 .map(entry -> {
264 try {
265
266 return parseCredential(entry.getKey(),
267 (Entry<JsonNode>) entry.getValue());
268 } catch (JsonProcessingException e) {
269 return Exceptions.throwUnsafely(e);
270 }
271 })
272 .collect(toImmutableList());
273 }
274
275 private Credential parseCredential(String credentialFile, Entry<JsonNode> entry)
276 throws JsonProcessingException {
277 final JsonNode credentialJson = entry.content();
278 if (!credentialJson.isObject()) {
279 throw newInvalidJsonTypeException(credentialFile, credentialJson);
280 }
281 return Jackson.treeToValue(credentialJson, Credential.class);
282 }
283
284 private RepositoryMetadataException newInvalidJsonTypeException(
285 String fileName, JsonNode credentialJson) {
286 return new RepositoryMetadataException(parent().name() + '/' + name() + fileName +
287 " must be an object: " + credentialJson.getNodeType());
288 }
289
290 private CompletableFuture<Map<String, Entry<?>>> find(String filePattern) {
291 return find(Revision.HEAD, filePattern, ImmutableMap.of());
292 }
293
294 @Override
295 public CompletableFuture<Command<CommitResult>> createMirrorPushCommand(
296 String repoName, MirrorRequest mirrorRequest, Author author,
297 @Nullable ZoneConfig zoneConfig, boolean update) {
298 validateMirror(mirrorRequest, zoneConfig);
299 if (update) {
300 final String summary = "Update the mirror '" + mirrorRequest.id() + "' in " + repoName;
301 return mirror(repoName, mirrorRequest.id()).thenApply(mirror -> {
302 return newMirrorCommand(repoName, mirrorRequest, author, summary);
303 });
304 } else {
305 String summary = "Create a new mirror from " + mirrorRequest.remoteUrl() +
306 mirrorRequest.remotePath() + '#' + mirrorRequest.remoteBranch() + " into " +
307 repoName + mirrorRequest.localPath();
308 if (MirrorDirection.valueOf(mirrorRequest.direction()) == MirrorDirection.REMOTE_TO_LOCAL) {
309 summary = "[Remote-to-local] " + summary;
310 } else {
311 summary = "[Local-to-remote] " + summary;
312 }
313 return UnmodifiableFuture.completedFuture(
314 newMirrorCommand(repoName, mirrorRequest, author, summary));
315 }
316 }
317
318 @Override
319 public CompletableFuture<Command<CommitResult>> createCredentialPushCommand(Credential credential,
320 Author author, boolean update) {
321 final String credentialName = credential.name();
322 if (update) {
323 return credential(credentialName).thenApply(c -> {
324 final String summary = "Update the mirror credential '" + credentialName + '\'';
325 return newCredentialCommand(credentialFile(credentialName), credential, author, summary);
326 });
327 }
328 final String summary = "Create a new mirror credential for " + credential.name();
329 return UnmodifiableFuture.completedFuture(newCredentialCommand(
330 credentialFile(credentialName), credential, author, summary));
331 }
332
333 @Override
334 public CompletableFuture<Command<CommitResult>> createCredentialPushCommand(String repoName,
335 Credential credential,
336 Author author, boolean update) {
337 final String credentialName = credential.name();
338 if (update) {
339 return credential(credentialName).thenApply(c -> {
340 final String summary =
341 "Update the mirror credential '" + credentialName + '\'';
342 return newCredentialCommand(
343 credentialFile(credentialName), credential, author, summary);
344 });
345 }
346 final String summary = "Create a new mirror credential '" + credentialName + '\'';
347 return UnmodifiableFuture.completedFuture(
348 newCredentialCommand(credentialFile(credentialName), credential, author, summary));
349 }
350
351 private Command<CommitResult> newCredentialCommand(String credentialFile, Credential credential,
352 Author author, String summary) {
353 final JsonNode jsonNode = Jackson.valueToTree(credential);
354 final Change<JsonNode> change = Change.ofJsonUpsert(credentialFile, jsonNode);
355 return Command.push(author, parent().name(), name(), Revision.HEAD, summary, "", Markup.PLAINTEXT,
356 change);
357 }
358
359 private Command<CommitResult> newMirrorCommand(String repoName, MirrorRequest mirrorRequest,
360 Author author, String summary) {
361 final MirrorConfig mirrorConfig = converterToMirrorConfig(mirrorRequest);
362 final JsonNode jsonNode = Jackson.valueToTree(mirrorConfig);
363 final Change<JsonNode> change =
364 Change.ofJsonUpsert(mirrorFile(repoName, mirrorConfig.id()), jsonNode);
365 return Command.push(author, parent().name(), name(), Revision.HEAD, summary, "", Markup.PLAINTEXT,
366 change);
367 }
368
369 private static void validateMirror(MirrorRequest mirror, @Nullable ZoneConfig zoneConfig) {
370 checkArgument(!Strings.isNullOrEmpty(mirror.id()), "Mirror ID is empty");
371 final String scheduleString = mirror.schedule();
372 if (scheduleString != null) {
373 final Cron schedule = MirrorConfig.CRON_PARSER.parse(scheduleString);
374 final CronField secondField = schedule.retrieve(CronFieldName.SECOND);
375 checkArgument(!secondField.getExpression().asString().contains("*"),
376 "The second field of the schedule must be specified. (seconds: *, expected: 0-59)");
377 }
378
379 final String zone = mirror.zone();
380 if (zone != null) {
381 checkArgument(zoneConfig != null, "Zone configuration is missing");
382 checkArgument(zoneConfig.allZones().contains(zone),
383 "The zone '%s' is not in the zone configuration: %s", zone, zoneConfig);
384 }
385 }
386 }