1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server;
18
19 import static com.google.common.base.MoreObjects.firstNonNull;
20 import static com.google.common.base.Preconditions.checkArgument;
21 import static com.google.common.collect.ImmutableList.toImmutableList;
22 import static com.linecorp.armeria.common.util.InetAddressPredicates.ofCidr;
23 import static com.linecorp.armeria.common.util.InetAddressPredicates.ofExact;
24 import static com.linecorp.armeria.server.ClientAddressSource.ofHeader;
25 import static com.linecorp.armeria.server.ClientAddressSource.ofProxyProtocol;
26 import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_MAX_NUM_BYTES_PER_MIRROR;
27 import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_MAX_NUM_FILES_PER_MIRROR;
28 import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_MAX_REMOVED_REPOSITORY_AGE_MILLIS;
29 import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_NUM_MIRRORING_THREADS;
30 import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_NUM_REPOSITORY_WORKERS;
31 import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_REPOSITORY_CACHE_SPEC;
32 import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.validateCacheSpec;
33 import static java.util.Objects.requireNonNull;
34
35 import java.io.File;
36 import java.io.IOException;
37 import java.net.InetAddress;
38 import java.net.InetSocketAddress;
39 import java.util.ArrayList;
40 import java.util.List;
41 import java.util.Map;
42 import java.util.Map.Entry;
43 import java.util.Optional;
44 import java.util.ServiceLoader;
45 import java.util.function.Predicate;
46
47 import javax.annotation.Nullable;
48
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
51
52 import com.fasterxml.jackson.annotation.JsonProperty;
53 import com.fasterxml.jackson.core.JsonGenerator;
54 import com.fasterxml.jackson.core.JsonParser;
55 import com.fasterxml.jackson.core.JsonProcessingException;
56 import com.fasterxml.jackson.databind.DeserializationContext;
57 import com.fasterxml.jackson.databind.JsonDeserializer;
58 import com.fasterxml.jackson.databind.JsonMappingException;
59 import com.fasterxml.jackson.databind.JsonNode;
60 import com.fasterxml.jackson.databind.JsonSerializer;
61 import com.fasterxml.jackson.databind.SerializerProvider;
62 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
63 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
64 import com.fasterxml.jackson.databind.node.JsonNodeType;
65 import com.fasterxml.jackson.databind.util.StdConverter;
66 import com.google.common.collect.ImmutableList;
67 import com.google.common.collect.ImmutableList.Builder;
68 import com.google.common.collect.ImmutableMap;
69 import com.google.common.collect.ImmutableSet;
70 import com.google.common.collect.Streams;
71
72 import com.linecorp.armeria.common.HttpHeaderNames;
73 import com.linecorp.armeria.common.SessionProtocol;
74 import com.linecorp.armeria.server.ClientAddressSource;
75 import com.linecorp.armeria.server.ServerPort;
76 import com.linecorp.centraldogma.internal.Jackson;
77 import com.linecorp.centraldogma.server.auth.AuthConfig;
78 import com.linecorp.centraldogma.server.storage.repository.Repository;
79
80 import io.netty.util.NetUtil;
81
82
83
84
85 public final class CentralDogmaConfig {
86
87 private static final Logger logger = LoggerFactory.getLogger(CentralDogmaConfig.class);
88
89 private static final Map<String, ConfigValueConverter> CONFIG_VALUE_CONVERTERS;
90
91 static {
92 final ArrayList<ConfigValueConverter> configValueConverters = new ArrayList<>();
93 Streams.stream(ServiceLoader.load(ConfigValueConverter.class)).forEach(configValueConverters::add);
94 configValueConverters.add(DefaultConfigValueConverter.INSTANCE);
95 final ImmutableMap.Builder<String, ConfigValueConverter> builder = ImmutableMap.builder();
96 for (ConfigValueConverter configValueConverter : configValueConverters) {
97 configValueConverter.supportedPrefixes()
98 .forEach(prefix -> builder.put(prefix, configValueConverter));
99 }
100 CONFIG_VALUE_CONVERTERS = ImmutableMap.copyOf(builder.buildOrThrow());
101 final StringBuilder sb = new StringBuilder();
102 sb.append('{');
103 CONFIG_VALUE_CONVERTERS.entrySet().stream().sorted(Entry.comparingByKey()).forEach(
104 entry -> sb.append(entry.getKey())
105 .append('=')
106 .append(entry.getValue().getClass().getName()).append(", "));
107 sb.setLength(sb.length() - 2);
108 sb.append('}');
109
110 logger.debug("Available {}s: {}", ConfigValueConverter.class.getName(), sb);
111 }
112
113
114
115
116
117 @Nullable
118 public static String convertValue(@Nullable String value, String propertyName) {
119 if (value == null) {
120 return null;
121 }
122
123 final int index = value.indexOf(':');
124 if (index <= 0) {
125
126 return value;
127 }
128
129 final String prefix = value.substring(0, index);
130 final String rest = value.substring(index + 1);
131
132 final ConfigValueConverter converter = CONFIG_VALUE_CONVERTERS.get(prefix);
133 if (converter != null) {
134 return converter.convert(prefix, rest);
135 }
136
137 throw new IllegalArgumentException("failed to convert " + propertyName + ". value: " + value);
138 }
139
140 private final File dataDir;
141
142
143 private final List<ServerPort> ports;
144 @Nullable
145 private final Integer numWorkers;
146 @Nullable
147 private final Integer maxNumConnections;
148 @Nullable
149 private final Long requestTimeoutMillis;
150 @Nullable
151 private final Long idleTimeoutMillis;
152 @Nullable
153 private final Integer maxFrameLength;
154 @Nullable
155 private final TlsConfig tls;
156 @Nullable
157 private final List<String> trustedProxyAddresses;
158 @Nullable
159 private final List<String> clientAddressSources;
160
161 private final Predicate<InetAddress> trustedProxyAddressPredicate;
162 private final List<ClientAddressSource> clientAddressSourceList;
163
164
165 private final Integer numRepositoryWorkers;
166 private final long maxRemovedRepositoryAgeMillis;
167
168
169 private final String repositoryCacheSpec;
170
171
172 private final boolean webAppEnabled;
173
174 @Nullable
175 private final String webAppTitle;
176
177
178 private final boolean mirroringEnabled;
179 private final int numMirroringThreads;
180 private final int maxNumFilesPerMirror;
181 private final long maxNumBytesPerMirror;
182
183
184 @Nullable
185 private final GracefulShutdownTimeout gracefulShutdownTimeout;
186
187
188 private final ReplicationConfig replicationConfig;
189
190
191 private final boolean csrfTokenRequiredForThrift;
192
193
194 @Nullable
195 private final String accessLogFormat;
196
197 @Nullable
198 private final AuthConfig authConfig;
199
200 @Nullable
201 private final QuotaConfig writeQuotaPerRepository;
202
203 @Nullable
204 private final CorsConfig corsConfig;
205
206 CentralDogmaConfig(
207 @JsonProperty(value = "dataDir", required = true) File dataDir,
208 @JsonProperty(value = "ports", required = true)
209 @JsonDeserialize(contentUsing = ServerPortDeserializer.class)
210 List<ServerPort> ports,
211 @JsonProperty("tls") @Nullable TlsConfig tls,
212 @JsonProperty("trustedProxyAddresses") @Nullable List<String> trustedProxyAddresses,
213 @JsonProperty("clientAddressSources") @Nullable List<String> clientAddressSources,
214 @JsonProperty("numWorkers") @Nullable Integer numWorkers,
215 @JsonProperty("maxNumConnections") @Nullable Integer maxNumConnections,
216 @JsonProperty("requestTimeoutMillis") @Nullable Long requestTimeoutMillis,
217 @JsonProperty("idleTimeoutMillis") @Nullable Long idleTimeoutMillis,
218 @JsonProperty("maxFrameLength") @Nullable Integer maxFrameLength,
219 @JsonProperty("numRepositoryWorkers") @Nullable Integer numRepositoryWorkers,
220 @JsonProperty("repositoryCacheSpec") @Nullable String repositoryCacheSpec,
221 @JsonProperty("maxRemovedRepositoryAgeMillis") @Nullable Long maxRemovedRepositoryAgeMillis,
222 @JsonProperty("gracefulShutdownTimeout") @Nullable GracefulShutdownTimeout gracefulShutdownTimeout,
223 @JsonProperty("webAppEnabled") @Nullable Boolean webAppEnabled,
224 @JsonProperty("webAppTitle") @Nullable String webAppTitle,
225 @JsonProperty("mirroringEnabled") @Nullable Boolean mirroringEnabled,
226 @JsonProperty("numMirroringThreads") @Nullable Integer numMirroringThreads,
227 @JsonProperty("maxNumFilesPerMirror") @Nullable Integer maxNumFilesPerMirror,
228 @JsonProperty("maxNumBytesPerMirror") @Nullable Long maxNumBytesPerMirror,
229 @JsonProperty("replication") @Nullable ReplicationConfig replicationConfig,
230 @JsonProperty("csrfTokenRequiredForThrift") @Nullable Boolean csrfTokenRequiredForThrift,
231 @JsonProperty("accessLogFormat") @Nullable String accessLogFormat,
232 @JsonProperty("authentication") @Nullable AuthConfig authConfig,
233 @JsonProperty("writeQuotaPerRepository") @Nullable QuotaConfig writeQuotaPerRepository,
234 @JsonProperty("cors") @Nullable CorsConfig corsConfig) {
235
236 this.dataDir = requireNonNull(dataDir, "dataDir");
237 this.ports = ImmutableList.copyOf(requireNonNull(ports, "ports"));
238 this.corsConfig = corsConfig;
239 checkArgument(!ports.isEmpty(), "ports must have at least one port.");
240 this.tls = tls;
241 this.trustedProxyAddresses = trustedProxyAddresses;
242 this.clientAddressSources = clientAddressSources;
243
244 this.numWorkers = numWorkers;
245
246 this.maxNumConnections = maxNumConnections;
247 this.requestTimeoutMillis = requestTimeoutMillis;
248 this.idleTimeoutMillis = idleTimeoutMillis;
249 this.maxFrameLength = maxFrameLength;
250 this.numRepositoryWorkers = firstNonNull(numRepositoryWorkers, DEFAULT_NUM_REPOSITORY_WORKERS);
251 checkArgument(this.numRepositoryWorkers > 0,
252 "numRepositoryWorkers: %s (expected: > 0)", this.numRepositoryWorkers);
253 this.maxRemovedRepositoryAgeMillis = firstNonNull(maxRemovedRepositoryAgeMillis,
254 DEFAULT_MAX_REMOVED_REPOSITORY_AGE_MILLIS);
255 checkArgument(this.maxRemovedRepositoryAgeMillis >= 0,
256 "maxRemovedRepositoryAgeMillis: %s (expected: >= 0)", this.maxRemovedRepositoryAgeMillis);
257 this.repositoryCacheSpec = validateCacheSpec(
258 firstNonNull(repositoryCacheSpec, DEFAULT_REPOSITORY_CACHE_SPEC));
259
260 this.webAppEnabled = firstNonNull(webAppEnabled, true);
261 this.webAppTitle = webAppTitle;
262 this.mirroringEnabled = firstNonNull(mirroringEnabled, true);
263 this.numMirroringThreads = firstNonNull(numMirroringThreads, DEFAULT_NUM_MIRRORING_THREADS);
264 checkArgument(this.numMirroringThreads > 0,
265 "numMirroringThreads: %s (expected: > 0)", this.numMirroringThreads);
266 this.maxNumFilesPerMirror = firstNonNull(maxNumFilesPerMirror, DEFAULT_MAX_NUM_FILES_PER_MIRROR);
267 checkArgument(this.maxNumFilesPerMirror > 0,
268 "maxNumFilesPerMirror: %s (expected: > 0)", this.maxNumFilesPerMirror);
269 this.maxNumBytesPerMirror = firstNonNull(maxNumBytesPerMirror, DEFAULT_MAX_NUM_BYTES_PER_MIRROR);
270 checkArgument(this.maxNumBytesPerMirror > 0,
271 "maxNumBytesPerMirror: %s (expected: > 0)", this.maxNumBytesPerMirror);
272 this.gracefulShutdownTimeout = gracefulShutdownTimeout;
273 this.replicationConfig = firstNonNull(replicationConfig, ReplicationConfig.NONE);
274 this.csrfTokenRequiredForThrift = firstNonNull(csrfTokenRequiredForThrift, true);
275 this.accessLogFormat = accessLogFormat;
276
277 this.authConfig = authConfig;
278
279 final boolean hasTrustedProxyAddrCfg =
280 trustedProxyAddresses != null && !trustedProxyAddresses.isEmpty();
281 trustedProxyAddressPredicate =
282 hasTrustedProxyAddrCfg ? toTrustedProxyAddressPredicate(trustedProxyAddresses)
283 : addr -> false;
284 clientAddressSourceList =
285 toClientAddressSourceList(clientAddressSources, hasTrustedProxyAddrCfg,
286 ports.stream().anyMatch(ServerPort::hasProxyProtocol));
287
288 this.writeQuotaPerRepository = writeQuotaPerRepository;
289 }
290
291
292
293
294 @JsonProperty
295 public File dataDir() {
296 return dataDir;
297 }
298
299
300
301
302 @JsonProperty
303 @JsonSerialize(contentUsing = ServerPortSerializer.class)
304 public List<ServerPort> ports() {
305 return ports;
306 }
307
308
309
310
311 @Nullable
312 @JsonProperty
313 public TlsConfig tls() {
314 return tls;
315 }
316
317
318
319
320
321 @Nullable
322 @JsonProperty
323 public List<String> trustedProxyAddresses() {
324 return trustedProxyAddresses;
325 }
326
327
328
329
330
331
332
333
334
335
336 @Nullable
337 @JsonProperty
338 public List<String> clientAddressSources() {
339 return clientAddressSources;
340 }
341
342
343
344
345 @JsonProperty
346 @JsonSerialize(converter = OptionalConverter.class)
347 public Optional<Integer> numWorkers() {
348 return Optional.ofNullable(numWorkers);
349 }
350
351
352
353
354 @JsonProperty
355 @JsonSerialize(converter = OptionalConverter.class)
356 public Optional<Integer> maxNumConnections() {
357 return Optional.ofNullable(maxNumConnections);
358 }
359
360
361
362
363 @JsonProperty
364 @JsonSerialize(converter = OptionalConverter.class)
365 public Optional<Long> requestTimeoutMillis() {
366 return Optional.ofNullable(requestTimeoutMillis);
367 }
368
369
370
371
372 @JsonProperty
373 @JsonSerialize(converter = OptionalConverter.class)
374 public Optional<Long> idleTimeoutMillis() {
375 return Optional.ofNullable(idleTimeoutMillis);
376 }
377
378
379
380
381 @JsonProperty
382 @JsonSerialize(converter = OptionalConverter.class)
383 public Optional<Integer> maxFrameLength() {
384 return Optional.ofNullable(maxFrameLength);
385 }
386
387
388
389
390 @JsonProperty
391 int numRepositoryWorkers() {
392 return numRepositoryWorkers;
393 }
394
395
396
397
398
399
400 @JsonProperty
401 public long maxRemovedRepositoryAgeMillis() {
402 return maxRemovedRepositoryAgeMillis;
403 }
404
405
406
407
408
409
410 @JsonProperty
411 @Deprecated
412 public String cacheSpec() {
413 return repositoryCacheSpec;
414 }
415
416
417
418
419 @JsonProperty
420 public String repositoryCacheSpec() {
421 return repositoryCacheSpec;
422 }
423
424
425
426
427 @JsonProperty
428 @JsonSerialize(converter = OptionalConverter.class)
429 public Optional<GracefulShutdownTimeout> gracefulShutdownTimeout() {
430 return Optional.ofNullable(gracefulShutdownTimeout);
431 }
432
433
434
435
436 @JsonProperty
437 public boolean isWebAppEnabled() {
438 return webAppEnabled;
439 }
440
441
442
443
444 @Nullable
445 @JsonProperty("webAppTitle")
446 public String webAppTitle() {
447 return webAppTitle;
448 }
449
450
451
452
453 @JsonProperty
454 public boolean isMirroringEnabled() {
455 return mirroringEnabled;
456 }
457
458
459
460
461 @JsonProperty
462 public int numMirroringThreads() {
463 return numMirroringThreads;
464 }
465
466
467
468
469 @JsonProperty
470 public int maxNumFilesPerMirror() {
471 return maxNumFilesPerMirror;
472 }
473
474
475
476
477 @JsonProperty
478 public long maxNumBytesPerMirror() {
479 return maxNumBytesPerMirror;
480 }
481
482
483
484
485 @JsonProperty("replication")
486 public ReplicationConfig replicationConfig() {
487 return replicationConfig;
488 }
489
490
491
492
493
494 @JsonProperty
495 public boolean isCsrfTokenRequiredForThrift() {
496 return csrfTokenRequiredForThrift;
497 }
498
499
500
501
502 @JsonProperty
503 @Nullable
504 public String accessLogFormat() {
505 return accessLogFormat;
506 }
507
508
509
510
511 @Nullable
512 @JsonProperty("authentication")
513 public AuthConfig authConfig() {
514 return authConfig;
515 }
516
517
518
519
520 @Nullable
521 @JsonProperty("writeQuotaPerRepository")
522 public QuotaConfig writeQuotaPerRepository() {
523 return writeQuotaPerRepository;
524 }
525
526
527
528
529 @Nullable
530 @JsonProperty("cors")
531 public CorsConfig corsConfig() {
532 return corsConfig;
533 }
534
535 @Override
536 public String toString() {
537 try {
538 return Jackson.writeValueAsPrettyString(this);
539 } catch (JsonProcessingException e) {
540 throw new IllegalStateException(e);
541 }
542 }
543
544 Predicate<InetAddress> trustedProxyAddressPredicate() {
545 return trustedProxyAddressPredicate;
546 }
547
548 List<ClientAddressSource> clientAddressSourceList() {
549 return clientAddressSourceList;
550 }
551
552 private static Predicate<InetAddress> toTrustedProxyAddressPredicate(List<String> trustedProxyAddresses) {
553 final String first = trustedProxyAddresses.get(0);
554 Predicate<InetAddress> predicate = first.indexOf('/') < 0 ? ofExact(first) : ofCidr(first);
555 for (int i = 1; i < trustedProxyAddresses.size(); i++) {
556 final String next = trustedProxyAddresses.get(i);
557 predicate = predicate.or(next.indexOf('/') < 0 ? ofExact(next) : ofCidr(next));
558 }
559 return predicate;
560 }
561
562 private static List<ClientAddressSource> toClientAddressSourceList(
563 @Nullable List<String> clientAddressSources,
564 boolean useDefaultSources, boolean specifiedProxyProtocol) {
565 if (clientAddressSources != null && !clientAddressSources.isEmpty()) {
566 return clientAddressSources.stream().map(
567 name -> "PROXY_PROTOCOL".equals(name) ? ofProxyProtocol() : ofHeader(name))
568 .collect(toImmutableList());
569 }
570
571 if (useDefaultSources) {
572 final Builder<ClientAddressSource> builder = new Builder<>();
573 builder.add(ofHeader(HttpHeaderNames.FORWARDED));
574 builder.add(ofHeader(HttpHeaderNames.X_FORWARDED_FOR));
575 if (specifiedProxyProtocol) {
576 builder.add(ofProxyProtocol());
577 }
578 return builder.build();
579 }
580
581 return ImmutableList.of();
582 }
583
584 static final class ServerPortSerializer extends JsonSerializer<ServerPort> {
585 @Override
586 public void serialize(ServerPort value,
587 JsonGenerator gen, SerializerProvider serializers) throws IOException {
588
589 final InetSocketAddress localAddr = value.localAddress();
590 final int port = localAddr.getPort();
591 final String host;
592
593 if (localAddr.getAddress().isAnyLocalAddress()) {
594 host = "*";
595 } else {
596 final String hs = localAddr.getHostString();
597 if (NetUtil.isValidIpV6Address(hs)) {
598
599 host = NetUtil.toAddressString(localAddr.getAddress());
600 } else {
601 host = hs;
602 }
603 }
604
605 gen.writeStartObject();
606 gen.writeObjectFieldStart("localAddress");
607 gen.writeStringField("host", host);
608 gen.writeNumberField("port", port);
609 gen.writeEndObject();
610 gen.writeArrayFieldStart("protocols");
611 for (final SessionProtocol protocol : value.protocols()) {
612 gen.writeString(protocol.uriText());
613 }
614 gen.writeEndArray();
615 gen.writeEndObject();
616 }
617 }
618
619 static final class ServerPortDeserializer extends JsonDeserializer<ServerPort> {
620 @Override
621 public ServerPort deserialize(JsonParser p, DeserializationContext ctx) throws IOException {
622
623 final JsonNode root = p.getCodec().readTree(p);
624 final JsonNode localAddress = root.get("localAddress");
625 if (localAddress == null || localAddress.getNodeType() != JsonNodeType.OBJECT) {
626 return fail(ctx, root);
627 }
628
629 final JsonNode host = localAddress.get("host");
630 if (host == null || host.getNodeType() != JsonNodeType.STRING) {
631 return fail(ctx, root);
632 }
633
634 final JsonNode port = localAddress.get("port");
635 if (port == null || port.getNodeType() != JsonNodeType.NUMBER) {
636 return fail(ctx, root);
637 }
638
639 final ImmutableSet.Builder<SessionProtocol> protocolsBuilder = ImmutableSet.builder();
640 final JsonNode protocols = root.get("protocols");
641 if (protocols != null) {
642 if (protocols.getNodeType() != JsonNodeType.ARRAY) {
643 return fail(ctx, root);
644 }
645 protocols.elements().forEachRemaining(
646 protocol -> protocolsBuilder.add(SessionProtocol.of(protocol.textValue())));
647 } else {
648 final JsonNode protocol = root.get("protocol");
649 if (protocol == null || protocol.getNodeType() != JsonNodeType.STRING) {
650 return fail(ctx, root);
651 }
652 protocolsBuilder.add(SessionProtocol.of(protocol.textValue()));
653 }
654
655 final String hostVal = host.textValue();
656 final int portVal = port.intValue();
657
658 final InetSocketAddress localAddressVal;
659 if ("*".equals(hostVal)) {
660 localAddressVal = new InetSocketAddress(portVal);
661 } else {
662 localAddressVal = new InetSocketAddress(hostVal, portVal);
663 }
664
665 return new ServerPort(localAddressVal, protocolsBuilder.build());
666 }
667
668 private static ServerPort fail(DeserializationContext ctx, JsonNode root) throws JsonMappingException {
669 ctx.reportInputMismatch(ServerPort.class, "invalid server port information: %s", root);
670 throw new Error();
671 }
672 }
673
674 static final class OptionalConverter extends StdConverter<Optional<?>, Object> {
675 @Override
676 @Nullable
677 public Object convert(Optional<?> value) {
678 return value.orElse(null);
679 }
680 }
681 }