1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage;
18
19 import static com.google.common.base.Preconditions.checkState;
20 import static com.linecorp.centraldogma.internal.Util.PROJECT_AND_REPO_NAME_PATTERN;
21 import static java.util.Objects.requireNonNull;
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.nio.charset.StandardCharsets;
26 import java.nio.file.FileAlreadyExistsException;
27 import java.nio.file.Files;
28 import java.time.Instant;
29 import java.time.format.DateTimeFormatter;
30 import java.util.Arrays;
31 import java.util.Collections;
32 import java.util.LinkedHashMap;
33 import java.util.Map;
34 import java.util.concurrent.ConcurrentHashMap;
35 import java.util.concurrent.ConcurrentMap;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.ThreadLocalRandom;
38 import java.util.concurrent.atomic.AtomicBoolean;
39 import java.util.concurrent.atomic.AtomicReference;
40 import java.util.function.Supplier;
41
42 import org.jspecify.annotations.Nullable;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45
46 import com.google.common.collect.ImmutableMap;
47
48 import com.linecorp.centraldogma.common.Author;
49 import com.linecorp.centraldogma.common.CentralDogmaException;
50 import com.linecorp.centraldogma.internal.Util;
51 import com.linecorp.centraldogma.server.storage.StorageException;
52 import com.linecorp.centraldogma.server.storage.StorageManager;
53 import com.linecorp.centraldogma.server.storage.encryption.EncryptionStorageManager;
54
55 public abstract class DirectoryBasedStorageManager<T> implements StorageManager<T> {
56
57 private static final Logger logger = LoggerFactory.getLogger(DirectoryBasedStorageManager.class);
58
59 public static final String SUFFIX_REMOVED = ".removed";
60 public static final String SUFFIX_PURGED = ".purged";
61 private static final String GIT_EXTENSION = ".git";
62
63 private final String childTypeName;
64 private final File rootDir;
65 private final StorageRemovalManager storageRemovalManager = new StorageRemovalManager();
66 private final ConcurrentMap<String, T> children = new ConcurrentHashMap<>();
67 private final AtomicReference<Supplier<CentralDogmaException>> closed = new AtomicReference<>();
68 private final Executor purgeWorker;
69 private final EncryptionStorageManager encryptionStorageManager;
70 private boolean initialized;
71
72 protected DirectoryBasedStorageManager(File rootDir, Class<? extends T> childType,
73 Executor purgeWorker,
74 EncryptionStorageManager encryptionStorageManager) {
75 requireNonNull(rootDir, "rootDir");
76 this.purgeWorker = requireNonNull(purgeWorker, "purgeWorker");
77 this.encryptionStorageManager = requireNonNull(encryptionStorageManager, "encryptionStorageManager");
78
79 if (!rootDir.exists()) {
80 if (!rootDir.mkdirs()) {
81 throw new StorageException("failed to create root directory at " + rootDir);
82 }
83 }
84
85 try {
86 rootDir = rootDir.getCanonicalFile();
87 } catch (IOException e) {
88 throw new StorageException("failed to get the canonical path of: " + rootDir, e);
89 }
90
91 if (!rootDir.isDirectory()) {
92 throw new StorageException("not a directory: " + rootDir);
93 }
94
95 this.rootDir = rootDir;
96 childTypeName = Util.simpleTypeName(requireNonNull(childType, "childTypeName"), true);
97 }
98
99 protected Executor purgeWorker() {
100 return purgeWorker;
101 }
102
103 protected EncryptionStorageManager encryptionStorageManager() {
104 return encryptionStorageManager;
105 }
106
107
108
109
110 protected final void init() {
111 checkState(!initialized, "initialized already");
112 Throwable cause = null;
113 try {
114 final File[] childFiles = rootDir.listFiles();
115 if (childFiles != null) {
116 for (File f : childFiles) {
117 loadChild(f);
118 }
119 }
120 initialized = true;
121 } catch (Throwable t) {
122 cause = t;
123 }
124
125 if (cause != null) {
126 final CentralDogmaException finalCause;
127 if (cause instanceof CentralDogmaException) {
128 finalCause = (CentralDogmaException) cause;
129 } else {
130 finalCause = new CentralDogmaException("Failed to load a child: " + cause, cause);
131 }
132 close(() -> finalCause);
133 throw finalCause;
134 }
135 }
136
137 @Nullable
138 private T loadChild(File f) {
139 final String name = f.getName();
140 if (!isValidChildName(name)) {
141 return null;
142 }
143
144 if (!f.isDirectory()) {
145 return null;
146 }
147
148 if (new File(f + SUFFIX_REMOVED).exists()) {
149 return null;
150 }
151
152 try {
153 final T child = openChild(f);
154 children.put(name, child);
155 return child;
156 } catch (RuntimeException e) {
157 throw e;
158 } catch (Exception e) {
159 throw new StorageException("failed to open " + childTypeName + ": " + f, e);
160 }
161 }
162
163 protected boolean replaceChild(String name, T oldChild, T newChild) {
164 return children.replace(name, oldChild, newChild);
165 }
166
167 protected abstract T openChild(File childDir) throws Exception;
168
169 protected abstract T createChild(
170 File childDir, Author author, long creationTimeMillis, boolean encrypt) throws Exception;
171
172 private void closeChild(String name, T child, Supplier<CentralDogmaException> failureCauseSupplier) {
173 closeChild(new File(rootDir, name), child, failureCauseSupplier);
174 }
175
176 protected void closeChild(File childDir, T child, Supplier<CentralDogmaException> failureCauseSupplier) {}
177
178 protected abstract CentralDogmaException newStorageExistsException(String name);
179
180 protected abstract CentralDogmaException newStorageNotFoundException(String name);
181
182 protected abstract void deletePurged(File file);
183
184 @Override
185 public void close(Supplier<CentralDogmaException> failureCauseSupplier) {
186 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
187 if (!closed.compareAndSet(null, failureCauseSupplier)) {
188 return;
189 }
190
191
192 for (Map.Entry<String, T> e : children.entrySet()) {
193 closeChild(e.getKey(), e.getValue(), failureCauseSupplier);
194 }
195 }
196
197 @Override
198 public boolean exists(String name) {
199 ensureOpen();
200 return children.containsKey(validateChildName(name));
201 }
202
203 @Override
204 public T get(String name) {
205 ensureOpen();
206 final T child = children.get(validateChildName(name));
207 if (child == null) {
208 throw newStorageNotFoundException(name);
209 }
210
211 return child;
212 }
213
214 @Override
215 public T create(String name, long creationTimeMillis, Author author, boolean encrypt) {
216 ensureOpen();
217 requireNonNull(author, "author");
218 validateChildName(name);
219
220 final AtomicBoolean created = new AtomicBoolean();
221 final T child = children.computeIfAbsent(name, n -> {
222 final T c = create0(author, n, creationTimeMillis, encrypt);
223 created.set(true);
224 return c;
225 });
226
227 if (created.get()) {
228 return child;
229 } else {
230 throw newStorageExistsException(name);
231 }
232 }
233
234 private T create0(Author author, String name, long creationTimeMillis, boolean encrypt) {
235 if (new File(rootDir, name + SUFFIX_REMOVED).exists()) {
236 throw newStorageExistsException(name + " (removed)");
237 }
238
239 final File f = new File(rootDir, name);
240 boolean success = false;
241 try {
242 final T newChild = createChild(f, author, creationTimeMillis, encrypt);
243 success = true;
244 return newChild;
245 } catch (RuntimeException e) {
246 throw e;
247 } catch (Exception e) {
248 throw new StorageException("failed to create a new " + childTypeName + ": " + f, e);
249 } finally {
250 if (!success && f.exists()) {
251
252 try {
253 Util.deleteFileTree(f);
254 } catch (IOException e) {
255 logger.warn("Failed to delete a partially created project: {}", f, e);
256 }
257 }
258 }
259 }
260
261 @Override
262 public Map<String, T> list() {
263 ensureOpen();
264
265 final int estimatedSize = children.size();
266 final String[] names = children.keySet().toArray(new String[estimatedSize]);
267 Arrays.sort(names);
268
269 final Map<String, T> ret = new LinkedHashMap<>(estimatedSize);
270 for (String k : names) {
271 final T v = children.get(k);
272 if (v != null) {
273 ret.put(k, v);
274 }
275 }
276
277 return Collections.unmodifiableMap(ret);
278 }
279
280 @Override
281 public Map<String, Instant> listRemoved() {
282 ensureOpen();
283 final File[] files = rootDir.listFiles();
284 if (files == null) {
285 return ImmutableMap.of();
286 }
287
288 Arrays.sort(files);
289 final ImmutableMap.Builder<String, Instant> builder = ImmutableMap.builder();
290
291 for (File f : files) {
292 if (!f.isDirectory()) {
293 continue;
294 }
295
296 String name = f.getName();
297 if (!name.endsWith(SUFFIX_REMOVED)) {
298 continue;
299 }
300
301 name = name.substring(0, name.length() - SUFFIX_REMOVED.length());
302 if (!isValidChildName(name) || children.containsKey(name)) {
303 continue;
304 }
305
306 builder.put(name, storageRemovalManager.readRemoval(f));
307 }
308
309 return builder.build();
310 }
311
312 @Override
313 public void remove(String name) {
314 ensureOpen();
315 final T child = children.remove(validateChildName(name));
316 if (child == null) {
317 throw newStorageNotFoundException(name);
318 }
319
320 closeChild(name, child, () -> newStorageNotFoundException(name));
321
322 final File file = new File(rootDir, name);
323 storageRemovalManager.mark(file);
324 if (!file.renameTo(new File(rootDir, name + SUFFIX_REMOVED))) {
325 throw new StorageException("failed to mark " + childTypeName + " as removed: " + name);
326 }
327 }
328
329 @Override
330 public T unremove(String name) {
331 ensureOpen();
332 validateChildName(name);
333
334 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
335 if (!removed.isDirectory()) {
336 throw newStorageNotFoundException(name);
337 }
338
339 final File unremoved = new File(rootDir, name);
340
341 if (!removed.renameTo(unremoved)) {
342 throw new StorageException("failed to mark " + childTypeName + " as unremoved: " + name);
343 }
344 storageRemovalManager.unmark(unremoved);
345
346 final T unremovedChild = loadChild(unremoved);
347 if (unremovedChild == null) {
348 throw newStorageNotFoundException(name);
349 }
350 return unremovedChild;
351 }
352
353 @Override
354 public void markForPurge(String name) {
355 ensureOpen();
356 validateChildName(name);
357 File marked;
358 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
359
360 final Supplier<File> newMarkedFile = () -> {
361 final String interfix = '.' + Long.toHexString(ThreadLocalRandom.current().nextLong());
362 return new File(rootDir, name + interfix + SUFFIX_PURGED);
363 };
364
365 synchronized (this) {
366 if (!removed.exists()) {
367 logger.warn("Tried to purge {}, but it's non-existent.", removed);
368 return;
369 }
370
371 if (!removed.isDirectory()) {
372 throw new StorageException("not a directory: " + removed);
373 }
374
375 marked = newMarkedFile.get();
376 boolean moved = false;
377 while (!moved) {
378 try {
379 Files.move(removed.toPath(), marked.toPath());
380 moved = true;
381 } catch (FileAlreadyExistsException e) {
382 marked = newMarkedFile.get();
383 } catch (IOException e) {
384 throw new StorageException("failed to mark " + childTypeName + " for purge: " + removed, e);
385 }
386 }
387 }
388 final File purged = marked;
389 try {
390 purgeWorker().execute(() -> deletePurged(purged));
391 } catch (Exception e) {
392 logger.warn("Failed to schedule a purge task for {}:", purged, e);
393 }
394 }
395
396 @Override
397 public void purgeMarked() {
398 ensureOpen();
399 final File[] files = rootDir.listFiles();
400 if (files == null) {
401 return;
402 }
403 for (File f : files) {
404 if (!f.isDirectory()) {
405 continue;
406 }
407 final String name = f.getName();
408 if (name.endsWith(SUFFIX_PURGED)) {
409 deletePurged(f);
410 }
411 }
412 }
413
414 @Override
415 public void ensureOpen() {
416 checkState(initialized, "not initialized yet");
417 if (closed.get() != null) {
418 throw closed.get().get();
419 }
420 }
421
422 private String validateChildName(String name) {
423 if (!isValidChildName(requireNonNull(name, "name"))) {
424 throw new IllegalArgumentException("invalid " + childTypeName + " name: " + name);
425 }
426 return name;
427 }
428
429 private static boolean isValidChildName(String name) {
430 if (name == null) {
431 return false;
432 }
433
434 if (!PROJECT_AND_REPO_NAME_PATTERN.matcher(name).matches()) {
435 return false;
436 }
437
438 return !name.endsWith(SUFFIX_REMOVED) && !name.endsWith(SUFFIX_PURGED) &&
439 !name.endsWith(GIT_EXTENSION);
440 }
441
442 @Override
443 public String toString() {
444 return Util.simpleTypeName(getClass()) + '(' + rootDir + ')';
445 }
446
447 private final class StorageRemovalManager {
448
449 private static final String REMOVAL_TIMESTAMP_NAME = "removal.timestamp";
450
451 void mark(File file) {
452 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
453 final String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
454 try {
455 Files.write(removal.toPath(), timestamp.getBytes(StandardCharsets.UTF_8));
456 } catch (IOException e) {
457 throw new StorageException(
458 "failed to write a removal timestamp for " + childTypeName + ": " + removal);
459 }
460 }
461
462 void unmark(File file) {
463 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
464 if (removal.exists()) {
465 if (!removal.delete()) {
466 logger.warn("Failed to delete a removal timestamp for {}: {}", childTypeName, removal);
467 }
468 }
469 }
470
471 Instant readRemoval(File file) {
472 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
473 if (!removal.exists()) {
474 return Instant.ofEpochMilli(file.lastModified());
475 }
476 try {
477 final String timestamp = new String(Files.readAllBytes(removal.toPath()),
478 StandardCharsets.UTF_8);
479 return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(timestamp));
480 } catch (Exception e) {
481 logger.warn("Failed to read a removal timestamp for {}: {}", childTypeName, removal, e);
482 return Instant.ofEpochMilli(file.lastModified());
483 }
484 }
485 }
486 }