1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage;
18
19 import static com.google.common.base.Preconditions.checkState;
20 import static com.linecorp.centraldogma.internal.Util.PROJECT_AND_REPO_NAME_PATTERN;
21 import static java.util.Objects.requireNonNull;
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.nio.charset.StandardCharsets;
26 import java.nio.file.FileAlreadyExistsException;
27 import java.nio.file.Files;
28 import java.time.Instant;
29 import java.time.format.DateTimeFormatter;
30 import java.util.Arrays;
31 import java.util.Collections;
32 import java.util.LinkedHashMap;
33 import java.util.Map;
34 import java.util.concurrent.ConcurrentHashMap;
35 import java.util.concurrent.ConcurrentMap;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.ThreadLocalRandom;
38 import java.util.concurrent.atomic.AtomicBoolean;
39 import java.util.concurrent.atomic.AtomicReference;
40 import java.util.function.Supplier;
41
42 import javax.annotation.Nullable;
43
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47 import com.google.common.collect.ImmutableMap;
48
49 import com.linecorp.centraldogma.common.Author;
50 import com.linecorp.centraldogma.common.CentralDogmaException;
51 import com.linecorp.centraldogma.internal.Util;
52 import com.linecorp.centraldogma.server.storage.StorageException;
53 import com.linecorp.centraldogma.server.storage.StorageManager;
54 import com.linecorp.centraldogma.server.storage.encryption.EncryptionStorageManager;
55
56 public abstract class DirectoryBasedStorageManager<T> implements StorageManager<T> {
57
58 private static final Logger logger = LoggerFactory.getLogger(DirectoryBasedStorageManager.class);
59
60 public static final String SUFFIX_REMOVED = ".removed";
61 public static final String SUFFIX_PURGED = ".purged";
62 private static final String GIT_EXTENSION = ".git";
63
64 private final String childTypeName;
65 private final File rootDir;
66 private final StorageRemovalManager storageRemovalManager = new StorageRemovalManager();
67 private final ConcurrentMap<String, T> children = new ConcurrentHashMap<>();
68 private final AtomicReference<Supplier<CentralDogmaException>> closed = new AtomicReference<>();
69 private final Executor purgeWorker;
70 private final EncryptionStorageManager encryptionStorageManager;
71 private boolean initialized;
72
73 protected DirectoryBasedStorageManager(File rootDir, Class<? extends T> childType,
74 Executor purgeWorker,
75 EncryptionStorageManager encryptionStorageManager) {
76 requireNonNull(rootDir, "rootDir");
77 this.purgeWorker = requireNonNull(purgeWorker, "purgeWorker");
78 this.encryptionStorageManager = requireNonNull(encryptionStorageManager, "encryptionStorageManager");
79
80 if (!rootDir.exists()) {
81 if (!rootDir.mkdirs()) {
82 throw new StorageException("failed to create root directory at " + rootDir);
83 }
84 }
85
86 try {
87 rootDir = rootDir.getCanonicalFile();
88 } catch (IOException e) {
89 throw new StorageException("failed to get the canonical path of: " + rootDir, e);
90 }
91
92 if (!rootDir.isDirectory()) {
93 throw new StorageException("not a directory: " + rootDir);
94 }
95
96 this.rootDir = rootDir;
97 childTypeName = Util.simpleTypeName(requireNonNull(childType, "childTypeName"), true);
98 }
99
100 protected Executor purgeWorker() {
101 return purgeWorker;
102 }
103
104 protected EncryptionStorageManager encryptionStorageManager() {
105 return encryptionStorageManager;
106 }
107
108
109
110
111 protected final void init() {
112 checkState(!initialized, "initialized already");
113 Throwable cause = null;
114 try {
115 final File[] childFiles = rootDir.listFiles();
116 if (childFiles != null) {
117 for (File f : childFiles) {
118 loadChild(f);
119 }
120 }
121 initialized = true;
122 } catch (Throwable t) {
123 cause = t;
124 }
125
126 if (cause != null) {
127 final CentralDogmaException finalCause;
128 if (cause instanceof CentralDogmaException) {
129 finalCause = (CentralDogmaException) cause;
130 } else {
131 finalCause = new CentralDogmaException("Failed to load a child: " + cause, cause);
132 }
133 close(() -> finalCause);
134 throw finalCause;
135 }
136 }
137
138 @Nullable
139 private T loadChild(File f) {
140 final String name = f.getName();
141 if (!isValidChildName(name)) {
142 return null;
143 }
144
145 if (!f.isDirectory()) {
146 return null;
147 }
148
149 if (new File(f + SUFFIX_REMOVED).exists()) {
150 return null;
151 }
152
153 try {
154 final T child = openChild(f);
155 children.put(name, child);
156 return child;
157 } catch (RuntimeException e) {
158 throw e;
159 } catch (Exception e) {
160 throw new StorageException("failed to open " + childTypeName + ": " + f, e);
161 }
162 }
163
164 protected boolean replaceChild(String name, T oldChild, T newChild) {
165 return children.replace(name, oldChild, newChild);
166 }
167
168 protected abstract T openChild(File childDir) throws Exception;
169
170 protected abstract T createChild(
171 File childDir, Author author, long creationTimeMillis, boolean encrypt) throws Exception;
172
173 private void closeChild(String name, T child, Supplier<CentralDogmaException> failureCauseSupplier) {
174 closeChild(new File(rootDir, name), child, failureCauseSupplier);
175 }
176
177 protected void closeChild(File childDir, T child, Supplier<CentralDogmaException> failureCauseSupplier) {}
178
179 protected abstract CentralDogmaException newStorageExistsException(String name);
180
181 protected abstract CentralDogmaException newStorageNotFoundException(String name);
182
183 protected abstract void deletePurged(File file);
184
185 @Override
186 public void close(Supplier<CentralDogmaException> failureCauseSupplier) {
187 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
188 if (!closed.compareAndSet(null, failureCauseSupplier)) {
189 return;
190 }
191
192
193 for (Map.Entry<String, T> e : children.entrySet()) {
194 closeChild(e.getKey(), e.getValue(), failureCauseSupplier);
195 }
196 }
197
198 @Override
199 public boolean exists(String name) {
200 ensureOpen();
201 return children.containsKey(validateChildName(name));
202 }
203
204 @Override
205 public T get(String name) {
206 ensureOpen();
207 final T child = children.get(validateChildName(name));
208 if (child == null) {
209 throw newStorageNotFoundException(name);
210 }
211
212 return child;
213 }
214
215 @Override
216 public T create(String name, long creationTimeMillis, Author author, boolean encrypt) {
217 ensureOpen();
218 requireNonNull(author, "author");
219 validateChildName(name);
220
221 final AtomicBoolean created = new AtomicBoolean();
222 final T child = children.computeIfAbsent(name, n -> {
223 final T c = create0(author, n, creationTimeMillis, encrypt);
224 created.set(true);
225 return c;
226 });
227
228 if (created.get()) {
229 return child;
230 } else {
231 throw newStorageExistsException(name);
232 }
233 }
234
235 private T create0(Author author, String name, long creationTimeMillis, boolean encrypt) {
236 if (new File(rootDir, name + SUFFIX_REMOVED).exists()) {
237 throw newStorageExistsException(name + " (removed)");
238 }
239
240 final File f = new File(rootDir, name);
241 boolean success = false;
242 try {
243 final T newChild = createChild(f, author, creationTimeMillis, encrypt);
244 success = true;
245 return newChild;
246 } catch (RuntimeException e) {
247 throw e;
248 } catch (Exception e) {
249 throw new StorageException("failed to create a new " + childTypeName + ": " + f, e);
250 } finally {
251 if (!success && f.exists()) {
252
253 try {
254 Util.deleteFileTree(f);
255 } catch (IOException e) {
256 logger.warn("Failed to delete a partially created project: {}", f, e);
257 }
258 }
259 }
260 }
261
262 @Override
263 public Map<String, T> list() {
264 ensureOpen();
265
266 final int estimatedSize = children.size();
267 final String[] names = children.keySet().toArray(new String[estimatedSize]);
268 Arrays.sort(names);
269
270 final Map<String, T> ret = new LinkedHashMap<>(estimatedSize);
271 for (String k : names) {
272 final T v = children.get(k);
273 if (v != null) {
274 ret.put(k, v);
275 }
276 }
277
278 return Collections.unmodifiableMap(ret);
279 }
280
281 @Override
282 public Map<String, Instant> listRemoved() {
283 ensureOpen();
284 final File[] files = rootDir.listFiles();
285 if (files == null) {
286 return ImmutableMap.of();
287 }
288
289 Arrays.sort(files);
290 final ImmutableMap.Builder<String, Instant> builder = ImmutableMap.builder();
291
292 for (File f : files) {
293 if (!f.isDirectory()) {
294 continue;
295 }
296
297 String name = f.getName();
298 if (!name.endsWith(SUFFIX_REMOVED)) {
299 continue;
300 }
301
302 name = name.substring(0, name.length() - SUFFIX_REMOVED.length());
303 if (!isValidChildName(name) || children.containsKey(name)) {
304 continue;
305 }
306
307 builder.put(name, storageRemovalManager.readRemoval(f));
308 }
309
310 return builder.build();
311 }
312
313 @Override
314 public void remove(String name) {
315 ensureOpen();
316 final T child = children.remove(validateChildName(name));
317 if (child == null) {
318 throw newStorageNotFoundException(name);
319 }
320
321 closeChild(name, child, () -> newStorageNotFoundException(name));
322
323 final File file = new File(rootDir, name);
324 storageRemovalManager.mark(file);
325 if (!file.renameTo(new File(rootDir, name + SUFFIX_REMOVED))) {
326 throw new StorageException("failed to mark " + childTypeName + " as removed: " + name);
327 }
328 }
329
330 @Override
331 public T unremove(String name) {
332 ensureOpen();
333 validateChildName(name);
334
335 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
336 if (!removed.isDirectory()) {
337 throw newStorageNotFoundException(name);
338 }
339
340 final File unremoved = new File(rootDir, name);
341
342 if (!removed.renameTo(unremoved)) {
343 throw new StorageException("failed to mark " + childTypeName + " as unremoved: " + name);
344 }
345 storageRemovalManager.unmark(unremoved);
346
347 final T unremovedChild = loadChild(unremoved);
348 if (unremovedChild == null) {
349 throw newStorageNotFoundException(name);
350 }
351 return unremovedChild;
352 }
353
354 @Override
355 public void markForPurge(String name) {
356 ensureOpen();
357 validateChildName(name);
358 File marked;
359 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
360
361 final Supplier<File> newMarkedFile = () -> {
362 final String interfix = '.' + Long.toHexString(ThreadLocalRandom.current().nextLong());
363 return new File(rootDir, name + interfix + SUFFIX_PURGED);
364 };
365
366 synchronized (this) {
367 if (!removed.exists()) {
368 logger.warn("Tried to purge {}, but it's non-existent.", removed);
369 return;
370 }
371
372 if (!removed.isDirectory()) {
373 throw new StorageException("not a directory: " + removed);
374 }
375
376 marked = newMarkedFile.get();
377 boolean moved = false;
378 while (!moved) {
379 try {
380 Files.move(removed.toPath(), marked.toPath());
381 moved = true;
382 } catch (FileAlreadyExistsException e) {
383 marked = newMarkedFile.get();
384 } catch (IOException e) {
385 throw new StorageException("failed to mark " + childTypeName + " for purge: " + removed, e);
386 }
387 }
388 }
389 final File purged = marked;
390 try {
391 purgeWorker().execute(() -> deletePurged(purged));
392 } catch (Exception e) {
393 logger.warn("Failed to schedule a purge task for {}:", purged, e);
394 }
395 }
396
397 @Override
398 public void purgeMarked() {
399 ensureOpen();
400 final File[] files = rootDir.listFiles();
401 if (files == null) {
402 return;
403 }
404 for (File f : files) {
405 if (!f.isDirectory()) {
406 continue;
407 }
408 final String name = f.getName();
409 if (name.endsWith(SUFFIX_PURGED)) {
410 deletePurged(f);
411 }
412 }
413 }
414
415 @Override
416 public void ensureOpen() {
417 checkState(initialized, "not initialized yet");
418 if (closed.get() != null) {
419 throw closed.get().get();
420 }
421 }
422
423 private String validateChildName(String name) {
424 if (!isValidChildName(requireNonNull(name, "name"))) {
425 throw new IllegalArgumentException("invalid " + childTypeName + " name: " + name);
426 }
427 return name;
428 }
429
430 private static boolean isValidChildName(String name) {
431 if (name == null) {
432 return false;
433 }
434
435 if (!PROJECT_AND_REPO_NAME_PATTERN.matcher(name).matches()) {
436 return false;
437 }
438
439 return !name.endsWith(SUFFIX_REMOVED) && !name.endsWith(SUFFIX_PURGED) &&
440 !name.endsWith(GIT_EXTENSION);
441 }
442
443 @Override
444 public String toString() {
445 return Util.simpleTypeName(getClass()) + '(' + rootDir + ')';
446 }
447
448 private final class StorageRemovalManager {
449
450 private static final String REMOVAL_TIMESTAMP_NAME = "removal.timestamp";
451
452 void mark(File file) {
453 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
454 final String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
455 try {
456 Files.write(removal.toPath(), timestamp.getBytes(StandardCharsets.UTF_8));
457 } catch (IOException e) {
458 throw new StorageException(
459 "failed to write a removal timestamp for " + childTypeName + ": " + removal);
460 }
461 }
462
463 void unmark(File file) {
464 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
465 if (removal.exists()) {
466 if (!removal.delete()) {
467 logger.warn("Failed to delete a removal timestamp for {}: {}", childTypeName, removal);
468 }
469 }
470 }
471
472 Instant readRemoval(File file) {
473 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
474 if (!removal.exists()) {
475 return Instant.ofEpochMilli(file.lastModified());
476 }
477 try {
478 final String timestamp = new String(Files.readAllBytes(removal.toPath()),
479 StandardCharsets.UTF_8);
480 return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(timestamp));
481 } catch (Exception e) {
482 logger.warn("Failed to read a removal timestamp for {}: {}", childTypeName, removal, e);
483 return Instant.ofEpochMilli(file.lastModified());
484 }
485 }
486 }
487 }