1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage;
18
19 import static com.google.common.base.Preconditions.checkState;
20 import static com.linecorp.centraldogma.internal.Util.PROJECT_AND_REPO_NAME_PATTERN;
21 import static java.util.Objects.requireNonNull;
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.nio.charset.StandardCharsets;
26 import java.nio.file.FileAlreadyExistsException;
27 import java.nio.file.Files;
28 import java.time.Instant;
29 import java.time.format.DateTimeFormatter;
30 import java.util.Arrays;
31 import java.util.Collections;
32 import java.util.LinkedHashMap;
33 import java.util.Map;
34 import java.util.concurrent.ConcurrentHashMap;
35 import java.util.concurrent.ConcurrentMap;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.ThreadLocalRandom;
38 import java.util.concurrent.atomic.AtomicBoolean;
39 import java.util.concurrent.atomic.AtomicReference;
40 import java.util.function.Supplier;
41
42 import javax.annotation.Nullable;
43
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47 import com.google.common.collect.ImmutableMap;
48
49 import com.linecorp.centraldogma.common.Author;
50 import com.linecorp.centraldogma.common.CentralDogmaException;
51 import com.linecorp.centraldogma.internal.Util;
52 import com.linecorp.centraldogma.server.storage.StorageException;
53 import com.linecorp.centraldogma.server.storage.StorageManager;
54
55 public abstract class DirectoryBasedStorageManager<T> implements StorageManager<T> {
56
57 private static final Logger logger = LoggerFactory.getLogger(DirectoryBasedStorageManager.class);
58
59 private static final String SUFFIX_REMOVED = ".removed";
60 private static final String SUFFIX_PURGED = ".purged";
61 private static final String GIT_EXTENSION = ".git";
62
63 private final String childTypeName;
64 private final File rootDir;
65 private final StorageRemovalManager storageRemovalManager = new StorageRemovalManager();
66 private final ConcurrentMap<String, T> children = new ConcurrentHashMap<>();
67 private final AtomicReference<Supplier<CentralDogmaException>> closed = new AtomicReference<>();
68 private final Executor purgeWorker;
69 private boolean initialized;
70
71 protected DirectoryBasedStorageManager(File rootDir, Class<? extends T> childType,
72 Executor purgeWorker) {
73 requireNonNull(rootDir, "rootDir");
74 this.purgeWorker = requireNonNull(purgeWorker, "purgeWorker");
75
76 if (!rootDir.exists()) {
77 if (!rootDir.mkdirs()) {
78 throw new StorageException("failed to create root directory at " + rootDir);
79 }
80 }
81
82 try {
83 rootDir = rootDir.getCanonicalFile();
84 } catch (IOException e) {
85 throw new StorageException("failed to get the canonical path of: " + rootDir, e);
86 }
87
88 if (!rootDir.isDirectory()) {
89 throw new StorageException("not a directory: " + rootDir);
90 }
91
92 this.rootDir = rootDir;
93 childTypeName = Util.simpleTypeName(requireNonNull(childType, "childTypeName"), true);
94 }
95
96 protected Executor purgeWorker() {
97 return purgeWorker;
98 }
99
100
101
102
103 protected final void init() {
104 checkState(!initialized, "initialized already");
105 Throwable cause = null;
106 try {
107 final File[] childFiles = rootDir.listFiles();
108 if (childFiles != null) {
109 for (File f : childFiles) {
110 loadChild(f);
111 }
112 }
113 initialized = true;
114 } catch (Throwable t) {
115 cause = t;
116 }
117
118 if (cause != null) {
119 final CentralDogmaException finalCause;
120 if (cause instanceof CentralDogmaException) {
121 finalCause = (CentralDogmaException) cause;
122 } else {
123 finalCause = new CentralDogmaException("Failed to load a child: " + cause, cause);
124 }
125 close(() -> finalCause);
126 throw finalCause;
127 }
128 }
129
130 @Nullable
131 private T loadChild(File f) {
132 final String name = f.getName();
133 if (!isValidChildName(name)) {
134 return null;
135 }
136
137 if (!f.isDirectory()) {
138 return null;
139 }
140
141 if (new File(f + SUFFIX_REMOVED).exists()) {
142 return null;
143 }
144
145 try {
146 final T child = openChild(f);
147 children.put(name, child);
148 return child;
149 } catch (RuntimeException e) {
150 throw e;
151 } catch (Exception e) {
152 throw new StorageException("failed to open " + childTypeName + ": " + f, e);
153 }
154 }
155
156 protected abstract T openChild(File childDir) throws Exception;
157
158 protected abstract T createChild(File childDir, Author author, long creationTimeMillis) throws Exception;
159
160 private void closeChild(String name, T child, Supplier<CentralDogmaException> failureCauseSupplier) {
161 closeChild(new File(rootDir, name), child, failureCauseSupplier);
162 }
163
164 protected void closeChild(File childDir, T child, Supplier<CentralDogmaException> failureCauseSupplier) {}
165
166 protected abstract CentralDogmaException newStorageExistsException(String name);
167
168 protected abstract CentralDogmaException newStorageNotFoundException(String name);
169
170 @Override
171 public void close(Supplier<CentralDogmaException> failureCauseSupplier) {
172 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
173 if (!closed.compareAndSet(null, failureCauseSupplier)) {
174 return;
175 }
176
177
178 for (Map.Entry<String, T> e : children.entrySet()) {
179 closeChild(e.getKey(), e.getValue(), failureCauseSupplier);
180 }
181 }
182
183 @Override
184 public boolean exists(String name) {
185 ensureOpen();
186 return children.containsKey(validateChildName(name));
187 }
188
189 @Override
190 public T get(String name) {
191 ensureOpen();
192 final T child = children.get(validateChildName(name));
193 if (child == null) {
194 throw newStorageNotFoundException(name);
195 }
196
197 return child;
198 }
199
200 @Override
201 public T create(String name, long creationTimeMillis, Author author) {
202 ensureOpen();
203 requireNonNull(author, "author");
204 validateChildName(name);
205
206 final AtomicBoolean created = new AtomicBoolean();
207 final T child = children.computeIfAbsent(name, n -> {
208 final T c = create0(author, n, creationTimeMillis);
209 created.set(true);
210 return c;
211 });
212
213 if (created.get()) {
214 return child;
215 } else {
216 throw newStorageExistsException(name);
217 }
218 }
219
220 private T create0(Author author, String name, long creationTimeMillis) {
221 if (new File(rootDir, name + SUFFIX_REMOVED).exists()) {
222 throw newStorageExistsException(name + " (removed)");
223 }
224
225 final File f = new File(rootDir, name);
226 boolean success = false;
227 try {
228 final T newChild = createChild(f, author, creationTimeMillis);
229 success = true;
230 return newChild;
231 } catch (RuntimeException e) {
232 throw e;
233 } catch (Exception e) {
234 throw new StorageException("failed to create a new " + childTypeName + ": " + f, e);
235 } finally {
236 if (!success && f.exists()) {
237
238 try {
239 Util.deleteFileTree(f);
240 } catch (IOException e) {
241 logger.warn("Failed to delete a partially created project: {}", f, e);
242 }
243 }
244 }
245 }
246
247 @Override
248 public Map<String, T> list() {
249 ensureOpen();
250
251 final int estimatedSize = children.size();
252 final String[] names = children.keySet().toArray(new String[estimatedSize]);
253 Arrays.sort(names);
254
255 final Map<String, T> ret = new LinkedHashMap<>(estimatedSize);
256 for (String k : names) {
257 final T v = children.get(k);
258 if (v != null) {
259 ret.put(k, v);
260 }
261 }
262
263 return Collections.unmodifiableMap(ret);
264 }
265
266 @Override
267 public Map<String, Instant> listRemoved() {
268 ensureOpen();
269 final File[] files = rootDir.listFiles();
270 if (files == null) {
271 return ImmutableMap.of();
272 }
273
274 Arrays.sort(files);
275 final ImmutableMap.Builder<String, Instant> builder = ImmutableMap.builder();
276
277 for (File f : files) {
278 if (!f.isDirectory()) {
279 continue;
280 }
281
282 String name = f.getName();
283 if (!name.endsWith(SUFFIX_REMOVED)) {
284 continue;
285 }
286
287 name = name.substring(0, name.length() - SUFFIX_REMOVED.length());
288 if (!isValidChildName(name) || children.containsKey(name)) {
289 continue;
290 }
291
292 builder.put(name, storageRemovalManager.readRemoval(f));
293 }
294
295 return builder.build();
296 }
297
298 @Override
299 public void remove(String name) {
300 ensureOpen();
301 final T child = children.remove(validateChildName(name));
302 if (child == null) {
303 throw newStorageNotFoundException(name);
304 }
305
306 closeChild(name, child, () -> newStorageNotFoundException(name));
307
308 final File file = new File(rootDir, name);
309 storageRemovalManager.mark(file);
310 if (!file.renameTo(new File(rootDir, name + SUFFIX_REMOVED))) {
311 throw new StorageException("failed to mark " + childTypeName + " as removed: " + name);
312 }
313 }
314
315 @Override
316 public T unremove(String name) {
317 ensureOpen();
318 validateChildName(name);
319
320 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
321 if (!removed.isDirectory()) {
322 throw newStorageNotFoundException(name);
323 }
324
325 final File unremoved = new File(rootDir, name);
326
327 if (!removed.renameTo(unremoved)) {
328 throw new StorageException("failed to mark " + childTypeName + " as unremoved: " + name);
329 }
330 storageRemovalManager.unmark(unremoved);
331
332 final T unremovedChild = loadChild(unremoved);
333 if (unremovedChild == null) {
334 throw newStorageNotFoundException(name);
335 }
336 return unremovedChild;
337 }
338
339 @Override
340 public void markForPurge(String name) {
341 ensureOpen();
342 validateChildName(name);
343 File marked;
344 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
345
346 final Supplier<File> newMarkedFile = () -> {
347 final String interfix = '.' + Long.toHexString(ThreadLocalRandom.current().nextLong());
348 return new File(rootDir, name + interfix + SUFFIX_PURGED);
349 };
350
351 synchronized (this) {
352 if (!removed.exists()) {
353 logger.warn("Tried to purge {}, but it's non-existent.", removed);
354 return;
355 }
356
357 if (!removed.isDirectory()) {
358 throw new StorageException("not a directory: " + removed);
359 }
360
361 marked = newMarkedFile.get();
362 boolean moved = false;
363 while (!moved) {
364 try {
365 Files.move(removed.toPath(), marked.toPath());
366 moved = true;
367 } catch (FileAlreadyExistsException e) {
368 marked = newMarkedFile.get();
369 } catch (IOException e) {
370 throw new StorageException("failed to mark " + childTypeName + " for purge: " + removed, e);
371 }
372 }
373 }
374 final File purged = marked;
375 try {
376 purgeWorker.execute(() -> deletePurgedFile(purged));
377 } catch (Exception e) {
378 logger.warn("Failed to schedule a purge task for {}:", purged, e);
379 }
380 }
381
382 @Override
383 public void purgeMarked() {
384 ensureOpen();
385 final File[] files = rootDir.listFiles();
386 if (files == null) {
387 return;
388 }
389 for (File f : files) {
390 if (!f.isDirectory()) {
391 continue;
392 }
393 final String name = f.getName();
394 if (name.endsWith(SUFFIX_PURGED)) {
395 deletePurgedFile(f);
396 }
397 }
398 }
399
400 private void deletePurgedFile(File file) {
401 try {
402 logger.info("Deleting a purged {}: {} ..", childTypeName, file);
403 Util.deleteFileTree(file);
404 logger.info("Deleted a purged {}: {}.", childTypeName, file);
405 } catch (IOException e) {
406 logger.warn("Failed to delete a purged {}: {}", childTypeName, file, e);
407 }
408 }
409
410 @Override
411 public void ensureOpen() {
412 checkState(initialized, "not initialized yet");
413 if (closed.get() != null) {
414 throw closed.get().get();
415 }
416 }
417
418 private String validateChildName(String name) {
419 if (!isValidChildName(requireNonNull(name, "name"))) {
420 throw new IllegalArgumentException("invalid " + childTypeName + " name: " + name);
421 }
422 return name;
423 }
424
425 private static boolean isValidChildName(String name) {
426 if (name == null) {
427 return false;
428 }
429
430 if (!PROJECT_AND_REPO_NAME_PATTERN.matcher(name).matches()) {
431 return false;
432 }
433
434 return !name.endsWith(SUFFIX_REMOVED) && !name.endsWith(SUFFIX_PURGED) &&
435 !name.endsWith(GIT_EXTENSION);
436 }
437
438 @Override
439 public String toString() {
440 return Util.simpleTypeName(getClass()) + '(' + rootDir + ')';
441 }
442
443 private final class StorageRemovalManager {
444
445 private static final String REMOVAL_TIMESTAMP_NAME = "removal.timestamp";
446
447 void mark(File file) {
448 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
449 final String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
450 try {
451 Files.write(removal.toPath(), timestamp.getBytes(StandardCharsets.UTF_8));
452 } catch (IOException e) {
453 throw new StorageException(
454 "failed to write a removal timestamp for " + childTypeName + ": " + removal);
455 }
456 }
457
458 void unmark(File file) {
459 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
460 if (removal.exists()) {
461 if (!removal.delete()) {
462 logger.warn("Failed to delete a removal timestamp for {}: {}", childTypeName, removal);
463 }
464 }
465 }
466
467 Instant readRemoval(File file) {
468 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
469 if (!removal.exists()) {
470 return Instant.ofEpochMilli(file.lastModified());
471 }
472 try {
473 final String timestamp = new String(Files.readAllBytes(removal.toPath()),
474 StandardCharsets.UTF_8);
475 return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(timestamp));
476 } catch (Exception e) {
477 logger.warn("Failed to read a removal timestamp for {}: {}", childTypeName, removal, e);
478 return Instant.ofEpochMilli(file.lastModified());
479 }
480 }
481 }
482 }