1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package com.linecorp.centraldogma.server.internal.storage;
18
19 import static com.google.common.base.Preconditions.checkState;
20 import static java.util.Objects.requireNonNull;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.nio.charset.StandardCharsets;
25 import java.nio.file.FileAlreadyExistsException;
26 import java.nio.file.Files;
27 import java.time.Instant;
28 import java.time.format.DateTimeFormatter;
29 import java.util.Arrays;
30 import java.util.Collections;
31 import java.util.LinkedHashMap;
32 import java.util.Map;
33 import java.util.concurrent.ConcurrentHashMap;
34 import java.util.concurrent.ConcurrentMap;
35 import java.util.concurrent.Executor;
36 import java.util.concurrent.ThreadLocalRandom;
37 import java.util.concurrent.atomic.AtomicBoolean;
38 import java.util.concurrent.atomic.AtomicReference;
39 import java.util.function.Supplier;
40 import java.util.regex.Pattern;
41
42 import javax.annotation.Nullable;
43
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47 import com.google.common.collect.ImmutableMap;
48
49 import com.linecorp.centraldogma.common.Author;
50 import com.linecorp.centraldogma.common.CentralDogmaException;
51 import com.linecorp.centraldogma.internal.Util;
52 import com.linecorp.centraldogma.server.storage.StorageException;
53 import com.linecorp.centraldogma.server.storage.StorageManager;
54
55 public abstract class DirectoryBasedStorageManager<T> implements StorageManager<T> {
56
57 private static final Logger logger = LoggerFactory.getLogger(DirectoryBasedStorageManager.class);
58
59
60
61
62
63
64 private static final Pattern CHILD_NAME =
65 Pattern.compile("^[0-9A-Za-z](?:[-+_0-9A-Za-z.]*[0-9A-Za-z])?$");
66 private static final String SUFFIX_REMOVED = ".removed";
67 private static final String SUFFIX_PURGED = ".purged";
68
69 private final String childTypeName;
70 private final File rootDir;
71 private final StorageRemovalManager storageRemovalManager = new StorageRemovalManager();
72 private final ConcurrentMap<String, T> children = new ConcurrentHashMap<>();
73 private final AtomicReference<Supplier<CentralDogmaException>> closed = new AtomicReference<>();
74 private final Executor purgeWorker;
75 private boolean initialized;
76
77 protected DirectoryBasedStorageManager(File rootDir, Class<? extends T> childType,
78 Executor purgeWorker) {
79 requireNonNull(rootDir, "rootDir");
80 this.purgeWorker = requireNonNull(purgeWorker, "purgeWorker");
81
82 if (!rootDir.exists()) {
83 if (!rootDir.mkdirs()) {
84 throw new StorageException("failed to create root directory at " + rootDir);
85 }
86 }
87
88 try {
89 rootDir = rootDir.getCanonicalFile();
90 } catch (IOException e) {
91 throw new StorageException("failed to get the canonical path of: " + rootDir, e);
92 }
93
94 if (!rootDir.isDirectory()) {
95 throw new StorageException("not a directory: " + rootDir);
96 }
97
98 this.rootDir = rootDir;
99 childTypeName = Util.simpleTypeName(requireNonNull(childType, "childTypeName"), true);
100 }
101
102 protected Executor purgeWorker() {
103 return purgeWorker;
104 }
105
106
107
108
109 protected final void init() {
110 checkState(!initialized, "initialized already");
111 Throwable cause = null;
112 try {
113 final File[] childFiles = rootDir.listFiles();
114 if (childFiles != null) {
115 for (File f : childFiles) {
116 loadChild(f);
117 }
118 }
119 initialized = true;
120 } catch (Throwable t) {
121 cause = t;
122 }
123
124 if (cause != null) {
125 final CentralDogmaException finalCause;
126 if (cause instanceof CentralDogmaException) {
127 finalCause = (CentralDogmaException) cause;
128 } else {
129 finalCause = new CentralDogmaException("Failed to load a child: " + cause, cause);
130 }
131 close(() -> finalCause);
132 throw finalCause;
133 }
134 }
135
136 @Nullable
137 private T loadChild(File f) {
138 final String name = f.getName();
139 if (!isValidChildName(name)) {
140 return null;
141 }
142
143 if (!f.isDirectory()) {
144 return null;
145 }
146
147 if (new File(f + SUFFIX_REMOVED).exists()) {
148 return null;
149 }
150
151 try {
152 final T child = openChild(f);
153 children.put(name, child);
154 return child;
155 } catch (RuntimeException e) {
156 throw e;
157 } catch (Exception e) {
158 throw new StorageException("failed to open " + childTypeName + ": " + f, e);
159 }
160 }
161
162 protected abstract T openChild(File childDir) throws Exception;
163
164 protected abstract T createChild(File childDir, Author author, long creationTimeMillis) throws Exception;
165
166 private void closeChild(String name, T child, Supplier<CentralDogmaException> failureCauseSupplier) {
167 closeChild(new File(rootDir, name), child, failureCauseSupplier);
168 }
169
170 protected void closeChild(File childDir, T child, Supplier<CentralDogmaException> failureCauseSupplier) {}
171
172 protected abstract CentralDogmaException newStorageExistsException(String name);
173
174 protected abstract CentralDogmaException newStorageNotFoundException(String name);
175
176 @Override
177 public void close(Supplier<CentralDogmaException> failureCauseSupplier) {
178 requireNonNull(failureCauseSupplier, "failureCauseSupplier");
179 if (!closed.compareAndSet(null, failureCauseSupplier)) {
180 return;
181 }
182
183
184 for (Map.Entry<String, T> e : children.entrySet()) {
185 closeChild(e.getKey(), e.getValue(), failureCauseSupplier);
186 }
187 }
188
189 @Override
190 public boolean exists(String name) {
191 ensureOpen();
192 return children.containsKey(validateChildName(name));
193 }
194
195 @Override
196 public T get(String name) {
197 ensureOpen();
198 final T child = children.get(validateChildName(name));
199 if (child == null) {
200 throw newStorageNotFoundException(name);
201 }
202
203 return child;
204 }
205
206 @Override
207 public T create(String name, long creationTimeMillis, Author author) {
208 ensureOpen();
209 requireNonNull(author, "author");
210 validateChildName(name);
211
212 final AtomicBoolean created = new AtomicBoolean();
213 final T child = children.computeIfAbsent(name, n -> {
214 final T c = create0(author, n, creationTimeMillis);
215 created.set(true);
216 return c;
217 });
218
219 if (created.get()) {
220 return child;
221 } else {
222 throw newStorageExistsException(name);
223 }
224 }
225
226 private T create0(Author author, String name, long creationTimeMillis) {
227 if (new File(rootDir, name + SUFFIX_REMOVED).exists()) {
228 throw newStorageExistsException(name + " (removed)");
229 }
230
231 final File f = new File(rootDir, name);
232 boolean success = false;
233 try {
234 final T newChild = createChild(f, author, creationTimeMillis);
235 success = true;
236 return newChild;
237 } catch (RuntimeException e) {
238 throw e;
239 } catch (Exception e) {
240 throw new StorageException("failed to create a new " + childTypeName + ": " + f, e);
241 } finally {
242 if (!success && f.exists()) {
243
244 try {
245 Util.deleteFileTree(f);
246 } catch (IOException e) {
247 logger.warn("Failed to delete a partially created project: {}", f, e);
248 }
249 }
250 }
251 }
252
253 @Override
254 public Map<String, T> list() {
255 ensureOpen();
256
257 final int estimatedSize = children.size();
258 final String[] names = children.keySet().toArray(new String[estimatedSize]);
259 Arrays.sort(names);
260
261 final Map<String, T> ret = new LinkedHashMap<>(estimatedSize);
262 for (String k : names) {
263 final T v = children.get(k);
264 if (v != null) {
265 ret.put(k, v);
266 }
267 }
268
269 return Collections.unmodifiableMap(ret);
270 }
271
272 @Override
273 public Map<String, Instant> listRemoved() {
274 ensureOpen();
275 final File[] files = rootDir.listFiles();
276 if (files == null) {
277 return ImmutableMap.of();
278 }
279
280 Arrays.sort(files);
281 final ImmutableMap.Builder<String, Instant> builder = ImmutableMap.builder();
282
283 for (File f : files) {
284 if (!f.isDirectory()) {
285 continue;
286 }
287
288 String name = f.getName();
289 if (!name.endsWith(SUFFIX_REMOVED)) {
290 continue;
291 }
292
293 name = name.substring(0, name.length() - SUFFIX_REMOVED.length());
294 if (!isValidChildName(name) || children.containsKey(name)) {
295 continue;
296 }
297
298 builder.put(name, storageRemovalManager.readRemoval(f));
299 }
300
301 return builder.build();
302 }
303
304 @Override
305 public void remove(String name) {
306 ensureOpen();
307 final T child = children.remove(validateChildName(name));
308 if (child == null) {
309 throw newStorageNotFoundException(name);
310 }
311
312 closeChild(name, child, () -> newStorageNotFoundException(name));
313
314 final File file = new File(rootDir, name);
315 storageRemovalManager.mark(file);
316 if (!file.renameTo(new File(rootDir, name + SUFFIX_REMOVED))) {
317 throw new StorageException("failed to mark " + childTypeName + " as removed: " + name);
318 }
319 }
320
321 @Override
322 public T unremove(String name) {
323 ensureOpen();
324 validateChildName(name);
325
326 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
327 if (!removed.isDirectory()) {
328 throw newStorageNotFoundException(name);
329 }
330
331 final File unremoved = new File(rootDir, name);
332
333 if (!removed.renameTo(unremoved)) {
334 throw new StorageException("failed to mark " + childTypeName + " as unremoved: " + name);
335 }
336 storageRemovalManager.unmark(unremoved);
337
338 final T unremovedChild = loadChild(unremoved);
339 if (unremovedChild == null) {
340 throw newStorageNotFoundException(name);
341 }
342 return unremovedChild;
343 }
344
345 @Override
346 public void markForPurge(String name) {
347 ensureOpen();
348 validateChildName(name);
349 File marked;
350 final File removed = new File(rootDir, name + SUFFIX_REMOVED);
351
352 final Supplier<File> newMarkedFile = () -> {
353 final String interfix = '.' + Long.toHexString(ThreadLocalRandom.current().nextLong());
354 return new File(rootDir, name + interfix + SUFFIX_PURGED);
355 };
356
357 synchronized (this) {
358 if (!removed.exists()) {
359 logger.warn("Tried to purge {}, but it's non-existent.", removed);
360 return;
361 }
362
363 if (!removed.isDirectory()) {
364 throw new StorageException("not a directory: " + removed);
365 }
366
367 marked = newMarkedFile.get();
368 boolean moved = false;
369 while (!moved) {
370 try {
371 Files.move(removed.toPath(), marked.toPath());
372 moved = true;
373 } catch (FileAlreadyExistsException e) {
374 marked = newMarkedFile.get();
375 } catch (IOException e) {
376 throw new StorageException("failed to mark " + childTypeName + " for purge: " + removed, e);
377 }
378 }
379 }
380 final File purged = marked;
381 try {
382 purgeWorker.execute(() -> deletePurgedFile(purged));
383 } catch (Exception e) {
384 logger.warn("Failed to schedule a purge task for {}:", purged, e);
385 }
386 }
387
388 @Override
389 public void purgeMarked() {
390 ensureOpen();
391 final File[] files = rootDir.listFiles();
392 if (files == null) {
393 return;
394 }
395 for (File f : files) {
396 if (!f.isDirectory()) {
397 continue;
398 }
399 final String name = f.getName();
400 if (name.endsWith(SUFFIX_PURGED)) {
401 deletePurgedFile(f);
402 }
403 }
404 }
405
406 private void deletePurgedFile(File file) {
407 try {
408 logger.info("Deleting a purged {}: {} ..", childTypeName, file);
409 Util.deleteFileTree(file);
410 logger.info("Deleted a purged {}: {}.", childTypeName, file);
411 } catch (IOException e) {
412 logger.warn("Failed to delete a purged {}: {}", childTypeName, file, e);
413 }
414 }
415
416 @Override
417 public void ensureOpen() {
418 checkState(initialized, "not initialized yet");
419 if (closed.get() != null) {
420 throw closed.get().get();
421 }
422 }
423
424 private String validateChildName(String name) {
425 if (!isValidChildName(requireNonNull(name, "name"))) {
426 throw new IllegalArgumentException("invalid " + childTypeName + " name: " + name);
427 }
428 return name;
429 }
430
431 private static boolean isValidChildName(String name) {
432 if (name == null) {
433 return false;
434 }
435
436 if (!CHILD_NAME.matcher(name).matches()) {
437 return false;
438 }
439
440 return !name.endsWith(SUFFIX_REMOVED) && !name.endsWith(SUFFIX_PURGED);
441 }
442
443 @Override
444 public String toString() {
445 return Util.simpleTypeName(getClass()) + '(' + rootDir + ')';
446 }
447
448 private final class StorageRemovalManager {
449
450 private static final String REMOVAL_TIMESTAMP_NAME = "removal.timestamp";
451
452 void mark(File file) {
453 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
454 final String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
455 try {
456 Files.write(removal.toPath(), timestamp.getBytes(StandardCharsets.UTF_8));
457 } catch (IOException e) {
458 throw new StorageException(
459 "failed to write a removal timestamp for " + childTypeName + ": " + removal);
460 }
461 }
462
463 void unmark(File file) {
464 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
465 if (removal.exists()) {
466 if (!removal.delete()) {
467 logger.warn("Failed to delete a removal timestamp for {}: {}", childTypeName, removal);
468 }
469 }
470 }
471
472 Instant readRemoval(File file) {
473 final File removal = new File(file, REMOVAL_TIMESTAMP_NAME);
474 if (!removal.exists()) {
475 return Instant.ofEpochMilli(file.lastModified());
476 }
477 try {
478 final String timestamp = new String(Files.readAllBytes(removal.toPath()),
479 StandardCharsets.UTF_8);
480 return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(timestamp));
481 } catch (Exception e) {
482 logger.warn("Failed to read a removal timestamp for {}: {}", childTypeName, removal, e);
483 return Instant.ofEpochMilli(file.lastModified());
484 }
485 }
486 }
487 }