squash! Extend index.json by additional file meta data.task-31204

 - Update to new metrics-lib release that contains the fix for #32194.

 - Update to latest metrics-base containing permission to run tests:
   permission java.nio.file.LinkPermission "hard";

 - Fix a bug where we would keep scheduling new indexer tasks for
   files that are already in the queue for being indexed.

 - Add another info log message to show progress during bulk index
   runs.

4 files changed, 31 insertions, 4 deletions

diff --git a/build.xml b/build.xml
index 8276e63..019461d 100644
--- a/build.xml
+++ b/build.xml
@@ -12,7 +12,7 @@
   <property name="release.version" value="1.12.0-dev" />
   <property name="project-main-class" value="org.torproject.metrics.collector.Main" />
   <property name="name" value="collector"/>
-  <property name="metricslibversion" value="2.8.0" />
+  <property name="metricslibversion" value="2.9.0" />
   <property name="jarincludes" value="collector.properties logback.xml" />
 
   <patternset id="runtime" >
diff --git a/src/build b/src/build
-Subproject d82fff984634fe006ac7b0b102e7f48a52ca20d
+Subproject eb16cb359db41722e6089bafb1e26808df4338d
diff --git a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
index fc7c796..15aa31d 100644
--- a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
+++ b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java
@@ -387,6 +387,7 @@ public class CreateIndexJson extends CollecTorMain {
    *     deleting links.
    */
   private void scheduleTasksAndUpdateLinks(Instant now) throws IOException {
+    int queuedIndexerTasks = 0;
     Map<Path, FileNode> indexingResults = new HashMap<>();
     SortedSet<Path> filesToIndex = new TreeSet<>();
     Map<Path, Path> linksToCreate = new HashMap<>();
@@ -398,10 +399,17 @@ public class CreateIndexJson extends CollecTorMain {
           .resolve(this.indexedPath.relativize(filePath));
       FileNode fileNode = e.getValue();
       if (Files.exists(filePath)) {
-        if (null != fileNode.indexerResult && fileNode.indexerResult.isDone()) {
+        if (null != fileNode.indexerResult) {
+          if (!fileNode.indexerResult.isDone()) {
+            /* This file is currently being indexed, so we should just skip it
+             * and wait until the indexer is done. */
+            queuedIndexerTasks++;
+            continue;
+          }
           try {
-            FileNode indexingResult = fileNode.indexerResult.get();
-            indexingResults.put(filePath, indexingResult);
+            /* Indexing is done, obtain index results. */
+            fileNode = fileNode.indexerResult.get();
+            indexingResults.put(filePath, fileNode);
           } catch (InterruptedException | ExecutionException ex) {
             /* Clear index result, so that we can give this file another try
              * next time. */
@@ -441,6 +449,10 @@ public class CreateIndexJson extends CollecTorMain {
         }
       }
     }
+    if (queuedIndexerTasks > 0) {
+      logger.info("Counting {} file(s) being currently indexed or in the "
+          + "queue.", queuedIndexerTasks);
+    }
     this.updateIndex(indexingResults);
     this.scheduleTasks(filesToIndex);
     this.createLinks(linksToCreate);
diff --git a/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java b/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
index 95b916c..db00032 100644
--- a/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
+++ b/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java
@@ -503,5 +503,20 @@ public class CreateIndexJsonTest {
     assertEquals(String.format(archiveExitListIndexJsonString,
         "2016-10-04 03:31"), readIndexJson());
   }
+
+  /**
+   * Test whether a long-running indexer task is being given the time to finish,
+   * rather than starting another task for the same file.
+   */
+  @Test
+  public void testLongRunningIndexerTask() {
+    createFile(archiveExitListFilePath, Instant.parse("2016-10-04T03:31:00Z"));
+    startProcessing(firstExecution);
+    startProcessing(secondExecution);
+    assertEquals(emptyIndexJsonString, readIndexJson());
+    finishIndexing(archiveExitListFileNode);
+    startProcessing(thirdExecution);
+    assertTrue(this.indexerTasks.isEmpty());
+  }
 }