squash! Archive snowflake statistics.task-29461

Break up the startProcessing() method to facilitate review and later
refactoring.

Update PROTOCOL to reflect that there will be monthly tarballs, not a
single tarball for all snowflake statistics.

2 files changed, 75 insertions, 36 deletions

diff --git a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
index 74e7849..63b4406 100644
--- a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
@@ -59,33 +59,13 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
 
     this.recentPathName = config.getPath(Key.RecentPath).toString();
     logger.debug("Downloading snowflake stats...");
-    ByteArrayOutputStream downloadedSnowflakeStats
-        = new ByteArrayOutputStream();
     URL url = config.getUrl(Key.SnowflakeStatsUrl);
-    try  {
-      HttpURLConnection huc = (HttpURLConnection) url.openConnection();
-      huc.setRequestMethod("GET");
-      huc.setReadTimeout(5000);
-      huc.connect();
-      int response = huc.getResponseCode();
-      if (response != 200) {
-        logger.warn("Could not download snowflake stats. Response code {}",
-            response);
-        return;
-      }
-      try (BufferedInputStream in = new BufferedInputStream(
-          huc.getInputStream())) {
-        int len;
-        byte[] data = new byte[1024];
-        while ((len = in.read(data, 0, 1024)) >= 0) {
-          downloadedSnowflakeStats.write(data, 0, len);
-        }
-      }
-      logger.debug("Finished downloading snowflake stats.");
-    } catch (IOException e) {
-      logger.warn("Failed downloading snowflake stats", e);
+    ByteArrayOutputStream downloadedSnowflakeStats
+        = this.downloadFromHttpServer(url);
+    if (null == downloadedSnowflakeStats) {
       return;
     }
+    logger.debug("Finished downloading {}.", url);
 
     DescriptorParser descriptorParser =
         DescriptorSourceFactory.createDescriptorParser();
@@ -108,16 +88,8 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
             + persistence.getRecentPath());
         File[] outputFiles = new File[] { tarballFile, rsyncFile };
         for (File outputFile : outputFiles) {
-          try {
-            outputFile.getParentFile().mkdirs();
-            OutputStream os = new FileOutputStream(outputFile);
-            os.write(Annotation.SnowflakeStats.bytes());
-            os.write(snowflakeStats.getRawDescriptorBytes());
-            os.close();
-          } catch (IOException e) {
-            logger.warn("Could not write downloaded snowflake stats to {}",
-                outputFile.getAbsolutePath(), e);
-          }
+          this.writeToFile(outputFile, Annotation.SnowflakeStats.bytes(),
+              snowflakeStats.getRawDescriptorBytes());
         }
       }
     }
@@ -133,6 +105,72 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
     this.cleanUpRsyncDirectory();
   }
 
+  /**
+   * Download the given URL from an HTTP server and return a stream with
+   * downloaded bytes.
+   *
+   * <p>If anything goes wrong while downloading, log a warning and return
+   * {@code null}.</p>
+   *
+   * @param url URL to download.
+   * @return Stream with downloaded bytes, or {@code null} if an error has
+   *     occurred.
+   */
+  private ByteArrayOutputStream downloadFromHttpServer(URL url) {
+    ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream();
+    try  {
+      HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+      huc.setRequestMethod("GET");
+      huc.setReadTimeout(5000);
+      huc.connect();
+      int response = huc.getResponseCode();
+      if (response != 200) {
+        logger.warn("Could not download {}. Response code {}", url, response);
+        return null;
+      }
+      try (BufferedInputStream in = new BufferedInputStream(
+          huc.getInputStream())) {
+        int len;
+        byte[] data = new byte[1024];
+        while ((len = in.read(data, 0, 1024)) >= 0) {
+          downloadedBytes.write(data, 0, len);
+        }
+      }
+    } catch (IOException e) {
+      logger.warn("Failed downloading {}.", url, e);
+      return null;
+    }
+    return downloadedBytes;
+  }
+
+  /**
+   * Write the given byte array(s) to the given file.
+   *
+   * <p>If the file already exists, it is overwritten. If the parent directory
+   * (or any of its parent directories) does not exist, it is created. If
+   * anything goes wrong, log a warning and return.</p>
+   *
+   * @param outputFile File to write to.
+   * @param bytes One or more byte arrays.
+   */
+  private void writeToFile(File outputFile, byte[] ... bytes) {
+    try {
+      if (!outputFile.getParentFile().exists() &&
+          !outputFile.getParentFile().mkdirs()) {
+        logger.warn("Could not create parent directories of {}.", outputFile);
+        return;
+      }
+      OutputStream os = new FileOutputStream(outputFile);
+      for (byte[] b : bytes) {
+        os.write(b);
+      }
+      os.close();
+    } catch (IOException e) {
+      logger.warn("Could not write downloaded snowflake stats to {}",
+          outputFile.getAbsolutePath(), e);
+    }
+  }
+
   /** Delete all files from the rsync directory that have not been modified
    * in the last three days. */
   public void cleanUpRsyncDirectory() {
diff --git a/src/main/resources/docs/PROTOCOL b/src/main/resources/docs/PROTOCOL
index 4560026..478f168 100644
--- a/src/main/resources/docs/PROTOCOL
+++ b/src/main/resources/docs/PROTOCOL
@@ -119,9 +119,10 @@
 
 2.5 'snowflakes' below 'archive'
 
-   'snowflakes' contains a single compressed tarball with snowflake statistics:
+   'snowflakes' contains compressed tarballs with snowflake statistics,
+   named in the following way:
 
-   * snowflakes.tar.xz
+   'snowflakes' DASH year DASH month DOT TAR DOT compression-type
 
 3.0 Index Files