Enable OnionPerf 'Sync' runs.

943deb30 · iwakeh · 0e5146e5 · 943deb30 · 943deb30 · 943deb30
Commit 943deb30 authored Jul 12, 2017 by iwakeh
--- a/src/main/java/org/torproject/collector/conf/Annotation.java
+++ b/src/main/java/org/torproject/collector/conf/Annotation.java
@@ -16,7 +16,7 @@ public enum Annotation {
  Microdescriptor("@type microdescriptor 1.0\n"),
  Server("@type server-descriptor 1.0\n"),
  Status("@type bridge-network-status 1.2\n"),
-  OnionPerf("@type torperf 1.0\n"),
+  OnionPerf("@type torperf 1.1\n"),
  Vote("@type network-status-vote-3 1.0\n");

  private final String annotation;


--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -34,6 +34,7 @@ public enum Key {
  BridgeSyncOrigins(URL[].class),
  BridgeLocalOrigins(Path.class),
  ExitlistSyncOrigins(URL[].class),
+  OnionPerfSyncOrigins(URL[].class),
  BridgedescsActivated(Boolean.class),
  BridgedescsOffsetMinutes(Integer.class),
  BridgedescsPeriodMinutes(Integer.class),


--- a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java
+++ b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java
@@ -51,6 +51,7 @@ public class OnionPerfDownloader extends CollecTorMain {
  /** Instanciate the OnionPerf module using the given configuration. */
  public OnionPerfDownloader(Configuration config) {
    super(config);
+    this.mapPathDescriptors.put("recent/torperf", TorperfResult.class);
  }

  /** File containing the download history, which is necessary, because


--- a/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java
+++ b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.persist;
+
+import org.torproject.collector.conf.Annotation;
+import org.torproject.descriptor.TorperfResult;
+
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+
+public class OnionPerfPersistence
+    extends DescriptorPersistence<TorperfResult> {
+
+  private static final String ONIONPERF = "torperf";
+
+  public OnionPerfPersistence(TorperfResult desc) {
+    super(desc, Annotation.OnionPerf.bytes());
+    calculatePaths();
+  }
+
+  private void calculatePaths() {
+    String[] parts = PersistenceUtils.dateTimeParts(desc.getStartMillis());
+    String name =  desc.getSource() + DASH + desc.getFileSize() + DASH
+        + parts[0] + DASH + parts[1] + DASH + parts[2] + ".tpf";
+    this.recentPath = Paths.get(ONIONPERF, name).toString();
+    this.storagePath = Paths.get(
+        ONIONPERF,
+        parts[0], // year
+        parts[1], // month
+        parts[2], // day
+        name).toString();
+  }
+
+  /** OnionPerf default storage appends. */
+  @Override
+  public boolean storeOut(String outRoot) {
+    return super.storeOut(outRoot, StandardOpenOption.APPEND);
+  }
+
+  /** OnionPerf default storage appends. */
+  @Override
+  public boolean storeAll(String recentRoot, String outRoot) {
+    return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND,
+        StandardOpenOption.APPEND);
+  }
+
+}
+
--- a/src/main/java/org/torproject/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/collector/sync/SyncPersistence.java
@@ -13,6 +13,7 @@ import org.torproject.collector.persist.DescriptorPersistence;
 import org.torproject.collector.persist.ExitlistPersistence;
 import org.torproject.collector.persist.ExtraInfoPersistence;
 import org.torproject.collector.persist.MicroConsensusPersistence;
+import org.torproject.collector.persist.OnionPerfPersistence;
 import org.torproject.collector.persist.PersistenceUtils;
 import org.torproject.collector.persist.ServerDescriptorPersistence;
 import org.torproject.collector.persist.StatusPersistence;
@@ -26,6 +27,7 @@ import org.torproject.descriptor.RelayExtraInfoDescriptor;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.descriptor.RelayNetworkStatusVote;
 import org.torproject.descriptor.RelayServerDescriptor;
+import org.torproject.descriptor.TorperfResult;

 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -127,6 +129,9 @@ public class SyncPersistence {
        case "ExitList": // downloaded is part of desc, which to use?
          descPersist = new ExitlistPersistence((ExitList) desc, received);
          break;
+        case "TorperfResult":
+          descPersist = new OnionPerfPersistence((TorperfResult) desc);
+          break;
        default:
          log.trace("Invalid descriptor type {} for sync-merge.",
              clazz.getName());


--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -142,8 +142,11 @@ ExitlistUrl = https://check.torproject.org/exit-addresses
 ######## OnionPerf downloader ########
 #
 ## Define descriptor sources
-#  possible values: Remote
+#  possible values: Remote,Sync
 OnionPerfSources = Remote
+#  Retrieve files from the following CollecTor instances.
+#  List of URLs separated by comma.
+OnionPerfSyncOrigins = https://collector.torproject.org
 #
 ## OnionPerf base URLs
 ## Hosts must be configured to use the first subdomain part of the given URL as