Skip to content
Snippets Groups Projects
Commit 3181179c authored by iwakeh's avatar iwakeh :palm_tree:
Browse files

Prevent updateindex NPE, b/c of dubious files that are regcognized as... 

Prevent updateindex NPE, b/c of dubious files that are regcognized as directories. Adjusted coverage rate. Modified scheduler test. Reduced repeated try-catch in run methods. Log cron package to all module logs.
parent 334ff75d
Branches
Tags
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 167 additions and 97 deletions
build.xml
+ 3
− 3
View file @ 3181179c
......@@ -227,9 +227,9 @@
<include name="**/*.java" />
</fileset>
</cobertura-report>
<cobertura-check branchrate="0" totallinerate="4" totalbranchrate="1" >
<cobertura-check totallinerate="5" totalbranchrate="1" >
<regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/>
<regex pattern="org.torproject.collector.cron.*" branchrate="66" linerate="73"/>
<regex pattern="org.torproject.collector.cron" branchrate="66" linerate="80" />
<regex pattern="org.torproject.collector.Main" branchrate="66" linerate="94" />
</cobertura-check>
</target>
......
src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+ 7
− 11
View file @ 3181179c
......@@ -81,18 +81,12 @@ public class SanitizedBridgesWriter extends CollecTorMain {
private SecureRandom secureRandom;
@Override
public void run() {
logger.info("Starting bridge-descriptors module of CollecTor.");
try {
startProcessing();
} catch (ConfigurationException ce) {
logger.error("Configuration failed: " + ce, ce);
throw new RuntimeException(ce);
}
logger.info("Terminating bridge-descriptors module of CollecTor.");
public String module() {
return "bridgedescs";
}
private void startProcessing() throws ConfigurationException {
@Override
protected void startProcessing() throws ConfigurationException {
File bridgeDirectoriesDirectory =
config.getPath(Key.BridgeSnapshotsDirectory).toFile();
......@@ -102,7 +96,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
if (bridgeDirectoriesDirectory == null
|| sanitizedBridgesDirectory == null || statsDirectory == null) {
throw new IllegalArgumentException();
throw new ConfigurationException("BridgeSnapshotsDirectory, "
+ "SanitizedBridgesWriteDirectory, StatsPath should be set. "
+ "Please, edit the 'collector.properties' file.");
}
/* Memorize argument values. */
......
src/main/java/org/torproject/collector/cron/CollecTorMain.java
+ 26
− 0
View file @ 3181179c
......@@ -4,6 +4,7 @@
package org.torproject.collector.cron;
import org.torproject.collector.conf.Configuration;
import org.torproject.collector.conf.ConfigurationException;
import org.torproject.collector.conf.Key;
import org.slf4j.Logger;
......@@ -18,11 +19,36 @@ import java.util.concurrent.TimeUnit;
public abstract class CollecTorMain implements Runnable {
private static Logger log = LoggerFactory.getLogger(CollecTorMain.class);
protected Configuration config;
public CollecTorMain( Configuration conf) {
this.config = conf;
}
/**
* Log errors preventing successful completion of the module.
*/
@Override
public final void run() {
log.info("Starting {} module of CollecTor.", module());
try {
startProcessing();
} catch (ConfigurationException | RuntimeException ce) {
log.error("The {} module failed: {}", module(), ce.getMessage(), ce);
}
log.info("Terminating {} module of CollecTor.", module());
}
/**
* Module specific code goes here.
*/
protected abstract void startProcessing() throws ConfigurationException;
/**
* Returns the module name for logging purposes.
*/
public abstract String module();
}
src/main/java/org/torproject/collector/cron/Scheduler.java
+ 27
− 9
View file @ 3181179c
......@@ -16,22 +16,28 @@ import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
* Scheduler that starts the modules configured in collector.properties.
*/
public class Scheduler {
public class Scheduler implements ThreadFactory {
public static final String ACTIVATED = "Activated";
public static final String PERIODMIN = "PeriodMinutes";
public static final String OFFSETMIN = "OffsetMinutes";
private final Logger log = LoggerFactory.getLogger(Scheduler.class);
private static final Logger log = LoggerFactory.getLogger(Scheduler.class);
private final ThreadFactory threads = Executors.defaultThreadFactory();
private int currentThreadNo = 0;
private final ScheduledExecutorService scheduler =
Executors.newScheduledThreadPool(1);
Executors.newScheduledThreadPool(10, this);
private static Scheduler instance = new Scheduler();
......@@ -60,11 +66,10 @@ public class Scheduler {
}
} catch (ConfigurationException | IllegalAccessException
| InstantiationException | InvocationTargetException
| NoSuchMethodException | RuntimeException ex) {
| NoSuchMethodException | RejectedExecutionException
| NullPointerException ex) {
log.error("Cannot schedule " + ctmEntry.getValue().getName()
+ ". Reason: " + ex.getMessage(), ex);
shutdownScheduler();
throw new RuntimeException("Halted scheduling.", ex);
}
}
}
......@@ -73,12 +78,13 @@ public class Scheduler {
this.log.info("Periodic updater started for " + ctm.getClass().getName()
+ "; offset=" + offset + ", period=" + period + ".");
int currentMinute = Calendar.getInstance().get(Calendar.MINUTE);
int initialDelay = (60 - currentMinute + offset) % 60;
int initialDelay = (period - (currentMinute % period) + offset) % period;
/* Run after initialDelay delay and then every period min. */
this.log.info("Periodic updater will start every " + period + "th min "
+ "at minute " + ((currentMinute + initialDelay) % 60) + ".");
this.scheduler.scheduleAtFixedRate(ctm, initialDelay, 60,
+ "at minute " + ((currentMinute + initialDelay) % period) + "."
+ " The first start will happen in " + initialDelay + " minute(s).");
this.scheduler.scheduleAtFixedRate(ctm, initialDelay, period,
TimeUnit.MINUTES);
}
......@@ -98,5 +104,17 @@ public class Scheduler {
}
}
}
/**
* Provide a nice name for debugging and log thread creation.
*/
@Override
public Thread newThread(Runnable runner) {
Thread newThread = threads.newThread(runner);
newThread.setName("CollecTor-Scheduled-Thread-" + ++currentThreadNo);
log.info("New Thread created: " + newThread.getName());
return newThread;
}
}
src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+ 5
− 11
View file @ 3181179c
......@@ -42,18 +42,12 @@ public class ExitListDownloader extends CollecTorMain {
}
@Override
public void run() {
logger.info("Starting exit-lists module of CollecTor.");
try {
startProcessing();
} catch (ConfigurationException ce) {
logger.error("Configuration failed: " + ce, ce);
throw new RuntimeException(ce);
}
logger.info("Terminating exit-lists module of CollecTor.");
public String module() {
return "exitlists";
}
private void startProcessing() throws ConfigurationException {
@Override
protected void startProcessing() throws ConfigurationException {
SimpleDateFormat dateTimeFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
......@@ -92,7 +86,7 @@ public class ExitListDownloader extends CollecTorMain {
return;
}
if (downloadedExitList == null) {
logger.warn("Failed downloading exit list");
logger.warn("Failed downloading exit list.");
return;
}
......
src/main/java/org/torproject/collector/index/CreateIndexJson.java
+ 40
− 17
View file @ 3181179c
......@@ -13,6 +13,8 @@ import com.google.gson.GsonBuilder;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedWriter;
import java.io.File;
......@@ -38,17 +40,19 @@ import java.util.zip.GZIPOutputStream;
* we'll likely have to do that. */
public class CreateIndexJson extends CollecTorMain {
private static Logger log = LoggerFactory.getLogger(CreateIndexJson.class);
private static File indexJsonFile;
private static String basePath;
private static File[] indexedDirectories;
static final String dateTimePattern = "yyyy-MM-dd HH:mm";
private static final String dateTimePattern = "yyyy-MM-dd HH:mm";
static final Locale dateTimeLocale = Locale.US;
private static final Locale dateTimeLocale = Locale.US;
static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");
private static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");
/** Creates indexes of directories containing archived and recent
* descriptors and write index files to disk. */
......@@ -57,7 +61,12 @@ public class CreateIndexJson extends CollecTorMain {
}
@Override
public void run() {
public String module() {
return "updateindex";
}
@Override
protected void startProcessing() throws ConfigurationException {
try {
indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(),
"index.json");
......@@ -67,12 +76,12 @@ public class CreateIndexJson extends CollecTorMain {
config.getPath(Key.RecentPath).toFile() };
writeIndex(indexDirectories());
} catch (Exception e) {
throw new RuntimeException("Cannot run index creation: " + e.getMessage(),
e);
log.error("Cannot run index creation: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}
static class DirectoryNode implements Comparable<DirectoryNode> {
private class DirectoryNode implements Comparable<DirectoryNode> {
String path;
SortedSet<FileNode> files;
SortedSet<DirectoryNode> directories;
......@@ -90,7 +99,7 @@ public class CreateIndexJson extends CollecTorMain {
}
@SuppressWarnings({"checkstyle:membername", "checkstyle:parametername"})
static class IndexNode {
private class IndexNode {
String index_created;
String path;
SortedSet<FileNode> files;
......@@ -107,7 +116,7 @@ public class CreateIndexJson extends CollecTorMain {
}
@SuppressWarnings({"checkstyle:membername", "checkstyle:parametername"})
static class FileNode implements Comparable<FileNode> {
private class FileNode implements Comparable<FileNode> {
String path;
long size;
String last_modified;
......@@ -123,7 +132,7 @@ public class CreateIndexJson extends CollecTorMain {
}
}
static DateFormat dateTimeFormat;
private static DateFormat dateTimeFormat;
static {
dateTimeFormat = new SimpleDateFormat(dateTimePattern,
......@@ -132,30 +141,44 @@ public class CreateIndexJson extends CollecTorMain {
dateTimeFormat.setTimeZone(dateTimezone);
}
static IndexNode indexDirectories() {
private IndexNode indexDirectories() {
SortedSet<DirectoryNode> directoryNodes =
new TreeSet<DirectoryNode>();
log.trace("indexing: " + indexedDirectories[0] + " "
+ indexedDirectories[1]);
for (File directory : indexedDirectories) {
if (directory.exists() && directory.isDirectory()) {
directoryNodes.add(indexDirectory(directory));
DirectoryNode dn = indexDirectory(directory);
if (null != dn) {
directoryNodes.add(dn);
}
}
}
return new IndexNode(dateTimeFormat.format(
System.currentTimeMillis()), basePath, null, directoryNodes);
}
static DirectoryNode indexDirectory(File directory) {
private DirectoryNode indexDirectory(File directory) {
SortedSet<FileNode> fileNodes = new TreeSet<FileNode>();
SortedSet<DirectoryNode> directoryNodes =
new TreeSet<DirectoryNode>();
for (File fileOrDirectory : directory.listFiles()) {
log.trace("indexing: " + directory);
File[] fileList = directory.listFiles();
if (null == fileList) {
log.warn("Indexing dubious directory: " + directory);
return null;
}
for (File fileOrDirectory : fileList) {
if (fileOrDirectory.getName().startsWith(".")) {
continue;
}
if (fileOrDirectory.isFile()) {
fileNodes.add(indexFile(fileOrDirectory));
} else {
directoryNodes.add(indexDirectory(fileOrDirectory));
DirectoryNode dn = indexDirectory(fileOrDirectory);
if (null != dn) {
directoryNodes.add(dn);
}
}
}
DirectoryNode directoryNode = new DirectoryNode(
......@@ -164,13 +187,13 @@ public class CreateIndexJson extends CollecTorMain {
return directoryNode;
}
static FileNode indexFile(File file) {
private FileNode indexFile(File file) {
FileNode fileNode = new FileNode(file.getName(), file.length(),
dateTimeFormat.format(file.lastModified()));
return fileNode;
}
static void writeIndex(IndexNode indexNode) throws IOException {
private void writeIndex(IndexNode indexNode) throws IOException {
Gson gson = new GsonBuilder().create();
String indexNodeString = gson.toJson(indexNode);
Writer[] writers = new Writer[] {
......
src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+ 19
− 24
View file @ 3181179c
......@@ -111,33 +111,24 @@ public class ArchiveWriter extends CollecTorMain {
}
@Override
public void run() {
logger.info("Starting relay-descriptors module of CollecTor.");
try {
public String module() {
return "relaydescs";
}
@Override
protected void startProcessing() throws ConfigurationException {
recentPath = config.getPath(Key.RecentPath);
recentPathName = recentPath.toString();
File statsDir = config.getPath(Key.StatsPath).toFile();
storedServerDescriptorsFile =
new File(statsDir, "stored-server-descriptors");
storedExtraInfoDescriptorsFile =
new File(statsDir, "stored-extra-info-descriptors");
storedMicrodescriptorsFile =
new File(statsDir, "stored-microdescriptors");
startProcessing();
new ReferenceChecker(recentPath.toFile(),
new File(statsDir, "references"),
new File(statsDir, "references-history")).check();
} catch (ConfigurationException ce) {
logger.error("Configuration failed: " + ce, ce);
throw new RuntimeException(ce);
}
logger.info("Terminating relay-descriptors module of CollecTor.");
}
private void startProcessing() throws ConfigurationException {
storedServerDescriptorsFile
= new File(statsDir, "stored-server-descriptors");
storedExtraInfoDescriptorsFile
= new File(statsDir, "stored-extra-info-descriptors");
storedMicrodescriptorsFile
= new File(statsDir, "stored-microdescriptors");
File statsDirectory = config.getPath(Key.StatsPath).toFile();
this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString();
this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory)
.toString();
SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
"yyyy-MM-dd-HH-mm-ss");
rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
......@@ -197,6 +188,10 @@ public class ArchiveWriter extends CollecTorMain {
this.cleanUpRsyncDirectory();
this.saveDescriptorDigests();
new ReferenceChecker(recentPath.toFile(),
new File(statsDir, "references"),
new File(statsDir, "references-history")).check();
}
private void loadDescriptorDigests() {
......
src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
+ 4
− 10
View file @ 3181179c
......@@ -46,18 +46,12 @@ public class TorperfDownloader extends CollecTorMain {
private File torperfLastMergedFile;
@Override
public void run() {
logger.info("Starting torperf module of CollecTor.");
try {
startProcessing();
} catch (ConfigurationException ce) {
logger.error("Configuration failed: " + ce, ce);
throw new RuntimeException(ce);
}
logger.info("Terminating torperf module of CollecTor.");
public String module() {
return "torperf";
}
private void startProcessing() throws ConfigurationException {
@Override
protected void startProcessing() throws ConfigurationException {
this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines);
this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory)
.toFile();
......
src/main/resources/logback.xml
+ 10
− 2
View file @ 3181179c
......@@ -93,7 +93,7 @@
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>TRACE</level>
<level>INFO</level>
</filter>
</appender>
......@@ -114,7 +114,7 @@
<appender-ref ref="FILETORPERF" />
</logger>
<logger name="org.torproject.collector.index" >
<logger name="org.torproject.collector.index" level="INFO" >
<appender-ref ref="FILEUPDATEINDEX" />
</logger>
......@@ -126,6 +126,14 @@
<appender-ref ref="FILEUPDATEINDEX" />
</logger>
<logger name="org.torproject.collector.cron" >
<appender-ref ref="FILEBRIDGEDESCS" />
<appender-ref ref="FILEEXITLISTS" />
<appender-ref ref="FILERELAYDESCS" />
<appender-ref ref="FILETORPERF" />
<appender-ref ref="FILEUPDATEINDEX" />
</logger>
<logger name="org.torproject" >
<appender-ref ref="CONSOLE" />
</logger>
......
src/test/java/org/torproject/collector/cron/Dummy.java
+ 12
− 6
View file @ 3181179c
package org.torproject.collector.cron;
import org.torproject.collector.conf.Configuration;
import org.torproject.collector.conf.ConfigurationException;
public class Dummy extends CollecTorMain {
......@@ -9,7 +10,12 @@ public class Dummy extends CollecTorMain {
}
@Override
public void run() {
public void startProcessing() throws ConfigurationException {
// dummy doesn't do anything.
}
@Override
public String module() {
return "dummy";
}
}
src/test/java/org/torproject/collector/cron/SchedulerTest.java
+ 14
− 4
View file @ 3181179c
......@@ -30,8 +30,16 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
public class SchedulerTest {
private static final String runConfigProperties = "TorperfActivated=true\n"
+ "TorperfPeriodMinutes=10\nTorperfOffsetMinutes=7\n";
private static final String runConfigProperties =
"TorperfActivated=true\nTorperfPeriodMinutes=1\nTorperfOffsetMinutes=0\n"
+ "RelaydescsActivated=true\nRelaydescsPeriodMinutes=1"
+ "\nRelaydescsOffsetMinutes=0\n"
+ "ExitlistsActivated=true\nExitlistsPeriodMinutes=1\n"
+ "ExitlistsOffsetMinutes=0\n"
+ "UpdateindexActivated=true\nUpdateindexPeriodMinutes=1\n"
+ "UpdateindexOffsetMinutes=0\n"
+ "BridgedescsActivated=true\nBridgedescsPeriodMinutes=1\n"
+ "BridgedescsOffsetMinutes=0\n";
@Test()
public void testSimpleSchedule() throws Exception {
......@@ -39,15 +47,17 @@ public class SchedulerTest {
Configuration conf = new Configuration();
conf.load(new ByteArrayInputStream(runConfigProperties.getBytes()));
ctms.put(Key.TorperfActivated, Dummy.class);
ctms.put(Key.BridgedescsActivated, Dummy.class);
ctms.put(Key.RelaydescsActivated, Dummy.class);
ctms.put(Key.ExitlistsActivated, Dummy.class);
ctms.put(Key.UpdateindexActivated, Dummy.class);
Field schedulerField = Scheduler.class.getDeclaredField("scheduler");
schedulerField.setAccessible(true);
ScheduledThreadPoolExecutor stpe = (ScheduledThreadPoolExecutor)
schedulerField.get(Scheduler.getInstance());
assertTrue(stpe.getQueue().isEmpty());
Scheduler.getInstance().scheduleModuleRuns(ctms, conf);
assertEquals(stpe.getQueue().size(), 1);
Scheduler.getInstance().shutdownScheduler();
assertTrue(stpe.isShutdown());
}
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment