Skip to content
Snippets Groups Projects
Commit 0fb5efe1 authored by Damian Johnson's avatar Damian Johnson
Browse files

Bandwidth file 1.4 headers

Parsing the headers added in bandwidth file version 1.4.0.

  https://trac.torproject.org/projects/tor/ticket/30160
parent f4c40e7a
Branches
Tags 1.7.0
No related merge requests found
......@@ -109,12 +109,11 @@ def log_traceback(sig, frame):
for p in multiprocessing.active_children():
try:
os.kill(p.pid, sig)
except OSError as e:
# If the process exited before we could kill it
if e.errno == errno.ESRCH: # No such process
pass
except OSError as exc:
if exc.errno == errno.ESRCH:
pass # already exited, no such process
else:
raise e
raise exc
if sig == signal.SIGABRT:
# we need to use os._exit() to abort every thread in the interpreter,
......
......@@ -38,6 +38,53 @@ HEADER_DIV = b'====='
HEADER_DIV_ALT = b'===='
class RecentStats(object):
"""
Statistical information collected over the last 'data_period' (by default
five days).
:var int consensus_count: number of consensuses published during this period
:var int prioritized_relays: number of relays prioritized to be measured
:var int prioritized_relay_lists: number of times a set of relays were
prioritized to be measured
:var int measurement_attempts: number of relay measurements we attempted
:var int measurement_failures: number of measurement attempts that failed
:var RelayFailures relay_failures: number of relays we failed to measure
"""
def __init__(self):
self.consensus_count = None
self.prioritized_relays = None
self.prioritized_relay_lists = None
self.measurement_attempts = None
self.measurement_failures = None
self.relay_failures = RelayFailures()
class RelayFailures(object):
"""
Summary of the number of relays we were unable to measure.
:var int no_measurement: number of relays that did not have any successful
measurements
:var int insuffient_period: number of relays whos measurements were collected
over a period that was too small (1 day by default)
:var int insufficient_measurements: number of relays we did not collect
enough measurements for (2 by default)
:var int stale: number of relays whos latest measurement is too old (5 days
by default)
"""
def __init__(self):
self.no_measurement = None
self.insuffient_period = None
self.insufficient_measurements = None
self.stale = None
# Converts header attributes to a given type. Malformed fields should be
# ignored according to the spec.
......@@ -56,9 +103,15 @@ def _date(val):
return None # not an iso formatted date
def _csv(val):
return map(lambda v: v.strip(), val.split(',')) if val is not None else None
# mapping of attributes => (header, type)
HEADER_ATTR = {
# version 1.1.0 introduced headers
'version': ('version', _str),
'software': ('software', _str),
......@@ -69,11 +122,32 @@ HEADER_ATTR = {
'created_at': ('file_created', _date),
'generated_at': ('generator_started', _date),
# version 1.2.0 additions
'consensus_size': ('number_consensus_relays', _int),
'eligible_count': ('number_eligible_relays', _int),
'eligible_percent': ('percent_eligible_relays', _int),
'min_count': ('minimum_number_eligible_relays', _int),
'min_percent': ('minimum_percent_eligible_relays', _int),
# version 1.3.0 additions
'scanner_country': ('scanner_country', _str),
'destinations_countries': ('destinations_countries', _csv),
# version 1.4.0 additions
'time_to_report_half_network': ('time_to_report_half_network', _int),
'recent_stats.consensus_count': ('recent_consensus_count', _int),
'recent_stats.prioritized_relay_lists': ('recent_priority_list_count', _int),
'recent_stats.prioritized_relays': ('recent_priority_relay_count', _int),
'recent_stats.measurement_attempts': ('recent_measurement_attempt_count', _int),
'recent_stats.measurement_failures': ('recent_measurement_failure_count', _int),
'recent_stats.relay_failures.no_measurement': ('recent_measurements_excluded_error_count', _int),
'recent_stats.relay_failures.insuffient_period': ('recent_measurements_excluded_near_count', _int),
'recent_stats.relay_failures.insufficient_measurements': ('recent_measurements_excluded_few_count', _int),
'recent_stats.relay_failures.stale': ('recent_measurements_excluded_old_count', _int),
}
HEADER_DEFAULT = {
......@@ -131,9 +205,15 @@ def _parse_header(descriptor, entries):
index += 1
descriptor.header = header
descriptor.recent_stats = RecentStats()
for full_attr, (keyword, cls) in HEADER_ATTR.items():
obj = descriptor
for attr in full_attr.split('.')[:-1]:
obj = getattr(obj, attr)
for attr, (keyword, cls) in HEADER_ATTR.items():
setattr(descriptor, attr, cls(header.get(keyword, HEADER_DEFAULT.get(attr))))
setattr(obj, full_attr.split('.')[-1], cls(header.get(keyword, HEADER_DEFAULT.get(full_attr))))
if version_index is not None and version_index != 1:
raise ValueError("The 'version' header must be in the second position")
......@@ -202,6 +282,15 @@ class BandwidthFile(Descriptor):
:var int min_count: minimum eligible relays for results to be provided
:var int min_percent: minimum measured percentage of the consensus
:var str scanner_country: country code where this scan took place
:var list destinations_countries: all country codes that were scanned
:var int time_to_report_half_network: estimated number of seconds required to
measure half the network, given recent measurements
:var RecentStats recent_stats: statistical information collected over the
last 'data_period' (by default five days)
**\*** attribute is either required when we're parsed with validation or has
a default value, others are left as **None** if undefined
"""
......
......@@ -52,6 +52,33 @@ EXPECTED_MEASUREMENT_2 = {
'error_misc': '0',
}
EXPECTED_MEASUREMENT_3 = {
'desc_bw_obs_last': '70423',
'success': '13',
'desc_bw_obs_mean': '81784',
'bw_median': '2603',
'nick': 'whitsun',
'bw': '1',
'desc_bw_avg': '1073741824',
'time': '2019-04-21T10:22:16',
'bw_mean': '2714',
'error_circ': '1',
'error_stream': '0',
'node_id': '$8F0F49F2341C7F706D5B475815DBD3E5761334B3',
'error_misc': '0',
'consensus_bandwidth': '1000',
'consensus_bandwidth_is_unmeasured': 'False',
'desc_bw_bur': '1073741824',
'error_destination': '0',
'error_second_relay': '0',
'master_key_ed25519': 'acShTw35dmVSTkhMdmo9RFRLsP4QV+qOZrEJQubnvWY',
'relay_in_recent_consensus_count': '22',
'relay_recent_measurement_attempt_count': '1',
'relay_recent_measurements_excluded_error_count': '1',
'relay_recent_priority_list_count': '1',
'success': '3',
}
EXPECTED_NEW_HEADER_CONTENT = """
1410723598
version=1.1.0
......@@ -103,6 +130,23 @@ class TestBandwidthFile(unittest.TestCase):
self.assertEqual(None, desc.min_count)
self.assertEqual(None, desc.min_percent)
self.assertEqual(None, desc.scanner_country)
self.assertEqual(None, desc.destinations_countries)
self.assertEqual(None, desc.time_to_report_half_network)
stats = desc.recent_stats
self.assertEqual(None, stats.consensus_count)
self.assertEqual(None, stats.prioritized_relays)
self.assertEqual(None, stats.prioritized_relay_lists)
self.assertEqual(None, stats.measurement_attempts)
self.assertEqual(None, stats.measurement_failures)
relay_failures = stats.relay_failures
self.assertEqual(None, relay_failures.no_measurement)
self.assertEqual(None, relay_failures.insuffient_period)
self.assertEqual(None, relay_failures.insufficient_measurements)
self.assertEqual(None, relay_failures.stale)
self.assertEqual(94, len(desc.measurements))
self.assertEqual(EXPECTED_MEASUREMENT_1, desc.measurements['D8B9CAA5B818DEFE80857F83FDABBB6429DCFCA0'])
......@@ -130,9 +174,70 @@ class TestBandwidthFile(unittest.TestCase):
self.assertEqual(3908, desc.min_count)
self.assertEqual(60, desc.min_percent)
self.assertEqual(None, desc.scanner_country)
self.assertEqual(None, desc.destinations_countries)
self.assertEqual(None, desc.time_to_report_half_network)
stats = desc.recent_stats
self.assertEqual(None, stats.consensus_count)
self.assertEqual(None, stats.prioritized_relays)
self.assertEqual(None, stats.prioritized_relay_lists)
self.assertEqual(None, stats.measurement_attempts)
self.assertEqual(None, stats.measurement_failures)
relay_failures = stats.relay_failures
self.assertEqual(None, relay_failures.no_measurement)
self.assertEqual(None, relay_failures.insuffient_period)
self.assertEqual(None, relay_failures.insufficient_measurements)
self.assertEqual(None, relay_failures.stale)
self.assertEqual(81, len(desc.measurements))
self.assertEqual(EXPECTED_MEASUREMENT_2, desc.measurements['9C7E1AFDACC53228F6FB57B3A08C7D36240B8F6F'])
def test_format_v1_4(self):
"""
Parse version 1.4 formatted files.
"""
desc = list(stem.descriptor.parse_file(get_resource('bandwidth_file_v1.4'), 'bandwidth-file 1.4'))[0]
self.assertEqual(datetime.datetime(2019, 4, 21, 21, 34, 57), desc.timestamp)
self.assertEqual('1.4.0', desc.version)
self.assertEqual('sbws', desc.software)
self.assertEqual('1.1.0', desc.software_version)
self.assertEqual(datetime.datetime(2019, 4, 16, 21, 35, 7), desc.earliest_bandwidth)
self.assertEqual(datetime.datetime(2019, 4, 21, 21, 34, 57), desc.latest_bandwidth)
self.assertEqual(datetime.datetime(2019, 4, 21, 21, 35, 4), desc.created_at)
self.assertEqual(datetime.datetime(2019, 4, 20, 11, 40, 1), desc.generated_at)
self.assertEqual(6684, desc.consensus_size)
self.assertEqual(6459, desc.eligible_count)
self.assertEqual(97, desc.eligible_percent)
self.assertEqual(4010, desc.min_count)
self.assertEqual(60, desc.min_percent)
self.assertEqual('US', desc.scanner_country)
self.assertEqual(['ZZ'], desc.destinations_countries)
self.assertEqual(223519, desc.time_to_report_half_network)
stats = desc.recent_stats
self.assertEqual(34, stats.consensus_count)
self.assertEqual(86417, stats.prioritized_relays)
self.assertEqual(260, stats.prioritized_relay_lists)
self.assertEqual(86417, stats.measurement_attempts)
self.assertEqual(57023, stats.measurement_failures)
relay_failures = stats.relay_failures
self.assertEqual(788, relay_failures.no_measurement)
self.assertEqual(182, relay_failures.insuffient_period)
self.assertEqual(663, relay_failures.insufficient_measurements)
self.assertEqual(0, relay_failures.stale)
self.assertEqual(58, len(desc.measurements))
self.assertEqual(EXPECTED_MEASUREMENT_3, desc.measurements['8F0F49F2341C7F706D5B475815DBD3E5761334B3'])
@patch('time.time', Mock(return_value = 1410723598.276578))
def test_minimal_bandwidth_file(self):
"""
......
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment