Check for new tor capabilities while testing

Testing improvement from clv. As part of testing we now check to see if tor
supports capabilities stem doesn't...

  https://trac.torproject.org/projects/tor/ticket/8250

If so we present this as part of our test output. For example...

  Your version of Tor has capabilities stem presently isn't taking advantage of.
  If you're running the latest version of stem then please file a ticket on:

    https://trac.torproject.org/projects/tor/wiki/doc/stem/bugs

  New capabilities are:

    [Event] CONN_BW
    [Event] CELL_STATS
    [Event] TB_EMPTY
    [Event] CIRC_BW
    [Event] TRANSPORT_LAUNCHED

8 files changed, 135 insertions, 77 deletions

diff --git a/run_tests.py b/run_tests.py
index e4478669..824b16ce 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -58,6 +58,15 @@ version 0.8.0 or later...
 https://pypi.python.org/pypi/mock/
 """
 
+NEW_CAPABILITIES_FOUND = """\
+Your version of Tor has capabilities stem presently isn't taking advantage of.
+If you're running the latest version of stem then please file a ticket on:
+
+  https://trac.torproject.org/projects/tor/wiki/doc/stem/bugs
+
+New capabilities are:
+"""
+
 PYFLAKES_TASK = Task(
   'running pyflakes',
   stem.util.test_tools.pyflakes_issues,
@@ -288,6 +297,14 @@ def main():
 
     println('TESTING PASSED %s\n' % runtime_label, SUCCESS)
 
+  new_capabilities = test.util.get_new_capabilities()
+
+  if new_capabilities:
+    println(NEW_CAPABILITIES_FOUND, ERROR)
+
+    for capability_type, msg in new_capabilities:
+      println('  [%s] %s' % (capability_type, msg), ERROR)
+
   sys.exit(1 if error_tracker.has_errors_occured() else 0)
 
 
diff --git a/stem/control.py b/stem/control.py
index 68883d99..32e50d55 100644
--- a/stem/control.py
+++ b/stem/control.py
@@ -171,37 +171,42 @@ If you're fine with allowing your script to raise exceptions then this can be mo
   Enums are mapped to :class:`~stem.response.events.Event` subclasses as
   follows...
 
-  ===================== ===========
-  EventType             Event Class
-  ===================== ===========
-  **ADDRMAP**           :class:`stem.response.events.AddrMapEvent`
-  **AUTHDIR_NEWDESCS**  :class:`stem.response.events.AuthDirNewDescEvent`
-  **BUILDTIMEOUT_SET**  :class:`stem.response.events.BuildTimeoutSetEvent`
-  **BW**                :class:`stem.response.events.BandwidthEvent`
-  **CIRC**              :class:`stem.response.events.CircuitEvent`
-  **CIRC_MINOR**        :class:`stem.response.events.CircMinorEvent`
-  **CLIENTS_SEEN**      :class:`stem.response.events.ClientsSeenEvent`
-  **CONF_CHANGED**      :class:`stem.response.events.ConfChangedEvent`
-  **DEBUG**             :class:`stem.response.events.LogEvent`
-  **DESCCHANGED**       :class:`stem.response.events.DescChangedEvent`
-  **ERR**               :class:`stem.response.events.LogEvent`
-  **GUARD**             :class:`stem.response.events.GuardEvent`
-  **HS_DESC**           :class:`stem.response.events.HSDescEvent`
-  **HS_DESC_CONTENT**   :class:`stem.response.events.HSDescContentEvent`
-  **INFO**              :class:`stem.response.events.LogEvent`
-  **NEWCONSENSUS**      :class:`stem.response.events.NewConsensusEvent`
-  **NEWDESC**           :class:`stem.response.events.NewDescEvent`
-  **NOTICE**            :class:`stem.response.events.LogEvent`
-  **NS**                :class:`stem.response.events.NetworkStatusEvent`
-  **ORCONN**            :class:`stem.response.events.ORConnEvent`
-  **SIGNAL**            :class:`stem.response.events.SignalEvent`
-  **STATUS_CLIENT**     :class:`stem.response.events.StatusEvent`
-  **STATUS_GENERAL**    :class:`stem.response.events.StatusEvent`
-  **STATUS_SERVER**     :class:`stem.response.events.StatusEvent`
-  **STREAM**            :class:`stem.response.events.StreamEvent`
-  **STREAM_BW**         :class:`stem.response.events.StreamBwEvent`
-  **WARN**              :class:`stem.response.events.LogEvent`
-  ===================== ===========
+  ======================= ===========
+  EventType               Event Class
+  ======================= ===========
+  **ADDRMAP**             :class:`stem.response.events.AddrMapEvent`
+  **AUTHDIR_NEWDESCS**    :class:`stem.response.events.AuthDirNewDescEvent`
+  **BUILDTIMEOUT_SET**    :class:`stem.response.events.BuildTimeoutSetEvent`
+  **BW**                  :class:`stem.response.events.BandwidthEvent`
+  **CELL_STATS**          :class:`stem.response.events.CellStatsEvent`
+  **CIRC**                :class:`stem.response.events.CircuitEvent`
+  **CIRC_BW**             :class:`stem.response.events.CircuitBandwidthEvent`
+  **CIRC_MINOR**          :class:`stem.response.events.CircMinorEvent`
+  **CLIENTS_SEEN**        :class:`stem.response.events.ClientsSeenEvent`
+  **CONF_CHANGED**        :class:`stem.response.events.ConfChangedEvent`
+  **CONN_BW**             :class:`stem.response.events.ConnectionBandwidthEvent`
+  **DEBUG**               :class:`stem.response.events.LogEvent`
+  **DESCCHANGED**         :class:`stem.response.events.DescChangedEvent`
+  **ERR**                 :class:`stem.response.events.LogEvent`
+  **GUARD**               :class:`stem.response.events.GuardEvent`
+  **HS_DESC**             :class:`stem.response.events.HSDescEvent`
+  **HS_DESC_CONTENT**     :class:`stem.response.events.HSDescContentEvent`
+  **INFO**                :class:`stem.response.events.LogEvent`
+  **NEWCONSENSUS**        :class:`stem.response.events.NewConsensusEvent`
+  **NEWDESC**             :class:`stem.response.events.NewDescEvent`
+  **NOTICE**              :class:`stem.response.events.LogEvent`
+  **NS**                  :class:`stem.response.events.NetworkStatusEvent`
+  **ORCONN**              :class:`stem.response.events.ORConnEvent`
+  **SIGNAL**              :class:`stem.response.events.SignalEvent`
+  **STATUS_CLIENT**       :class:`stem.response.events.StatusEvent`
+  **STATUS_GENERAL**      :class:`stem.response.events.StatusEvent`
+  **STATUS_SERVER**       :class:`stem.response.events.StatusEvent`
+  **STREAM**              :class:`stem.response.events.StreamEvent`
+  **STREAM_BW**           :class:`stem.response.events.StreamBwEvent`
+  **TB_EMPTY**            :class:`stem.response.events.TokenBucketEmptyEvent`
+  **TRANSPORT_LAUNCHED**  :class:`stem.response.events.TransportLaunchedEvent`
+  **WARN**                :class:`stem.response.events.LogEvent`
+  ======================= ===========
 
 .. data:: Listener (enum)
 
@@ -269,9 +274,12 @@ EventType = stem.util.enum.UppercaseEnum(
   'AUTHDIR_NEWDESCS',
   'BUILDTIMEOUT_SET',
   'BW',
+  'CELL_STATS',
   'CIRC',
+  'CIRC_BW',
   'CIRC_MINOR',
   'CONF_CHANGED',
+  'CONN_BW',
   'CLIENTS_SEEN',
   'DEBUG',
   'DESCCHANGED',
@@ -291,6 +299,8 @@ EventType = stem.util.enum.UppercaseEnum(
   'STATUS_SERVER',
   'STREAM',
   'STREAM_BW',
+  'TB_EMPTY',
+  'TRANSPORT_LAUNCHED',
   'WARN',
 )
 
diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py
index ca18292d..7ce3fdc4 100644
--- a/test/integ/control/controller.py
+++ b/test/integ/control/controller.py
@@ -27,10 +27,13 @@ from stem.control import EventType, Listener, State
 from stem.exit_policy import ExitPolicy
 from stem.version import Requirement
 
+from test.util import register_new_capability
+
 from test.runner import (
   require_controller,
   require_version,
   require_online,
+  only_run_once,
 )
 
 # Router status entry for a relay with a nickname other than 'Unnamed'. This is
@@ -48,6 +51,29 @@ def random_fingerprint():
 
 
 class TestController(unittest.TestCase):
+  @only_run_once
+  @require_controller
+  def test_missing_capabilities(self):
+    """
+    Check to see if tor supports any events, signals, or features that we
+    don't.
+    """
+
+    with test.runner.get_runner().get_tor_controller() as controller:
+      for event in controller.get_info('events/names').split():
+        if event not in EventType:
+          register_new_capability('Event', event)
+
+      for signal in controller.get_info('signal/names').split():
+        if signal not in Signal:
+          register_new_capability('Signal', signal)
+
+      # new features should simply be added to enable_feature()'s docs
+
+      for feature in controller.get_info('features/names').split():
+        if feature not in ('EXTENDED_EVENTS', 'VERBOSE_NAMES'):
+          register_new_capability('Feature', feature)
+
   def test_from_port(self):
     """
     Basic sanity check for the from_port constructor.
@@ -1084,10 +1110,8 @@ class TestController(unittest.TestCase):
         self.assertTrue(desc.fingerprint is not None)
         self.assertTrue(desc.nickname is not None)
 
-        unrecognized_lines = desc.get_unrecognized_lines()
-
-        if unrecognized_lines:
-          self.fail('Unrecognized descriptor content: %s' % unrecognized_lines)
+        for line in desc.get_unrecognized_lines():
+          register_new_capability('Consensus Line', line)
 
         count += 1
         if count > 10:
diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py
index d102fea3..740ac59a 100644
--- a/test/integ/descriptor/extrainfo_descriptor.py
+++ b/test/integ/descriptor/extrainfo_descriptor.py
@@ -9,6 +9,7 @@ import stem.descriptor
 import test.runner
 
 from test.runner import only_run_once
+from test.util import register_new_capability
 
 
 class TestExtraInfoDescriptor(unittest.TestCase):
@@ -28,7 +29,8 @@ class TestExtraInfoDescriptor(unittest.TestCase):
 
     with open(descriptor_path, 'rb') as descriptor_file:
       for desc in stem.descriptor.parse_file(descriptor_file, 'extra-info 1.0', validate = True):
-        unrecognized_lines = desc.get_unrecognized_lines()
+        for line in desc.get_unrecognized_lines():
+          register_new_capability('Extra-info Line', line)
 
         if desc.dir_v2_responses_unknown:
           self.fail('Unrecognized statuses on dirreq-v2-resp lines: %s' % desc.dir_v2_responses_unknown)
@@ -40,11 +42,3 @@ class TestExtraInfoDescriptor(unittest.TestCase):
           self.fail('Unrecognized stats on dirreq-v3-direct-dl lines: %s' % desc.dir_v2_direct_dl_unknown)
         elif desc.dir_v2_tunneled_dl_unknown:
           self.fail('Unrecognized stats on dirreq-v2-tunneled-dl lines: %s' % desc.dir_v2_tunneled_dl_unknown)
-        elif unrecognized_lines:
-          # TODO: This isn't actually a problem, and rather than failing we
-          # should alert the user about these entries at the end of the tests
-          # (along with new events, getinfo options, and such). For now though
-          # there doesn't seem to be anything in practice to trigger this so
-          # failing to get our attention if it does.
-
-          self.fail('Unrecognized descriptor content: %s' % unrecognized_lines)
diff --git a/test/integ/descriptor/microdescriptor.py b/test/integ/descriptor/microdescriptor.py
index ebd02877..fad7ef85 100644
--- a/test/integ/descriptor/microdescriptor.py
+++ b/test/integ/descriptor/microdescriptor.py
@@ -9,6 +9,7 @@ import stem.descriptor
 import test.runner
 
 from test.runner import only_run_once
+from test.util import register_new_capability
 
 
 class TestMicrodescriptor(unittest.TestCase):
@@ -28,7 +29,5 @@ class TestMicrodescriptor(unittest.TestCase):
 
     with open(descriptor_path, 'rb') as descriptor_file:
       for desc in stem.descriptor.parse_file(descriptor_file, 'microdescriptor 1.0', validate = True):
-        unrecognized_lines = desc.get_unrecognized_lines()
-
-        if unrecognized_lines:
-          self.fail('Unrecognized microdescriptor content: %s' % unrecognized_lines)
+        for line in desc.get_unrecognized_lines():
+          register_new_capability('Microdescriptor Line', line)
diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py
index 8f97b26e..e02f1784 100644
--- a/test/integ/descriptor/networkstatus.py
+++ b/test/integ/descriptor/networkstatus.py
@@ -12,6 +12,7 @@ import stem.version
 import test.runner
 
 from test.runner import only_run_once
+from test.util import register_new_capability
 
 
 class TestNetworkStatus(unittest.TestCase):
@@ -33,22 +34,19 @@ class TestNetworkStatus(unittest.TestCase):
       test.runner.skip(self, '(unavailable on windows)')
       return
 
-    count = 0
+    count, reported_flags = 0, []
+
     with open(consensus_path, 'rb') as descriptor_file:
       for router in stem.descriptor.parse_file(descriptor_file, 'network-status-consensus-3 1.0', validate = True):
         count += 1
 
-        # check if there's any unknown flags
-        # TODO: this should be a 'new capability' check later rather than
-        # failing the tests
         for flag in router.flags:
-          if flag not in stem.Flag:
-            raise ValueError('Unrecognized flag type: %s, found on relay %s (%s)' % (flag, router.fingerprint, router.nickname))
-
-        unrecognized_lines = router.get_unrecognized_lines()
+          if flag not in stem.Flag and flag not in reported_flags:
+            register_new_capability('Flag', flag)
+            reported_flags.append(flag)
 
-        if unrecognized_lines:
-          self.fail('Unrecognized descriptor content: %s' % unrecognized_lines)
+        for line in router.get_unrecognized_lines():
+          register_new_capability('Consensus Line', line)
 
     # Sanity test that there's at least a hundred relays. If that's not the
     # case then this probably isn't a real, complete tor consensus.
@@ -70,21 +68,18 @@ class TestNetworkStatus(unittest.TestCase):
       test.runner.skip(self, '(unavailable on windows)')
       return
 
-    count = 0
+    count, reported_flags = 0, []
+
     with open(consensus_path, 'rb') as descriptor_file:
       for router in stem.descriptor.parse_file(descriptor_file, 'network-status-microdesc-consensus-3 1.0', validate = True):
         count += 1
 
-        # check if there's any unknown flags
-        # TODO: this should be a 'new capability' check later rather than
-        # failing the tests
         for flag in router.flags:
           if flag not in stem.Flag:
-            raise ValueError('Unrecognized flag type: %s, found on microdescriptor relay %s (%s)' % (flag, router.fingerprint, router.nickname))
-
-        unrecognized_lines = router.get_unrecognized_lines()
+            register_new_capability('Flag (microdescriptor)', flag)
+            reported_flags.append(flag)
 
-        if unrecognized_lines:
-          self.fail('Unrecognized descriptor content: %s' % unrecognized_lines)
+        for line in router.get_unrecognized_lines():
+          register_new_capability('Microdescriptor Consensus Line', line)
 
     self.assertTrue(count > 100)
diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py
index 6d5d3486..86e4942c 100644
--- a/test/integ/descriptor/server_descriptor.py
+++ b/test/integ/descriptor/server_descriptor.py
@@ -10,6 +10,7 @@ import stem.descriptor
 import test.runner
 
 from test.runner import only_run_once
+from test.util import register_new_capability
 
 
 class TestServerDescriptor(unittest.TestCase):
@@ -35,13 +36,5 @@ class TestServerDescriptor(unittest.TestCase):
         self.assertEqual(None, desc.eventdns)
         self.assertEqual(None, desc.socks_port)
 
-        unrecognized_lines = desc.get_unrecognized_lines()
-
-        if unrecognized_lines:
-          # TODO: This isn't actually a problem, and rather than failing we
-          # should alert the user about these entries at the end of the tests
-          # (along with new events, getinfo options, and such). For now though
-          # there doesn't seem to be anything in practice to trigger this so
-          # failing to get our attention if it does.
-
-          self.fail('Unrecognized descriptor content: %s' % unrecognized_lines)
+        for line in desc.get_unrecognized_lines():
+          register_new_capability('Server Descriptor Line', line)
diff --git a/test/util.py b/test/util.py
index 5614e4f1..87fc47ed 100644
--- a/test/util.py
+++ b/test/util.py
@@ -80,6 +80,10 @@ Target = stem.util.enum.UppercaseEnum(
 
 STEM_BASE = os.path.sep.join(__file__.split(os.path.sep)[:-2])
 
+# Store new capabilities (events, descriptor entries, etc.)
+
+NEW_CAPABILITIES = []
+
 
 def get_unit_tests(module_prefix = None):
   """
@@ -175,6 +179,17 @@ def get_torrc_entries(target):
   return torrc_opts
 
 
+def get_new_capabilities():
+  """
+  Provides a list of capabilities tor supports but stem doesn't, as discovered
+  while running our tests.
+
+  :returns: **list** of (type, message) tuples for the capabilities
+  """
+
+  return NEW_CAPABILITIES
+
+
 def check_stem_version():
   return stem.__version__
 
@@ -265,6 +280,17 @@ def check_for_unused_tests(paths):
     raise ValueError('Test modules are missing from our test/settings.cfg:\n%s' % '\n'.join(unused_tests))
 
 
+def register_new_capability(capability_type, msg):
+  """
+  Register new capability found during the tests.
+
+  :param str capability_type: type of capability this is
+  :param str msg: description of what we found
+  """
+
+  NEW_CAPABILITIES.append((capability_type, msg))
+
+
 def _is_test_data(path):
   return os.path.normpath(CONFIG['integ.test_directory']) in path