Check consensus signatures

We already validate crypto of most descriptor types but lacked this check for arguably the most important thing: the consensus. This requires key certificates so unlike other descriptors this isn't validated by default. Rather, callers need to call validate_signatures() with the authority certificates. This branch has been a collaboration between Tyler and Damian over a couple weeks of bouncing remotes back and forth. :P https://trac.torproject.org/projects/tor/ticket/11045

Check consensus signatures
8c84eea7 · Tyler Parks · Damian Johnson · cf6df8bb · 8c84eea7 · 8c84eea7
Commit 8c84eea7 authored May 15, 2017 by Tyler Parks Committed by Damian Johnson May 15, 2017
--- a/docs/change_log.rst
+++ b/docs/change_log.rst
@@ -55,6 +55,7 @@ The following are only available within Stem's `git repository

  * Supporting `descriptor creation <tutorials/mirror_mirror_on_the_wall.html#can-i-create-descriptors>`_ (:trac:`10227`)
  * Support and validation for `ed25519 certificates <api/descriptor/certificate.html>`_ (`spec <https://gitweb.torproject.org/torspec.git/tree/cert-spec.txt>`_, :trac:`21558`)
+  * Added :func:`~stem.descriptor.networkstatus.NetworkStatusDocumentV3.validate_signatures` to check our key certificate signatures (:trac:`11045`)
  * Moved from the deprecated `pycrypto <https://www.dlitz.net/software/pycrypto/>`_ module to `cryptography <https://pypi.python.org/pypi/cryptography>`_ for validating signatures (:trac:`21086`)
  * Sped descriptor reading by ~25% by deferring defaulting when validating
  * Added server descriptor's new extra_info_sha256_digest attribute (:spec:`0f03581`)

--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -681,17 +681,14 @@ class Descriptor(object):
    from cryptography.hazmat.backends import default_backend
    from cryptography.hazmat.primitives.serialization import load_der_public_key
    from cryptography.utils import int_to_bytes, int_from_bytes
-
    key = load_der_public_key(_bytes_for_block(signing_key), default_backend())
    modulus = key.public_numbers().n
    public_exponent = key.public_numbers().e
-
    sig_as_bytes = _bytes_for_block(signature)
    sig_as_long = int_from_bytes(sig_as_bytes, byteorder='big')  # convert signature to an int
-    blocksize = 128  # block size will always be 128 for a 1024 bit key
+    blocksize = len(sig_as_bytes)  # 256B for NetworkStatusDocuments, 128B for others

    # use the public exponent[e] & the modulus[n] to decrypt the int
-
    decrypted_int = pow(sig_as_long, public_exponent, modulus)

    # convert the int to a byte array
@@ -708,7 +705,6 @@ class Descriptor(object):
    # More info here http://www.ietf.org/rfc/rfc2313.txt
    #                esp the Notes in section 8.1
    ############################################################################
-
    try:
      if decrypted_bytes.index(b'\x00\x01') != 0:
        raise ValueError('Verification failed, identifier missing')

--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -1055,6 +1055,39 @@ class NetworkStatusDocumentV3(NetworkStatusDocument):
    self.routers = dict((desc.fingerprint, desc) for desc in router_iter)
    self._footer(document_file, validate)

+  def validate_signatures(self, key_certs):
+    """
+    Validates we're properly signed by the signing certificates.
+
+    .. versionadded:: 1.6.0
+
+    :param list key_certs: :class:`~stem.descriptor.networkstatus.KeyCertificates`
+      to validate the consensus against
+
+    :raises: **ValueError** if an insufficient number of valid signatures are present.
+    """
+
+    # sha1 hash of the body and header
+
+    local_digest = self._digest_for_content(b'network-status-version', b'directory-signature ')
+
+    valid_digests, total_digests = 0, 0
+    required_digests = len(self.signatures) / 2.0
+    signing_keys = dict([(cert.fingerprint, cert.signing_key) for cert in key_certs])
+
+    for sig in self.signatures:
+      if sig.identity not in signing_keys:
+        continue
+
+      signed_digest = self._digest_for_signature(signing_keys[sig.identity], sig.signature)
+      total_digests += 1
+
+      if signed_digest == local_digest:
+        valid_digests += 1
+
+    if valid_digests < required_digests:
+      raise ValueError('Network Status Document has %i valid signatures out of %i total, needed %i' % (valid_digests, total_digests, required_digests))
+
  def get_unrecognized_lines(self):
    if self._lazy_loading:
      self._parse(self._header_entries, False, parser_for_line = self.HEADER_PARSER_FOR_LINE)

--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -98,6 +98,7 @@ except ImportError:
  import urllib2 as urllib

 import stem.descriptor
+import stem.prereq

 from stem import Flag
 from stem.util import _hash_attr, connection, log, str_tools, tor_tools
@@ -628,7 +629,17 @@ class DescriptorDownloader(object):
    if authority_v3ident:
      resource += '/%s' % authority_v3ident

-    return self.query(resource + '.z', **query_args)
+    consensus_query = self.query(resource + '.z', **query_args)
+
+    # if we're performing validation then check that it's signed by the
+    # authority key certificates
+
+    if consensus_query.validate and consensus_query.document_handler == stem.descriptor.DocumentHandler.DOCUMENT and stem.prereq.is_crypto_available():
+      consensus = list(consensus_query.run())[0]
+      key_certs = self.get_key_certificates(**query_args).run()
+      consensus.validate_signatures(key_certs)
+
+    return consensus_query

  def get_vote(self, authority, **query_args):
    """

--- a/test/integ/descriptor/__init__.py
+++ b/test/integ/descriptor/__init__.py
@@ -5,7 +5,7 @@ Integration tests for stem.descriptor.* contents.
 __all__ = [
  'extrainfo_descriptor',
  'microdescriptor',
+  'networkstatus',
+  'remote'
  'server_descriptor',
-  'get_resource',
-  'open_desc',
 ]
--- a/test/integ/descriptor/networkstatus.py
+++ b/test/integ/descriptor/networkstatus.py
@@ -7,17 +7,30 @@ import unittest

 import stem
 import stem.descriptor
-import stem.descriptor.networkstatus
+import stem.descriptor.remote
 import stem.version
 import test.runner

 from test.util import (
  register_new_capability,
  only_run_once,
+  require_cryptography,
+  require_online,
 )


 class TestNetworkStatus(unittest.TestCase):
+  @require_online
+  @require_cryptography
+  @only_run_once
+  def test_signature_validation(self):
+    """
+    The full consensus is pretty sizable so rather than storing a copy of it
+    using the remote module. Chekcing the signature on the current consensus.
+    """
+
+    stem.descriptor.remote.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT, validate = True).run()
+
  @only_run_once
  def test_cached_consensus(self):
    """