diff options
| author | Philipp Winter <phw@nymity.ch> | 2020-02-19 10:08:03 -0800 |
|---|---|---|
| committer | Philipp Winter <phw@nymity.ch> | 2020-02-19 10:08:03 -0800 |
| commit | c3a820daeffa1acc02b9c0458ba8495ff387ecd4 (patch) | |
| tree | 8ea30b1ab83aed095405c16f4a356be6697879bc | |
| parent | 7b142b899766dea9c503fa3bdf999421ee4b7f37 (diff) | |
| parent | 1f614896ed3442c8758e7ea10bef0c1fc366a4c1 (diff) | |
Merge branch 'defect/30946' into develop
77 files changed, 693 insertions, 699 deletions
diff --git a/.travis.requirements.txt b/.travis.requirements.txt index 6f3dec7..1a1d7ff 100644 --- a/.travis.requirements.txt +++ b/.travis.requirements.txt @@ -15,7 +15,7 @@ #------------------------------------------------------------------------------ attrs==19.2.0 Babel==2.8.0 -BeautifulSoup==3.2.2 +beautifulsoup4==4.8.2 Mako==1.1.1 pycryptodome==3.9.6 Twisted==19.10.0 @@ -25,6 +25,7 @@ gnupg==2.3.1 ipaddr==2.2.0 mechanize==0.4.5 Pillow==6.2.2 +pyOpenSSL==19.0.0 pygeoip==0.3.2 qrcode==6.1 service_identity==18.1.0 diff --git a/.travis.yml b/.travis.yml index c0499b0..e57481d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ notifications: on_failure: change python: - - "2.7" + - "3.7" addons: hosts: @@ -1,3 +1,7 @@ + * FIXES https://bugs.torproject.org/30946 + This patch ports BridgeDB to Python 3. Python 2 is no longer supported + since Jan 1, 2020. + Changes in version 0.9.3 - 2020-02-18 * FIXES <https://bugs.torproject.org/33299> diff --git a/MANIFEST.in b/MANIFEST.in index bbe1e74..bb16c8c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include versioneer.py include bridgedb/_version.py include requirements.txt -recursice-include bridgedb/i18n *.po *.pot +recursive-include bridgedb/i18n *.po *.pot diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py index 3c94fdf..c29a2bf 100644 --- a/bridgedb/Bridges.py +++ b/bridgedb/Bridges.py @@ -10,6 +10,7 @@ them into hashrings for distributors. """ +import binascii import bisect import logging import re @@ -28,12 +29,6 @@ from bridgedb.parse.fingerprint import isValidFingerprint from bridgedb.parse.fingerprint import toHex from bridgedb.safelog import logSafely -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - - ID_LEN = 20 # XXX Only used in commented out line in Storage.py DIGEST_LEN = 20 PORTSPEC_LEN = 16 @@ -335,7 +330,7 @@ class BridgeRing(object): else: logging.debug( "Got duplicate bridge %r in main hashring for position %r." - % (logSafely(k.encode('hex')), pos.encode('hex'))) + % (logSafely(binascii.hexlify(k).decode('utf-8')), binascii.hexlify(pos).decode('utf-8'))) keys.sort() if filterBySubnet: @@ -361,7 +356,7 @@ class BridgeRing(object): def dumpAssignments(self, f, description=""): logging.info("Dumping bridge assignments for %s..." % self.name) - for b in self.bridges.itervalues(): + for b in self.bridges.values(): desc = [ description ] for tp,val,_,subring in self.subrings: if subring.getBridgeByID(b.identity): @@ -380,7 +375,7 @@ class FixedBridgeSplitter(object): def insert(self, bridge): # Grab the first 4 bytes digest = self.hmac(bridge.identity) - pos = long( digest[:8], 16 ) + pos = int( digest[:8], 16 ) which = pos % len(self.rings) self.rings[which].insert(bridge) @@ -405,7 +400,7 @@ class FixedBridgeSplitter(object): description is ``"IPv6 obfs2 bridges"`` the line would read: ``"IPv6 obfs2 bridges ring=3"``. """ - for index, ring in zip(xrange(len(self.rings)), self.rings): + for index, ring in zip(range(len(self.rings)), self.rings): ring.dumpAssignments(filename, "%s ring=%s" % (description, index)) @@ -544,7 +539,7 @@ class BridgeSplitter(object): logging.info("Current rings: %s" % " ".join(self.ringsByName)) def dumpAssignments(self, f, description=""): - for name,ring in self.ringsByName.iteritems(): + for name,ring in self.ringsByName.items(): ring.dumpAssignments(f, "%s %s" % (description, name)) @@ -633,8 +628,8 @@ class FilteredBridgeSplitter(object): """ filterNames = [] - for filterName in [x.func_name for x in list(ringname)]: - # Using `assignBridgesToSubring.func_name` gives us a messy + for filterName in [x.__name__ for x in list(ringname)]: + # Using `assignBridgesToSubring.__name__` gives us a messy # string which includes all parameters and memory addresses. Get # rid of this by partitioning at the first `(`: realFilterName = filterName.partition('(')[0] diff --git a/bridgedb/Stability.py b/bridgedb/Stability.py index f2f99ce..34ea51d 100644 --- a/bridgedb/Stability.py +++ b/bridgedb/Stability.py @@ -33,7 +33,7 @@ from bridgedb.schedule import toUnixSeconds # tunables weighting_factor = float(19)/float(20) -discountIntervalMillis = long(60*60*12*1000) +discountIntervalMillis = 60*60*12*1000 class BridgeHistory(object): @@ -76,15 +76,15 @@ class BridgeHistory(object): self.fingerprint = fingerprint self.ip = ip self.port = port - self.weightedUptime = long(weightedUptime) - self.weightedTime = long(weightedTime) - self.weightedRunLength = long(weightedRunLength) + self.weightedUptime = int(weightedUptime) + self.weightedTime = int(weightedTime) + self.weightedRunLength = int(weightedRunLength) self.totalRunWeights = float(totalRunWeights) self.lastSeenWithDifferentAddressAndPort = \ - long(lastSeenWithDifferentAddressAndPort) - self.lastSeenWithThisAddressAndPort = long(lastSeenWithThisAddressAndPort) - self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues) - self.lastUpdatedWeightedTime = long(lastUpdatedWeightedTime) + int(lastSeenWithDifferentAddressAndPort) + self.lastSeenWithThisAddressAndPort = int(lastSeenWithThisAddressAndPort) + self.lastDiscountedHistoryValues = int(lastDiscountedHistoryValues) + self.lastUpdatedWeightedTime = int(lastUpdatedWeightedTime) def discountWeightedFractionalUptimeAndWeightedTime(self, discountUntilMillis): """ discount weighted times """ @@ -111,8 +111,8 @@ class BridgeHistory(object): @property def weightedFractionalUptime(self): """Weighted Fractional Uptime""" - if self.weightedTime <0.0001: return long(0) - return long(10000) * self.weightedUptime / self.weightedTime + if self.weightedTime <0.0001: return 0 + return 10000 * self.weightedUptime / self.weightedTime @property def tosa(self): @@ -127,7 +127,7 @@ class BridgeHistory(object): more recently than it, or if it has been around for a Weighted Time of 8 days. """ # if this bridge has been around longer than 8 days - if self.weightedTime >= long(8 * 24 * 60 * 60): + if self.weightedTime >= 8 * 24 * 60 * 60: return True # return True if self.weightedTime is greater than the weightedTime @@ -146,10 +146,10 @@ class BridgeHistory(object): """Weighted Mean Time Between Address Change""" totalRunLength = self.weightedRunLength + \ ((self.lastSeenWithThisAddressAndPort - - self.lastSeenWithDifferentAddressAndPort) / long(1000)) + self.lastSeenWithDifferentAddressAndPort) / 1000) totalWeights = self.totalRunWeights + 1.0 - if totalWeights < 0.0001: return long(0) + if totalWeights < 0.0001: return 0 assert(isinstance(long,totalRunLength)) assert(isinstance(long,totalWeights)) return totalRunlength / totalWeights @@ -159,9 +159,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp): bhe = db.getBridgeHistory(bridge.fingerprint) if not bhe: # This is the first status, assume 60 minutes. - secondsSinceLastStatusPublication = long(60*60) - lastSeenWithDifferentAddressAndPort = timestamp * long(1000) - lastSeenWithThisAddressAndPort = timestamp * long(1000) + secondsSinceLastStatusPublication = 60*60 + lastSeenWithDifferentAddressAndPort = timestamp * 1000 + lastSeenWithThisAddressAndPort = timestamp * 1000 bhe = BridgeHistory( bridge.fingerprint, bridge.address, bridge.orPort, @@ -179,9 +179,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp): # Calculate the seconds since the last parsed status. If this is # the first status or we haven't seen a status for more than 60 # minutes, assume 60 minutes. - statusPublicationMillis = long(timestamp * 1000) + statusPublicationMillis = timestamp * 1000 if (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort) > 60*60*1000: - secondsSinceLastStatusPublication = long(60*60) + secondsSinceLastStatusPublication = 60*60 logging.debug("Capping secondsSinceLastStatusPublication to 1 hour") # otherwise, roll with it else: @@ -278,7 +278,7 @@ def updateBridgeHistory(bridges, timestamps): logging.debug("Beginning bridge stability calculations") sortedTimestamps = {} - for fingerprint, stamps in timestamps.items()[:]: + for fingerprint, stamps in timestamps.items(): stamps.sort() bridge = bridges[fingerprint] for timestamp in stamps: diff --git a/bridgedb/Storage.py b/bridgedb/Storage.py index 4182c4b..cfd60bb 100644 --- a/bridgedb/Storage.py +++ b/bridgedb/Storage.py @@ -8,7 +8,6 @@ import binascii import sqlite3 import time import hashlib -from contextlib import GeneratorContextManager from functools import wraps from ipaddr import IPAddress import sys @@ -212,7 +211,7 @@ class Database(object): cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,)) def getEmailTime(self, addr): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() cur = self._cur cur.execute("SELECT when_mailed FROM EmailedBridges WHERE email = ?", (addr,)) v = cur.fetchone() @@ -221,7 +220,7 @@ class Database(object): return strToTime(v[0]) def setEmailTime(self, addr, whenMailed): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() cur = self._cur t = timeToStr(whenMailed) cur.execute("INSERT OR REPLACE INTO EmailedBridges " @@ -262,7 +261,7 @@ class Database(object): (distributor, hex_key)) def getWarnedEmail(self, addr): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() cur = self._cur cur.execute("SELECT * FROM WarnedEmails WHERE email = ?", (addr,)) v = cur.fetchone() @@ -271,7 +270,7 @@ class Database(object): return True def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() t = timeToStr(whenWarned) cur = self._cur if warned == True: @@ -345,11 +344,18 @@ def openDatabase(sqlite_file): return conn -class DBGeneratorContextManager(GeneratorContextManager): +class DBGeneratorContextManager(object): """Helper for @contextmanager decorator. Overload __exit__() so we can call the generator many times """ + + def __init__(self, gen): + self.gen = gen + + def __enter__(self): + return next(self.gen) + def __exit__(self, type, value, traceback): """Handle exiting a with statement block @@ -362,7 +368,7 @@ class DBGeneratorContextManager(GeneratorContextManager): """ if type is None: try: - self.gen.next() + next(self.gen) except StopIteration: return return @@ -374,7 +380,7 @@ class DBGeneratorContextManager(GeneratorContextManager): try: self.gen.throw(type, value, traceback) raise RuntimeError("generator didn't stop after throw()") - except StopIteration, exc: + except StopIteration as exc: # Suppress the exception *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration # raised inside the "with" statement from being suppressed diff --git a/bridgedb/bridgerequest.py b/bridgedb/bridgerequest.py index 7bd7b64..821bd7c 100644 --- a/bridgedb/bridgerequest.py +++ b/bridgedb/bridgerequest.py @@ -164,7 +164,7 @@ class BridgeRequestBase(object): # Get an HMAC with the key of the client identifier: digest = getHMACFunc(key)(client) # Take the lower 8 bytes of the digest and convert to a long: - position = long(digest[:8], 16) + position = int(digest[:8], 16) return position def isValid(self, valid=None): diff --git a/bridgedb/bridges.py b/bridgedb/bridges.py index ca1330b..1dc32af 100644 --- a/bridgedb/bridges.py +++ b/bridgedb/bridges.py @@ -240,7 +240,7 @@ class BridgeAddressBase(object): :param str value: The binary-encoded SHA-1 hash digest of the public half of this Bridge's identity key. """ - self.fingerprint = toHex(value) + self.fingerprint = toHex(value).decode('utf-8') @identity.deleter def identity(self): @@ -743,7 +743,7 @@ class BridgeBackwardsCompatibility(BridgeBase): if not fingerprint: if not len(idDigest) == 20: raise TypeError("Bridge with invalid ID") - self.fingerprint = toHex(idDigest) + self.fingerprint = toHex(idDigest).decode('utf-8') elif fingerprint: if not isValidFingerprint(fingerprint): raise TypeError("Bridge with invalid fingerprint (%r)" @@ -1037,7 +1037,7 @@ class Bridge(BridgeBackwardsCompatibility): if safelog.safe_logging: prefix = '$$' if fingerprint: - fingerprint = hashlib.sha1(fingerprint).hexdigest().upper() + fingerprint = hashlib.sha1(fingerprint.encode('utf-8')).hexdigest().upper() if not fingerprint: fingerprint = '0' * 40 @@ -1177,7 +1177,7 @@ class Bridge(BridgeBackwardsCompatibility): # their ``methodname`` matches the requested transport: transports = filter(lambda pt: pt.methodname == desired, self.transports) # Filter again for whichever of IPv4 or IPv6 was requested: - transports = filter(lambda pt: pt.address.version == ipVersion, transports) + transports = list(filter(lambda pt: pt.address.version == ipVersion, transports)) if not transports: raise PluggableTransportUnavailable( @@ -1377,7 +1377,7 @@ class Bridge(BridgeBackwardsCompatibility): :meth:`_getBlockKey`. :param str countryCode: A two-character country code specifier. """ - if self._blockedIn.has_key(key): + if key in self._blockedIn: self._blockedIn[key].append(countryCode.lower()) else: self._blockedIn[key] = [countryCode.lower(),] @@ -1665,7 +1665,7 @@ class Bridge(BridgeBackwardsCompatibility): logging.info("Verifying extrainfo signature for %s..." % self) # Get the bytes of the descriptor signature without the headers: - document, signature = descriptor.get_bytes().split(TOR_BEGIN_SIGNATURE) + document, signature = str(descriptor).split(TOR_BEGIN_SIGNATURE) signature = signature.replace(TOR_END_SIGNATURE, '') signature = signature.replace('\n', '') signature = signature.strip() @@ -1709,8 +1709,8 @@ class Bridge(BridgeBackwardsCompatibility): # This is the hexadecimal SHA-1 hash digest of the descriptor document # as it was signed: - signedDigest = codecs.encode(unpadded, 'hex_codec') - actualDigest = hashlib.sha1(document).hexdigest() + signedDigest = codecs.encode(unpadded, 'hex_codec').decode('utf-8') + actualDigest = hashlib.sha1(document.encode('utf-8')).hexdigest() except Exception as error: logging.debug("Error verifying extrainfo signature: %s" % error) diff --git a/bridgedb/captcha.py b/bridgedb/captcha.py index b66972c..2bdf6b9 100644 --- a/bridgedb/captcha.py +++ b/bridgedb/captcha.py @@ -63,11 +63,10 @@ import logging import random import os import time -import urllib2 +import urllib.request -from BeautifulSoup import BeautifulSoup - -from zope.interface import Interface, Attribute, implements +from bs4 import BeautifulSoup +from zope.interface import Interface, Attribute, implementer from bridgedb import crypto from bridgedb import schedule @@ -101,6 +100,7 @@ class ICaptcha(Interface): """Retrieve a new CAPTCHA image.""" +@implementer(ICaptcha) class Captcha(object): """A generic CAPTCHA base class. @@ -117,7 +117,6 @@ class Captcha(object): :ivar secretKey: A private key used for decrypting challenge strings during CAPTCHA solution verification. """ - implements(ICaptcha) def __init__(self, publicKey=None, secretKey=None): """Obtain a new CAPTCHA for a client.""" @@ -185,14 +184,14 @@ class ReCaptcha(Captcha): form = "/noscript?k=%s" % self.publicKey # Extract and store image from recaptcha - html = urllib2.urlopen(urlbase + form).read() + html = urllib.request.urlopen(urlbase + form).read() # FIXME: The remaining lines currently cannot be reliably unit tested: soup = BeautifulSoup(html) # pragma: no cover imgurl = urlbase + "/" + soup.find('img')['src'] # pragma: no cover cField = soup.find( # pragma: no cover 'input', {'name': 'recaptcha_challenge_field'}) # pragma: no cover self.challenge = str(cField['value']) # pragma: no cover - self.image = urllib2.urlopen(imgurl).read() # pragma: no cover + self.image = urllib.request.urlopen(imgurl).read() # pragma: no cover class GimpCaptcha(Captcha): @@ -271,6 +270,10 @@ class GimpCaptcha(Captcha): :returns: ``True`` if the CAPTCHA solution was correct and not stale. ``False`` otherwise. """ + + if isinstance(solution, bytes): + solution = solution.decode('utf-8') + hmacIsValid = False if not solution: @@ -290,12 +293,12 @@ class GimpCaptcha(Captcha): if hmacIsValid: try: answerBlob = secretKey.decrypt(encBlob) - timestamp = answerBlob[:12].lstrip('0') + timestamp = answerBlob[:12].lstrip(b'0') then = cls.sched.nextIntervalStarts(int(timestamp)) now = int(time.time()) - answer = answerBlob[12:] + answer = answerBlob[12:].decode('utf-8') except Exception as error: - logging.warn(error.message) + logging.warn(str(error)) else: # If the beginning of the 'next' interval (the interval # after the one when the CAPTCHA timestamp was created) @@ -367,10 +370,10 @@ class GimpCaptcha(Captcha): """ timestamp = str(int(time.time())).zfill(12) blob = timestamp + answer - encBlob = self.publicKey.encrypt(blob) + encBlob = self.publicKey.encrypt(blob.encode('utf-8')) hmac = crypto.getHMAC(self.hmacKey, encBlob) challenge = urlsafe_b64encode(hmac + encBlob) - return challenge + return challenge.decode("utf-8") def get(self): """Get a random CAPTCHA from the cache directory. @@ -388,7 +391,7 @@ class GimpCaptcha(Captcha): try: imageFilename = random.choice(os.listdir(self.cacheDir)) imagePath = os.path.join(self.cacheDir, imageFilename) - with open(imagePath) as imageFile: + with open(imagePath, 'rb') as imageFile: self.image = imageFile.read() except IndexError: raise GimpCaptchaError("CAPTCHA cache dir appears empty: %r" diff --git a/bridgedb/configure.py b/bridgedb/configure.py index 0bc4dd1..e069ea3 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -82,7 +82,7 @@ def loadConfig(configFile=None, configCls=None): if itsSafeToUseLogging: logging.info("Loading settings from config file: '%s'" % conffile) compiled = compile(open(conffile).read(), '<string>', 'exec') - exec compiled in configuration + exec(compiled, configuration) if itsSafeToUseLogging: logging.debug("New configuration settings:") diff --git a/bridgedb/crypto.py b/bridgedb/crypto.py index ac2e3ec..db3d083 100644 --- a/bridgedb/crypto.py +++ b/bridgedb/crypto.py @@ -47,7 +47,7 @@ import io import logging import os import re -import urllib +import urllib.parse import OpenSSL @@ -57,29 +57,9 @@ from Crypto.PublicKey import RSA from twisted.internet import ssl from twisted.python.procutils import which - #: The hash digest to use for HMACs. DIGESTMOD = hashlib.sha1 -# Test to see if we have the old or new style buffer() interface. Trying -# to use an old-style buffer on Python2.7 prior to version 2.7.5 will produce: -# -# TypeError: 'buffer' does not have the buffer interface -# -#: ``True`` if we have the new-style `buffer`_ interface; ``False`` otherwise. -#: -#: .. _buffer: https://docs.python.org/2/c-api/buffer.html -NEW_BUFFER_INTERFACE = False -try: - io.BytesIO(buffer('test')) -except TypeError: # pragma: no cover - logging.warn( - "This Python version is too old! "\ - "It doesn't support new-style buffer interfaces: "\ - "https://mail.python.org/pipermail/python-dev/2010-October/104917.html") -else: - NEW_BUFFER_INTERFACE = True - class PKCS1PaddingError(Exception): """Raised when there is a problem adding or removing PKCS#1 padding.""" @@ -89,7 +69,7 @@ class RSAKeyGenerationError(Exception): def writeKeyToFile(key, filename): - """Write **key** to **filename**, with ``0400`` permissions. + """Write **key** to **filename**, with ``400`` octal permissions. If **filename** doesn't exist, it will be created. If it does exist already, and is writable by the owner of the current process, then it will @@ -102,7 +82,7 @@ def writeKeyToFile(key, filename): """ logging.info("Writing key to file: %r" % filename) flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0) - fd = os.open(filename, flags, 0400) + fd = os.open(filename, flags, 0o400) os.write(fd, key) os.fsync(fd) os.close(fd) @@ -210,6 +190,12 @@ def getKey(filename): def getHMAC(key, value): """Return the HMAC of **value** using the **key**.""" + + # normalize inputs to be bytes + + key = key.encode('utf-8') if isinstance(key, str) else key + value = value.encode('utf-8') if isinstance(value, str) else value + h = hmac.new(key, value, digestmod=DIGESTMOD) return h.digest() @@ -220,14 +206,19 @@ def getHMACFunc(key, hex=True): :rtype: callable :returns: A function which can be uses to generate HMACs. """ + + key = key.encode('utf-8') if isinstance(key, str) else key h = hmac.new(key, digestmod=DIGESTMOD) + def hmac_fn(value): + value = value.encode('utf-8') if isinstance(value, str) else value h_tmp = h.copy() h_tmp.update(value) if hex: return h_tmp.hexdigest() else: return h_tmp.digest() + return hmac_fn def removePKCS1Padding(message): @@ -319,11 +310,12 @@ def initializeGnuPG(config): logging.warn("No secret keys found in %s!" % gpg.secring) return ret - primarySK = filter(lambda key: key['fingerprint'] == primary, secrets) - primaryPK = filter(lambda key: key['fingerprint'] == primary, publics) + primarySK = list(filter(lambda key: key['fingerprint'] == primary, secrets)) + primaryPK = list(filter(lambda key: key['fingerprint'] == primary, publics)) if primarySK and primaryPK: logging.info("Found GnuPG primary key with fingerprint: %s" % primary) + for sub in primaryPK[0]['subkeys']: logging.info(" Subkey: %s Usage: %s" % (sub[0], sub[1].upper())) else: @@ -419,7 +411,8 @@ class SSLVerifyingContextFactory(ssl.CertificateOptions): :rtype: str :returns: The full hostname (including any subdomains). """ - hostname = urllib.splithost(urllib.splittype(url)[1])[0] + + hostname = urllib.parse.urlparse(url).netloc logging.debug("Parsed hostname %r for cert CN matching." % hostname) return hostname diff --git a/bridgedb/distribute.py b/bridgedb/distribute.py index ec8fb26..42706c9 100644 --- a/bridgedb/distribute.py +++ b/bridgedb/distribute.py @@ -108,7 +108,7 @@ import math from zope import interface from zope.interface import Attribute -from zope.interface import implements +from zope.interface import implementer # from bridgedb.hashring import IHashring from bridgedb.interfaces import IName @@ -155,12 +155,12 @@ class IDistribute(IName): """Get bridges based on a client's **bridgeRequest**.""" +@implementer(IDistribute) class Distributor(Named): """A :class:`Distributor` distributes bridges to clients. Inherit from me to create a new type of ``Distributor``. """ - implements(IDistribute) _bridgesPerResponseMin = 1 _bridgesPerResponseMax = 3 diff --git a/bridgedb/distributors/email/__init__.py b/bridgedb/distributors/email/__init__.py index 5c0ef50..16f439d 100644 --- a/bridgedb/distributors/email/__init__.py +++ b/bridgedb/distributors/email/__init__.py @@ -16,9 +16,3 @@ distributor. ''' -import autoresponder -import distributor -import dkim -import request -import server -import templates diff --git a/bridgedb/distributors/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py index e69f78a..3711eae 100644 --- a/bridgedb/distributors/email/autoresponder.py +++ b/bridgedb/distributors/email/autoresponder.py @@ -39,6 +39,7 @@ Functionality for autoresponding to incoming emails. from __future__ import unicode_literals from __future__ import print_function +import email import io import logging import time @@ -51,7 +52,6 @@ from twisted.python import failure from bridgedb import strings from bridgedb import metrics from bridgedb import safelog -from bridgedb.crypto import NEW_BUFFER_INTERFACE from bridgedb.distributors.email import dkim from bridgedb.distributors.email import request from bridgedb.distributors.email import templates @@ -183,22 +183,12 @@ class EmailResponse(object): the email.) - :vartype _buff: :any:`unicode` or :any:`buffer` - :var _buff: Used internally to write lines for the response email into the - ``_mailfile``. The reason why both of these attributes have two - possible types is for the same Python-buggy reasons which require - :data:`~bridgedb.crypto.NEW_BUFFER_INTERFACE`. - :vartype mailfile: :class:`io.StringIO` or :class:`io.BytesIO` - :var mailfile: An in-memory file-like object for storing the formatted - headers and body of the response email. :var str delimiter: Delimiter between lines written to the :data:`mailfile`. :var bool closed: ``True`` if :meth:`close` has been called. :vartype to: :api:`twisted.mail.smtp.Address` :var to: The client's email address, to which this response should be sent. """ - _buff = buffer if NEW_BUFFER_INTERFACE else unicode - mailfile = io.BytesIO if NEW_BUFFER_INTERFACE else io.StringIO def __init__(self, gpgSignFunc=None): """Create a response to an email we have recieved. @@ -212,7 +202,7 @@ class EmailResponse(object): obtaining a pre-configured **gpgSignFunc**. """ self.gpgSign = gpgSignFunc - self.mailfile = self.mailfile() + self.mailfile = io.StringIO() self.delimiter = '\n' self.closed = False self.to = None @@ -274,6 +264,9 @@ class EmailResponse(object): :param str line: Something to append into the :data:`mailfile`. """ + + line = line.decode('utf-8') if isinstance(line, bytes) else line + if line.find('\r\n') != -1: # If **line** contains newlines, send it to :meth:`writelines` to # break it up so that we can replace them: @@ -281,7 +274,7 @@ class EmailResponse(object): self.writelines(line) else: line += self.delimiter - self.mailfile.write(self._buff(line.encode('utf8'))) + self.mailfile.write(line) self.mailfile.flush() def writelines(self, lines): @@ -291,10 +284,11 @@ class EmailResponse(object): (i.e. ``'\\n'``). See :api:`twisted.mail.smtp.SMTPClient.getMailData` for the reason. - :type lines: :any:`basestring` or :any:`list` + :type lines: :any:`str` or :any:`list` :param lines: The lines to write to the :attr:`mailfile`. """ - if isinstance(lines, basestring): + if isinstance(lines, (str, bytes)): + lines = lines.decode('utf-8') if isinstance(lines, bytes) else lines lines = lines.replace('\r\n', '\n') for ln in lines.split('\n'): self.write(ln) @@ -321,20 +315,24 @@ class EmailResponse(object): :kwargs: If given, the key will become the name of the header, and the value will become the contents of that header. """ + + fromAddress = fromAddress.decode('utf-8') if isinstance(fromAddress, bytes) else fromAddress + toAddress = toAddress.decode('utf-8') if isinstance(toAddress, bytes) else toAddress + self.write("From: %s" % fromAddress) self.write("To: %s" % toAddress) if includeMessageID: - self.write("Message-ID: %s" % smtp.messageid().encode('utf-8')) + self.write("Message-ID: %s" % smtp.messageid()) if inReplyTo: - self.write("In-Reply-To: %s" % inReplyTo.encode('utf-8')) - self.write("Content-Type: %s" % contentType.encode('utf-8')) - self.write("Date: %s" % smtp.rfc822date().encode('utf-8')) + self.write("In-Reply-To: %s" % inReplyTo) + self.write("Content-Type: %s" % contentType) + self.write("Date: %s" % smtp.rfc822date().decode('utf-8')) if not subject: subject = '[no subject]' if not subject.lower().startswith('re'): subject = "Re: " + subject - self.write("Subject: %s" % subject.encode('utf-8')) + self.write("Subject: %s" % subject) if kwargs: for headerName, headerValue in kwargs.items(): @@ -342,7 +340,7 @@ class EmailResponse(object): headerName = headerName.replace(' ', '-') headerName = headerName.replace('_', '-') header = "%s: %s" % (headerName, headerValue) - self.write(header.encode('utf-8')) + self.write(header) # The first blank line designates that the headers have ended: self.write(self.delimiter) @@ -442,8 +440,8 @@ class SMTPAutoresponder(smtp.SMTPClient): if not body: return # The client was already warned. - messageID = self.incoming.message.getheader("Message-ID", None) - subject = self.incoming.message.getheader("Subject", None) + messageID = self.incoming.message.get("Message-ID", None) + subject = self.incoming.message.get("Subject", None) response = generateResponse(recipient, client, body, subject, messageID, self.incoming.context.gpgSignFunc) @@ -464,13 +462,13 @@ class SMTPAutoresponder(smtp.SMTPClient): """ clients = [] addrHeader = None - try: fromAddr = self.incoming.message.getaddr("From")[1] + try: fromAddr = email.utils.parseaddr(self.incoming.message.get("From"))[1] except (IndexError, TypeError, AttributeError): pass else: addrHeader = fromAddr if not addrHeader: logging.warn("No From header on incoming mail.") - try: senderHeader = self.incoming.message.getaddr("Sender")[1] + try: senderHeader = email.utils.parseaddr(self.incoming.message.get("Sender"))[1] except (IndexError, TypeError, AttributeError): pass else: addrHeader = senderHeader if not addrHeader: @@ -512,10 +510,10 @@ class SMTPAutoresponder(smtp.SMTPClient): try: ourAddress = smtp.Address(self.incoming.context.fromAddr) - allRecipients = self.incoming.message.getaddrlist("To") + allRecipients = self.incoming.message.get_all("To") - for _, addr in allRecipients: - recipient = smtp.Address(addr) + for address in allRecipients: + recipient = smtp.Address(address) if not ourAddress.domain in recipient.domain: logging.debug(("Not our domain (%s) or subdomain, skipping" " email address: %s") @@ -528,11 +526,11 @@ class SMTPAutoresponder(smtp.SMTPClient): " email address: %s") % str(recipient)) continue # Only check the username before the first '+': - beforePlus = recipient.local.split('+', 1)[0] + beforePlus = recipient.local.split(b'+', 1)[0] if beforePlus == ourAddress.local: ours = str(recipient) if not ours: - raise addr.BadEmail(allRecipients) + raise addr.BadEmail('No email address accepted, please see log', allRecipients) except Exception as error: logging.error(("Couldn't find our email address in incoming email " diff --git a/bridgedb/distributors/email/dkim.py b/bridgedb/distributors/email/dkim.py index be33d59..5cee31d 100644 --- a/bridgedb/distributors/email/dkim.py +++ b/bridgedb/distributors/email/dkim.py @@ -63,10 +63,8 @@ def checkDKIM(message, rules): if 'dkim' in rules: # getheader() returns the last of a given kind of header; we want # to get the first, so we use getheaders() instead. - dkimHeaders = message.getheaders("X-DKIM-Authentication-Results") - dkimHeader = "<no header>" - if dkimHeaders: - dkimHeader = dkimHeaders[0] + dkimHeaders = message.get("X-DKIM-Authentication-Results") + dkimHeader = dkimHeaders if dkimHeaders else "<no header>" if not dkimHeader.startswith("pass"): logging.info("Rejecting bad DKIM header on incoming email: %r " % dkimHeader) diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py index 8bd4b36..063b640 100644 --- a/bridgedb/distributors/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -49,10 +49,10 @@ Servers which interface with clients and distribute bridges over SMTP. from __future__ import unicode_literals +import email.message import logging import io import socket -import rfc822 from twisted.internet import defer from twisted.internet import reactor @@ -62,7 +62,7 @@ from twisted.mail import smtp from twisted.mail.smtp import rfc822date from twisted.python import failure -from zope.interface import implements +from zope.interface import implementer from bridgedb import __version__ from bridgedb import safelog @@ -126,7 +126,7 @@ class MailServerContext(object): self.nBridges = config.EMAIL_N_BRIDGES_PER_ANSWER self.username = (config.EMAIL_USERNAME or "bridges") - self.hostname = socket.gethostname() + self.hostname = socket.gethostname().encode("utf-8") self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges@torproject.org") self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr) self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25) @@ -165,6 +165,7 @@ class MailServerContext(object): return canon +@implementer(smtp.IMessage) class SMTPMessage(object): """Plugs into the Twisted Mail and receives an incoming message. @@ -186,7 +187,6 @@ class SMTPMessage(object): :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.reply` email and :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.send` it. """ - implements(smtp.IMessage) def __init__(self, context, canonicalFromSMTP=None): """Create a new SMTPMessage. @@ -222,7 +222,7 @@ class SMTPMessage(object): if self.nBytes > self.context.maximumSize: self.ignoring = True else: - self.lines.append(line) + self.lines.append(line.decode('utf-8') if isinstance(line, bytes) else line) if not safelog.safe_logging: try: ln = line.rstrip("\r\n").encode('utf-8', 'replace') @@ -251,13 +251,11 @@ class SMTPMessage(object): :rtype: :api:`twisted.mail.smtp.rfc822.Message` :returns: A ``Message`` comprised of all lines received thus far. """ - rawMessage = io.StringIO() - for line in self.lines: - rawMessage.writelines(unicode(line.decode('utf8')) + u'\n') - rawMessage.seek(0) - return rfc822.Message(rawMessage) + return email.message_from_string('\n'.join(self.lines)) + +@implementer(smtp.IMessageDelivery) class SMTPIncomingDelivery(smtp.SMTP): """Plugs into :class:`SMTPIncomingServerFactory` and handles SMTP commands for incoming connections. @@ -272,7 +270,6 @@ class SMTPIncomingDelivery(smtp.SMTP): :var fromCanonicalSMTP: If set, this is the canonicalized domain name of the address we received from incoming connection's ``MAIL FROM:``. """ - implements(smtp.IMessageDelivery) context = None deferred = defer.Deferred() @@ -293,11 +290,11 @@ class SMTPIncomingDelivery(smtp.SMTP): :type recipients: list :param recipients: A list of :api:`twisted.mail.smtp.User` instances. """ - helo_ = ' helo={0}'.format(helo[0]) if helo[0] else '' - from_ = 'from %s ([%s]%s)' % (helo[0], helo[1], helo_) - by_ = 'by %s with BridgeDB (%s)' % (smtp.DNSNAME, __version__) - for_ = 'for %s; %s ' % (' '.join(map(str, recipients)), rfc822date()) - return str('Received: %s\n\t%s\n\t%s' % (from_, by_, for_)) + helo_ = b' helo=%s' % (helo[0] if helo[0] else '') + from_ = b'from %s ([%s]%s)' % (helo[0], helo[1], helo_) + by_ = b'by %s with BridgeDB (%s)' % (smtp.DNSNAME, __version__.encode('utf-8')) + for_ = b'for %s; %s ' % (b' '.join([str(r).encode('utf-8') for r in recipients]), rfc822date()) + return 'Received: %s\n\t%s\n\t%s' % (from_.decode('utf-8'), by_.decode('utf-8'), for_.decode('utf-8')) def validateFrom(self, helo, origin): """Validate the ``MAIL FROM:`` address on the incoming SMTP connection. @@ -376,10 +373,10 @@ class SMTPIncomingDelivery(smtp.SMTP): ourAddress = smtp.Address(self.context.smtpFromAddr) if not ((ourAddress.domain in recipient.domain) or - (recipient.domain == "bridgedb")): + (recipient.domain == b"bridgedb")): logging.debug(("Not our domain (%s) or subdomain, skipping" " SMTP 'RCPT TO' address: %s") - % (ourAddress.domain, str(recipient))) + % (ourAddress.domain.decode('utf-8'), str(recipient))) raise smtp.SMTPBadRcpt(str(recipient)) # The recipient's username should at least start with ours, # but it still might be a '+' address. @@ -388,13 +385,14 @@ class SMTPIncomingDelivery(smtp.SMTP): " SMTP 'RCPT TO' address: %s") % str(recipient)) raise smtp.SMTPBadRcpt(str(recipient)) # Ignore everything after the first '+', if there is one. - beforePlus = recipient.local.split('+', 1)[0] + beforePlus = recipient.local.split(b'+', 1)[0] if beforePlus != ourAddress.local: raise smtp.SMTPBadRcpt(str(recipient)) return lambda: SMTPMessage(self.context, self.fromCanonicalSMTP) +@implementer(smtp.IMessageDeliveryFactory) class SMTPIncomingDeliveryFactory(object): """Factory for :class:`SMTPIncomingDelivery` s. @@ -408,7 +406,6 @@ class SMTPIncomingDeliveryFactory(object): :var delivery: A :class:`SMTPIncomingDelivery` to deliver incoming SMTP messages to. """ - implements(smtp.IMessageDeliveryFactory) context = None delivery = SMTPIncomingDelivery @@ -499,7 +496,7 @@ def addServer(config, distributor): reactor.listenTCP(port, factory, interface=addr) except CannotListenError as error: # pragma: no cover logging.fatal(error) - raise SystemExit(error.message) + raise SystemExit(str(error)) # Set up a LoopingCall to run every 30 minutes and forget old email times. lc = LoopingCall(distributor.cleanDatabase) diff --git a/bridgedb/distributors/email/templates.py b/bridgedb/distributors/email/templates.py index 85dd105..188f052 100644 --- a/bridgedb/distributors/email/templates.py +++ b/bridgedb/distributors/email/templates.py @@ -59,6 +59,7 @@ def addCommands(template): def addGreeting(template, clientName=None, welcome=False): greeting = "" + clientName = clientName.decode('utf-8') if isinstance(clientName, bytes) else clientName if not clientName: greeting = template.gettext(strings.EMAIL_MISC_TEXT[7]) diff --git a/bridgedb/distributors/https/distributor.py b/bridgedb/distributors/https/distributor.py index 7bce8d9..f3e2e9b 100644 --- a/bridgedb/distributors/https/distributor.py +++ b/bridgedb/distributors/https/distributor.py @@ -20,6 +20,7 @@ A Distributor that hands out bridges through a web interface. :parts: 1 """ +import binascii import ipaddr import logging @@ -324,9 +325,9 @@ class HTTPSDistributor(Distributor): logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) - logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Assigned client to subhashring position: %s" % binascii.hexlify(position).decode('utf-8')) logging.debug("Total bridges: %d" % len(self.hashring)) - logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) + logging.debug("Bridge filters: %s" % ' '.join([x.__name__ for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.hashring.filterRings.keys(): diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index 660d34e..503744e 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -95,6 +95,25 @@ supported_langs = [] metrix = metrics.HTTPSMetrics() +def stringifyRequestArgs(args): + """Turn the given HTTP request arguments from bytes to str. + + :param dict args: A dictionary of request arguments. + :rtype: dict + :returns: A dictionary of request arguments. + """ + + # Convert all key/value pairs from bytes to str. + str_args = {} + for arg, values in args.items(): + arg = arg if isinstance(arg, str) else arg.decode("utf-8") + values = [value.decode("utf-8") if isinstance(value, bytes) + else value for value in values] + str_args[arg] = values + + return str_args + + def replaceErrorPage(request, error, template_name=None, html=True): """Create a general error page for displaying in place of tracebacks. @@ -114,9 +133,9 @@ def replaceErrorPage(request, error, template_name=None, html=True): or if **html** is ``False``, then we return a very simple HTML page (without CSS, Javascript, images, etc.) which simply says ``"Sorry! Something went wrong with your request."`` - :rtype: str - :returns: A string containing some content to serve to the client (rather - than serving a Twisted traceback). + :rtype: bytes + :returns: A bytes object containing some content to serve to the client + (rather than serving a Twisted traceback). """ logging.error("Error while attempting to render %s: %s" % (template_name or 'template', @@ -135,13 +154,13 @@ def replaceErrorPage(request, error, template_name=None, html=True): errorMessage = _("Sorry! Something went wrong with your request.") if not html: - return bytes(errorMessage) + return errorMessage.encode("utf-8") try: rendered = resource500.render(request) except Exception as err: logging.exception(err) - rendered = bytes(errorMessage) + rendered = errorMessage.encode("utf-8") return rendered @@ -336,6 +355,7 @@ class ErrorResource(CSPResource): self.setCSPHeader(request) request.setHeader("Content-Type", "text/html; charset=utf-8") request.setResponseCode(self.code) + request.args = stringifyRequestArgs(request.args) try: template = lookup.get_template(self.template) @@ -373,12 +393,13 @@ class TranslatedTemplateResource(CustomErrorHandlingResource, CSPResource): """Create a new :api:`Resource <twisted.web.resource.Resource>` for a Mako-templated webpage. """ - gettext.install("bridgedb", unicode=True) + gettext.install("bridgedb") CSPResource.__init__(self) self.template = template def render_GET(self, request): self.setCSPHeader(request) + request.args = stringifyRequestArgs(request.args) rtl = False try: langs = translations.getLocaleFromHTTPRequest(request) @@ -452,11 +473,11 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): :rtype: tuple :returns: A 2-tuple of ``(image, challenge)``, where ``image`` is a - binary, JPEG-encoded image, and ``challenge`` is a unique + JPEG-encoded image of type bytes, and ``challenge`` is a unique string. If unable to retrieve a CAPTCHA, returns a tuple - containing two empty strings. + containing (b'', ''). """ - return ('', '') + return (b'', '') def extractClientSolution(self, request): """Extract the client's CAPTCHA solution from a POST request. @@ -468,7 +489,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object for 'bridges.html'. :returns: A redirect for a request for a new CAPTCHA if there was a - problem. Otherwise, returns a 2-tuple of strings, the first is the + problem. Otherwise, returns a 2-tuple of bytes, the first is the client's CAPTCHA solution from the text input area, and the second is the challenge string. """ @@ -491,16 +512,17 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): return False def render_GET(self, request): - """Retrieve a ReCaptcha from the API server and serve it to the client. + """Retrieve a CAPTCHA and serve it to the client. :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object for a page which should be protected by a CAPTCHA. - :rtype: str - :returns: A rendered HTML page containing a ReCaptcha challenge image + :rtype: bytes + :returns: A rendered HTML page containing a CAPTCHA challenge image for the client to solve. """ self.setCSPHeader(request) + request.args = stringifyRequestArgs(request.args) rtl = False image, challenge = self.getCaptchaImage(request) @@ -509,14 +531,14 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) # TODO: this does not work for versions of IE < 8.0 - imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(image) + imgstr = b'data:image/jpeg;base64,%s' % base64.b64encode(image) template = lookup.get_template('captcha.html') rendered = template.render(strings, getSortedLangList(), rtl=rtl, lang=langs[0], langOverride=translations.isLangOverridden(request), - imgstr=imgstr, + imgstr=imgstr.decode("utf-8"), challenge_field=challenge) except Exception as err: rendered = replaceErrorPage(request, err, 'captcha.html') @@ -543,15 +565,16 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): """ self.setCSPHeader(request) request.setHeader("Content-Type", "text/html; charset=utf-8") + request.args = stringifyRequestArgs(request.args) try: if self.checkSolution(request) is True: metrix.recordValidHTTPSRequest(request) return self.resource.render(request) except ValueError as err: - logging.debug(err.message) + logging.debug(str(err)) except MaliciousRequest as err: - logging.debug(err.message) + logging.debug(str(err)) # Make them wait a bit, then redirect them to a "daring # work of art" as pennance for their sins. d = task.deferLater(reactor, 1, lambda: request) @@ -559,7 +582,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): metrix.recordInvalidHTTPSRequest(request) return NOT_DONE_YET except Exception as err: - logging.debug(err.message) + logging.debug(str(err)) metrix.recordInvalidHTTPSRequest(request) return replaceErrorPage(request, err) @@ -730,7 +753,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): rendered = redirectTo(request.uri, request) try: - request.write(rendered) + request.write(rendered.encode('utf-8') if isinstance(rendered, str) else rendered) request.finish() except Exception as err: # pragma: no cover logging.exception(err) @@ -862,6 +885,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): HTML to the client. """ self.setCSPHeader(request) + request.args = stringifyRequestArgs(request.args) d = self.checkSolution(request) d.addCallback(self._renderDeferred) return NOT_DONE_YET @@ -888,7 +912,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): :param bool includeFingerprints: Do we include the bridge's fingerprint in the response? """ - gettext.install("bridgedb", unicode=True) + gettext.install("bridgedb") CSPResource.__init__(self) self.distributor = distributor self.schedule = schedule @@ -908,10 +932,11 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object containing the HTTP method, full URI, and any URL/POST arguments and headers present. - :rtype: str + :rtype: bytes :returns: A plaintext or HTML response to serve. """ self.setCSPHeader(request) + request.args = stringifyRequestArgs(request.args) try: response = self.getBridgeRequestAnswer(request) @@ -919,7 +944,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): logging.exception(err) response = self.renderAnswer(request) - return response + return response.encode('utf-8') if isinstance(response, str) else response def getClientIP(self, request): """Get the client's IP address from the ``'X-Forwarded-For:'`` @@ -949,6 +974,14 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): logging.info("Replying to web request from %s. Parameters were %r" % (ip, request.args)) + # Convert all key/value pairs from bytes to str. + str_args = {} + for arg, values in request.args.items(): + arg = arg if isinstance(arg, str) else arg.decode("utf-8") + values = [value.decode("utf-8") if isinstance(value, bytes) else value for value in values] + str_args[arg] = values + request.args = str_args + if ip: bridgeRequest = HTTPSBridgeRequest() bridgeRequest.client = ip @@ -1015,7 +1048,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): to use a bridge. If ``None``, then the returned page will instead explain that there were no bridges of the type they requested, with instructions on how to proceed. - :rtype: str + :rtype: bytes :returns: A plaintext or HTML response to serve. """ rtl = False @@ -1024,7 +1057,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): if format == 'plain': request.setHeader("Content-Type", "text/plain") try: - rendered = bytes('\n'.join(bridgeLines)) + rendered = '\n'.join(bridgeLines).encode('utf-8') except Exception as err: rendered = replaceErrorPage(request, err, html=False) else: @@ -1033,7 +1066,9 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): qrjpeg = generateQR(bridgeLines) if qrjpeg: - qrcode = 'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg) + qrcode = b'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg) + qrcode = qrcode.decode("utf-8") + try: langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) @@ -1048,7 +1083,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): except Exception as err: rendered = replaceErrorPage(request, err) - return rendered + return rendered.encode("utf-8") if isinstance(rendered, str) else rendered def addWebServer(config, distributor): @@ -1100,21 +1135,21 @@ def addWebServer(config, distributor): howto = HowtoResource() robots = static.File(os.path.join(TEMPLATE_DIR, 'robots.txt')) assets = static.File(os.path.join(TEMPLATE_DIR, 'assets/')) - keys = static.Data(bytes(strings.BRIDGEDB_OPENPGP_KEY), 'text/plain') + keys = static.Data(strings.BRIDGEDB_OPENPGP_KEY.encode('utf-8'), 'text/plain') csp = CSPResource(enabled=config.CSP_ENABLED, includeSelf=config.CSP_INCLUDE_SELF, reportViolations=config.CSP_REPORT_ONLY, useForwardedHeader=fwdHeaders) root = CustomErrorHandlingResource() - root.putChild('', index) - root.putChild('robots.txt', robots) - root.putChild('keys', keys) - root.putChild('assets', assets) - root.putChild('options', options) - root.putChild('howto', howto) - root.putChild('maintenance', maintenance) - root.putChild('error', resource500) + root.putChild(b'', index) + root.putChild(b'robots.txt', robots) + root.putChild(b'keys', keys) + root.putChild(b'assets', assets) + root.putChild(b'options', options) + root.putChild(b'howto', howto) + root.putChild(b'maintenance', maintenance) + root.putChild(b'error', resource500) root.putChild(CSPResource.reportURI, csp) if config.RECAPTCHA_ENABLED: @@ -1135,7 +1170,7 @@ def addWebServer(config, distributor): if config.HTTPS_ROTATION_PERIOD: count, period = config.HTTPS_ROTATION_PERIOD.split() - sched = ScheduledInterval(count, period) + sched = ScheduledInterval(int(count), period) else: sched = Unscheduled() @@ -1147,10 +1182,10 @@ def addWebServer(config, distributor): secretKey=secretKey, useForwardedHeader=fwdHeaders, protectedResource=bridges) - root.putChild('bridges', protected) + root.putChild(b'bridges', protected) logging.info("Protecting resources with %s." % captcha.func.__name__) else: - root.putChild('bridges', bridges) + root.putChild(b'bridges', bridges) site = Site(root) site.displayTracebacks = False diff --git a/bridgedb/distributors/moat/server.py b/bridgedb/distributors/moat/server.py index 97dfc42..cdf2022 100644 --- a/bridgedb/distributors/moat/server.py +++ b/bridgedb/distributors/moat/server.py @@ -178,11 +178,11 @@ class JsonAPIResource(resource.Resource): :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. :returns: The encoded data. """ - request.responseHeaders.addRawHeader(b"Content-Type", b"application/vnd.api+json") - request.responseHeaders.addRawHeader(b"Server", b"moat/%s" % MOAT_API_VERSION) + request.responseHeaders.addRawHeader("Content-Type", "application/vnd.api+json") + request.responseHeaders.addRawHeader("Server", "moat/%s" % MOAT_API_VERSION) if data: - rendered = json.dumps(data) + rendered = json.dumps(data).encode("utf-8") else: rendered = b"" @@ -240,7 +240,7 @@ class CustomErrorHandlingResource(resource.Resource): response = resource501 response.detail = "moat version %s does not implement %s %s" % \ - (MOAT_API_VERSION, request.method, request.uri) + (MOAT_API_VERSION, request.method.decode('utf-8'), request.uri.decode('utf-8')) return response @@ -368,7 +368,7 @@ class CaptchaFetchResource(CaptchaResource): logging.error("Unhandled error while retrieving Gimp captcha!") logging.error(impossible) - return (capt.image, capt.challenge) + return (capt.image, capt.challenge.decode('utf-8') if isinstance(capt.challenge, bytes) else capt.challenge) def getPreferredTransports(self, supportedTransports): """Choose which transport a client should request, based on their list @@ -476,7 +476,7 @@ class CaptchaFetchResource(CaptchaResource): } try: - data["data"][0]["image"] = base64.b64encode(image) + data["data"][0]["image"] = base64.b64encode(image).decode('utf-8') except Exception as impossible: logging.error("Could not construct or encode captcha!") logging.error(impossible) @@ -602,7 +602,7 @@ class CaptchaCheckResource(CaptchaResource): logging.warn(("Error processing client POST request: " "Client JSON API data missing '%s' field.") % err) except ValueError as err: - logging.warn("Error processing client POST request: %s" % err.message) + logging.warn("Error processing client POST request: %s" % err) except Exception as impossible: logging.error(impossible) @@ -824,13 +824,13 @@ def addMoatServer(config, distributor): hmacKey, publicKey, secretKey, fwdHeaders, skipLoopback) - moat.putChild("fetch", fetch) - moat.putChild("check", check) - meek.putChild("moat", moat) + moat.putChild(b"fetch", fetch) + moat.putChild(b"check", check) + meek.putChild(b"moat", moat) root = CustomErrorHandlingResource() - root.putChild("meek", meek) - root.putChild("moat", moat) + root.putChild(b"meek", meek) + root.putChild(b"moat", moat) site = Site(root) site.displayTracebacks = False diff --git a/bridgedb/filters.py b/bridgedb/filters.py index a02661e..3ea0723 100644 --- a/bridgedb/filters.py +++ b/bridgedb/filters.py @@ -13,6 +13,7 @@ """Functions for filtering :class:`Bridges <bridgedb.bridges.Bridge>`.""" +import binascii import logging from ipaddr import IPv4Address @@ -43,7 +44,7 @@ def bySubring(hmac, assigned, total): logging.debug(("Creating a filter for assigning bridges to subhashring " "%s-of-%s...") % (assigned, total)) - name = "-".join([str(hmac("")[:8]).encode('hex'), + name = "-".join([binascii.hexlify(str(hmac("")[:8]).encode('utf-8')).decode('utf-8'), str(assigned), "of", str(total)]) try: return _cache[name] @@ -119,7 +120,7 @@ def byIPv(ipVersion=None): return False setattr(_byIPv, "description", "ip=%d" % ipVersion) _byIPv.__name__ = "byIPv%d()" % ipVersion - _byIPv.func_doc = _byIPv.func_doc.format(ipVersion) + _byIPv.__doc__ = _byIPv.__doc__.format(ipVersion) _byIPv.name = name _cache[name] = _byIPv return _byIPv diff --git a/bridgedb/main.py b/bridgedb/main.py index 7c2df6d..bc309c5 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -198,7 +198,7 @@ def load(state, hashring, clear=False): inserted = 0 logging.info("Trying to insert %d bridges into hashring, %d of which " "have the 'Running' flag..." % (len(bridges), - len(filter(lambda b: b.flags.running, bridges.values())))) + len(list(filter(lambda b: b.flags.running, bridges.values()))))) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the @@ -423,17 +423,17 @@ def run(options, reactor=reactor): logging.info("Reloading decoy bridges...") antibot.loadDecoyBridges(config.DECOY_BRIDGES_FILE) - logging.info("Reparsing bridge descriptors...") (hashring, emailDistributorTmp, ipDistributorTmp, moatDistributorTmp) = createBridgeRings(cfg, proxies, key) - logging.info("Bridges loaded: %d" % len(hashring)) # Initialize our DB. bridgedb.Storage.initializeDBLock() bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") + logging.info("Reparsing bridge descriptors...") load(state, hashring, clear=False) + logging.info("Bridges loaded: %d" % len(hashring)) if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings diff --git a/bridgedb/metrics.py b/bridgedb/metrics.py index 6b2e5cb..bb888d9 100644 --- a/bridgedb/metrics.py +++ b/bridgedb/metrics.py @@ -184,16 +184,13 @@ class Singleton(type): pass -class Metrics(object): +class Metrics(metaclass=Singleton): """Base class representing metrics. This class provides functionality that our three distribution mechanisms share. """ - # We're using a meta class to implement a singleton for Metrics. - __metaclass__ = Singleton - def __init__(self, binSize=BIN_SIZE): logging.debug("Instantiating metrics class.") self.binSize = binSize @@ -235,7 +232,7 @@ class Metrics(object): :returns: A list of metric lines. """ lines = [] - for key, value in self.coldMetrics.iteritems(): + for key, value in self.coldMetrics.items(): # Round up our value to the nearest multiple of self.binSize to # reduce the accuracy of our real values. if (value % self.binSize) > 0: @@ -282,6 +279,9 @@ class Metrics(object): combinations. """ + if isinstance(countryOrProvider, bytes): + countryOrProvider = countryOrProvider.decode('utf-8') + countryOrProvider = countryOrProvider.lower() bridgeType = bridgeType.lower() success = "success" if success else "fail" @@ -403,7 +403,7 @@ class EmailMetrics(Metrics): logging.debug("Recording %svalid email request for %s from %s." % ("" if success else "in", bridgeType, emailAddr)) - sld = emailAddr.domain.split(".")[0] + sld = emailAddr.domain.split(b".")[0] # Now update our metrics. key = self.createKey(self.keyPrefix, bridgeType, sld, success, diff --git a/bridgedb/parse/addr.py b/bridgedb/parse/addr.py index 90bc1d5..1d59c9a 100644 --- a/bridgedb/parse/addr.py +++ b/bridgedb/parse/addr.py @@ -231,6 +231,7 @@ def canonicalizeEmailDomain(domain, domainmap): :returns: The canonical domain name for the email address. """ permitted = None + domain = domain.decode('utf-8') if isinstance(domain, bytes) else domain try: permitted = domainmap.get(domain) @@ -297,7 +298,7 @@ def extractEmailAddress(emailaddr): def isIPAddress(ip, compressed=True): """Check if an arbitrary string is an IP address, and that it's valid. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to check. :param boolean compressed: If True, return a string representing the compressed form of the address. Otherwise, return an @@ -347,7 +348,7 @@ def isIPv4(ip): .. attention:: This does *not* check validity. See :func:`isValidIP`. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to check. :rtype: boolean :returns: True if the address is an IPv4 address. @@ -359,7 +360,7 @@ def isIPv6(ip): .. attention:: This does *not* check validity. See :func:`isValidIP`. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to check. :rtype: boolean :returns: True if the address is an IPv6 address. @@ -407,7 +408,7 @@ def isValidIP(ip): reasons = [] try: - if isinstance(ip, basestring): + if isinstance(ip, str): ip = ipaddr.IPAddress(ip) if ip.is_link_local: @@ -445,7 +446,7 @@ def isLoopback(ip): otherwise. """ try: - if isinstance(ip, basestring): + if isinstance(ip, str): ip = ipaddr.IPAddress(ip) if ip.is_loopback: @@ -572,7 +573,7 @@ class PortList(object): for arg in args: portlist = [] try: - if isinstance(arg, basestring): + if isinstance(arg, str): ports = set([int(p) for p in arg.split(',')][:self.PORTSPEC_LEN]) portlist.extend([self._sanitycheck(p) for p in ports]) diff --git a/bridgedb/parse/descriptors.py b/bridgedb/parse/descriptors.py index 1b01023..f6dee83 100644 --- a/bridgedb/parse/descriptors.py +++ b/bridgedb/parse/descriptors.py @@ -122,10 +122,10 @@ def parseNetworkStatusFile(filename, validate=True, skipAnnotations=True, routers = [] logging.info("Parsing networkstatus file: %s" % filename) - with open(filename) as fh: + with open(filename, 'rb') as fh: position = fh.tell() if skipAnnotations: - while not fh.readline().startswith('r '): + while not fh.readline().startswith(b'r '): position = fh.tell() logging.debug("Skipping %d bytes of networkstatus file." % position) fh.seek(position) @@ -161,42 +161,8 @@ def parseServerDescriptorsFile(filename, validate=True): logging.info("Parsing server descriptors with Stem: %s" % filename) descriptorType = 'server-descriptor 1.0' document = parse_file(filename, descriptorType, validate=validate) - routers = list() + return list(document) - # Work around https://bugs.torproject.org/26023 by parsing each descriptor - # at a time and catching any errors not handled in stem: - while True: - try: - routers.append(document.next()) - except StopIteration: - break - except Exception as error: - logging.debug("Error while parsing a bridge server descriptor: %s" - % error) - - return routers - -def __cmp_published__(x, y): - """A custom ``cmp()`` which sorts descriptors by published date. - - :rtype: int - :returns: Return negative if x<y, zero if x==y, positive if x>y. - """ - if x.published < y.published: - return -1 - elif x.published == y.published: - # This *shouldn't* happen. It would mean that two descriptors for - # the same router had the same timestamps, probably meaning there - # is a severely-messed up OR implementation out there. Let's log - # its fingerprint (no matter what!) so that we can look up its - # ``platform`` line in its server-descriptor and tell whoever - # wrote that code that they're probably (D)DOSing the Tor network. - logging.warn(("Duplicate descriptor with identical timestamp (%s) " - "for bridge %s with fingerprint %s !") % - (x.published, x.nickname, x.fingerprint)) - return 0 - elif x.published > y.published: - return 1 def deduplicate(descriptors, statistics=False): """Deduplicate some descriptors, returning only the newest for each router. @@ -227,7 +193,7 @@ def deduplicate(descriptors, statistics=False): duplicates[fingerprint] = [descriptor,] for fingerprint, dupes in duplicates.items(): - dupes.sort(cmp=__cmp_published__) + dupes.sort(key=lambda x: x.published) first = dupes.pop() newest[fingerprint] = first duplicates[fingerprint] = dupes diff --git a/bridgedb/parse/headers.py b/bridgedb/parse/headers.py index 5c3597a..29dd091 100644 --- a/bridgedb/parse/headers.py +++ b/bridgedb/parse/headers.py @@ -64,8 +64,8 @@ def parseAcceptLanguage(header): for only in langsOnly: if only not in langs: # Add the fallback after the other languages like it: - insertAfter = filter(lambda x: x.startswith(only), - [x for x in langs]) + insertAfter = list(filter(lambda x: x.startswith(only), + [x for x in langs])) if insertAfter: placement = langs.index(insertAfter[0]) + 1 langs.insert(placement, only) @@ -75,5 +75,4 @@ def parseAcceptLanguage(header): # Gettext wants underderscores, because that is how it creates the # directories under i18n/, not hyphens: - langs = map(lambda x: x.replace('-', '_'), [x for x in langs]) - return langs + return list(map(lambda x: x.replace('-', '_'), [x for x in langs])) diff --git a/bridgedb/parse/nickname.py b/bridgedb/parse/nickname.py index dcc328b..54bcf86 100644 --- a/bridgedb/parse/nickname.py +++ b/bridgedb/parse/nickname.py @@ -38,7 +38,7 @@ def isValidRouterNickname(nickname): :rtype: bool :returns: ``True`` if the nickname is valid, ``False`` otherwise. """ - ALPHANUMERIC = string.letters + string.digits + ALPHANUMERIC = string.ascii_letters + string.digits try: if not (1 <= len(nickname) <= 19): diff --git a/bridgedb/parse/options.py b/bridgedb/parse/options.py index e5c4d1e..c32ada3 100644 --- a/bridgedb/parse/options.py +++ b/bridgedb/parse/options.py @@ -117,12 +117,12 @@ def parseOptions(): try: options.parseOptions() except usage.UsageError as uerr: - print(uerr.message) + print(str(uerr)) print(options.getUsage()) sys.exit(1) except Exception as error: # pragma: no cover exc, value, tb = sys.exc_info() - print("Unhandled Error: %s" % error.message) + print("Unhandled Error: %s" % error) print(traceback.format_exc(tb)) return options @@ -206,7 +206,7 @@ class BaseOptions(usage.Options): if rundir is not None: gRundir = os.path.abspath(os.path.expanduser(rundir)) else: - gRundir = os.getcwdu() + gRundir = os.getcwd() setRundir(gRundir) if not os.path.isdir(gRundir): # pragma: no cover diff --git a/bridgedb/persistent.py b/bridgedb/persistent.py index c46c4a5..3d52ec5 100644 --- a/bridgedb/persistent.py +++ b/bridgedb/persistent.py @@ -128,6 +128,11 @@ class State(jelly.Jellyable): :param string statefile: The filename of the statefile. """ + + if filename is None: + self._statefile = None + return + filename = os.path.abspath(os.path.expanduser(filename)) logging.debug("Setting statefile to '%s'" % filename) self._statefile = filename @@ -173,8 +178,8 @@ class State(jelly.Jellyable): err = '' try: - if isinstance(statefile, basestring): - fh = open(statefile, 'r') + if isinstance(statefile, (str, bytes)): + fh = open(statefile, 'rb') elif not statefile.closed: fh = statefile except (IOError, OSError) as error: # pragma: no cover @@ -183,7 +188,7 @@ class State(jelly.Jellyable): except (AttributeError, TypeError) as error: err += "Failed statefile.open() and statefile.closed:" err += "\n\t{0}\nstatefile type = '{1}'".format( - error.message, type(statefile)) + error, type(statefile)) else: try: status = pickle.load(fh) @@ -209,7 +214,7 @@ class State(jelly.Jellyable): fh = None try: - fh = open(statefile, 'w') + fh = open(statefile, 'wb') except (IOError, OSError) as error: # pragma: no cover logging.warn("Error writing state file to '%s': %s" % (statefile, error)) @@ -218,7 +223,7 @@ class State(jelly.Jellyable): pickle.dump(jelly.jelly(self), fh) except AttributeError as error: logging.debug("Tried jellying an unjelliable object: %s" - % error.message) + % error) if fh is not None: fh.flush() diff --git a/bridgedb/proxy.py b/bridgedb/proxy.py index 6d93c48..654b40c 100644 --- a/bridgedb/proxy.py +++ b/bridgedb/proxy.py @@ -161,7 +161,7 @@ class ProxySet(MutableSet): only added if it passes the checks in :func:`~bridgedb.parse.addr.isIPAddress`. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to add. :param tag: An optional value to link to **ip**. If not given, it will be a timestamp (seconds since epoch, as a float) for when **ip** @@ -183,7 +183,7 @@ class ProxySet(MutableSet): def __contains__(self, ip): """x.__contains__(y) <==> y in x. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to check. :rtype: boolean :returns: True if ``ip`` is in this set; False otherwise. @@ -196,7 +196,7 @@ class ProxySet(MutableSet): def __sub__(self, ip): """Entirely remove **ip** from this set. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to remove. """ try: @@ -244,7 +244,7 @@ class ProxySet(MutableSet): if len(x) == 2: self.add(x[0], x[1]) elif len(x) == 1: self.add(x, tag) else: raise ValueError(self._getErrorMessage(x, proxies)) - elif isinstance(x, (basestring, int)): + elif isinstance(x, (str, int)): self.add(x, tag) else: raise ValueError(self._getErrorMessage(x, proxies)) @@ -270,9 +270,9 @@ class ProxySet(MutableSet): def getTag(self, ip): """Get the tag for an **ip** in this ``ProxySet``, if available. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to obtain the tag for. - :rtype: ``None`` or basestring or int + :rtype: ``None`` or str or int :returns: The tag for that **ip**, iff **ip** exists in this ``ProxySet`` and it has a tag. """ @@ -281,7 +281,7 @@ class ProxySet(MutableSet): def getAllWithTag(self, tag): """Get all proxies in this ``ProxySet`` with a given tag. - :param basestring tag: A tag to search for. + :param str tag: A tag to search for. :rtype: set :returns: A set of all proxies which are contained within this :class:`~bridgedb.proxy.ProxySet` which are also tagged with @@ -293,7 +293,7 @@ class ProxySet(MutableSet): def firstSeen(self, ip): """Get the timestamp when **ip** was first seen, if available. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to obtain a timestamp for. :rtype: float or None :returns: The timestamp (in seconds since epoch) if available. @@ -306,7 +306,7 @@ class ProxySet(MutableSet): def isExitRelay(self, ip): """Check if ``ip`` is a known Tor exit relay. - :type ip: basestring or int + :type ip: str or int :param ip: The IP address to check. :rtype: boolean :returns: True if ``ip`` is a known Tor exit relay; False otherwise. @@ -425,7 +425,7 @@ class ExitListProtocol(protocol.ProcessProtocol): def outReceived(self, data): """Some data was received from stdout.""" - self.data.append(data) + self.data.append(data.decode("utf-8")) def outConnectionLost(self): """This will be called when stdout is closed.""" diff --git a/bridgedb/qrcodes.py b/bridgedb/qrcodes.py index b3c4546..3cb3a89 100644 --- a/bridgedb/qrcodes.py +++ b/bridgedb/qrcodes.py @@ -13,7 +13,7 @@ """Utilities for working with QRCodes.""" -import cStringIO +import io import logging try: @@ -26,15 +26,15 @@ except ImportError: # pragma: no cover "python-qrcode package.")) -def generateQR(bridgelines, imageFormat=u'JPEG', bridgeSchema=False): +def generateQR(bridgelines, imageFormat='JPEG', bridgeSchema=False): """Generate a QRCode for the client's bridge lines. :param str bridgelines: The Bridge Lines which we are distributing to the client. :param bool bridgeSchema: If ``True``, prepend ``'bridge://'`` to the beginning of each bridge line before QR encoding. - :rtype: str or ``None`` - :returns: The generated QRCode, as a string. + :rtype: bytes or ``None`` + :returns: The generated QRCode, as a bytes. """ logging.debug("Attempting to encode bridge lines into a QRCode...") @@ -60,7 +60,7 @@ def generateQR(bridgelines, imageFormat=u'JPEG', bridgeSchema=False): qr = qrcode.QRCode() qr.add_data(bridgelines) - buf = cStringIO.StringIO() + buf = io.BytesIO() img = qr.make_image().resize([350, 350]) img.save(buf, imageFormat) buf.seek(0) diff --git a/bridgedb/schedule.py b/bridgedb/schedule.py index e962c39..a2ebf6a 100644 --- a/bridgedb/schedule.py +++ b/bridgedb/schedule.py @@ -49,7 +49,7 @@ def fromUnixSeconds(timestamp): :param int timestamp: A timestamp in Unix Era seconds. :rtype: :any:`datetime.datetime` """ - return datetime.fromtimestamp(timestamp) + return datetime.utcfromtimestamp(timestamp) class ISchedule(interface.Interface): @@ -200,13 +200,14 @@ class ScheduledInterval(Unscheduled): :raises UnknownInterval: if the specified **count** was invalid. """ try: - if not count > 0: + if count is None: count = 1 count = int(count) except (TypeError, ValueError): - raise UnknownInterval("%s.intervalCount: %r ist not an integer." + raise UnknownInterval("%s.intervalCount: %r is not an integer." % (self.__class__.__name__, count)) - self.intervalCount = count + + self.intervalCount = max(1, count) def _setIntervalPeriod(self, period=None): """Set our :attr:`intervalPeriod`. diff --git a/bridgedb/test/deprecated.py b/bridgedb/test/deprecated.py index ab7c502..1a71bf1 100644 --- a/bridgedb/test/deprecated.py +++ b/bridgedb/test/deprecated.py @@ -126,7 +126,7 @@ class Bridge(object): if not request: request = 'default' digest = getHMACFunc('Order-Or-Addresses')(request) - pos = long(digest[:8], 16) # lower 8 bytes -> long + pos = int(digest[:8], 16) # lower 8 bytes -> int # default address type if not addressClass: addressClass = ipaddr.IPv4Address diff --git a/bridgedb/test/email_helpers.py b/bridgedb/test/email_helpers.py index 645fc93..edc8196 100644 --- a/bridgedb/test/email_helpers.py +++ b/bridgedb/test/email_helpers.py @@ -52,7 +52,7 @@ EMAIL_FROM_ADDR = "bridges@localhost" EMAIL_BIND_IP = "127.0.0.1" EMAIL_PORT = 5225 -TEST_CONFIG_FILE = io.StringIO(unicode("""\ +TEST_CONFIG_FILE = io.StringIO("""\ EMAIL_DIST = %s EMAIL_ROTATION_PERIOD = %s EMAIL_INCLUDE_FINGERPRINTS = %s @@ -96,14 +96,14 @@ EMAIL_PORT = %s repr(EMAIL_N_BRIDGES_PER_ANSWER), repr(EMAIL_FROM_ADDR), repr(EMAIL_BIND_IP), - repr(EMAIL_PORT)))) + repr(EMAIL_PORT))) def _createConfig(configFile=TEST_CONFIG_FILE): configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(configFile.read(), '<string>', 'exec') - exec compiled in configuration + exec(compiled, configuration) config = Conf(**configuration) return config @@ -138,7 +138,7 @@ class DummyEmailDistributor(object): self.answerParameters = answerParameters def getBridges(self, bridgeRequest, epoch): - return [util.DummyBridge() for _ in xrange(self._bridgesPerResponseMin)] + return [util.DummyBridge() for _ in range(self._bridgesPerResponseMin)] def cleanDatabase(self): pass @@ -167,7 +167,7 @@ class DummyEmailDistributorWithState(DummyEmailDistributor): self.alreadySeen[bridgeRequest.client] += 1 if self.alreadySeen[bridgeRequest.client] <= 1: - return [util.DummyBridge() for _ in xrange(self._bridgesPerResponseMin)] + return [util.DummyBridge() for _ in range(self._bridgesPerResponseMin)] elif self.alreadySeen[bridgeRequest.client] == 2: raise TooSoonEmail( "Seen client '%s' %d times" diff --git a/bridgedb/test/https_helpers.py b/bridgedb/test/https_helpers.py index fad841e..ca268c9 100644 --- a/bridgedb/test/https_helpers.py +++ b/bridgedb/test/https_helpers.py @@ -50,7 +50,7 @@ CSP_ENABLED = True CSP_REPORT_ONLY = True CSP_INCLUDE_SELF = True -TEST_CONFIG_FILE = io.StringIO(unicode("""\ +TEST_CONFIG_FILE = io.StringIO("""\ SERVER_PUBLIC_FQDN = %r SERVER_PUBLIC_EXTERNAL_IP = %r HTTPS_DIST = %r @@ -100,14 +100,14 @@ CSP_INCLUDE_SELF = %r GIMP_CAPTCHA_RSA_KEYFILE, CSP_ENABLED, CSP_REPORT_ONLY, - CSP_INCLUDE_SELF))) + CSP_INCLUDE_SELF)) def _createConfig(configFile=TEST_CONFIG_FILE): configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(configFile.read(), '<string>', 'exec') - exec compiled in configuration + exec(compiled, configuration) config = Conf(**configuration) return config diff --git a/bridgedb/test/legacy_Tests.py b/bridgedb/test/legacy_Tests.py index 00e1d12..93b52a7 100644 --- a/bridgedb/test/legacy_Tests.py +++ b/bridgedb/test/legacy_Tests.py @@ -61,17 +61,17 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, transports=False): ip = randomIPv4() nn = "bridge-%s" % int(ip) - fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + fp = "".join([random.choice("0123456789ABCDEF") for _ in range(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) oraddrs = [] if or_addresses: - for i in xrange(8): + for i in range(8): b.orAddresses.append((randomValidIPv6(), randomPort(), 6)) if transports: - for i in xrange(0,8): + for i in range(0,8): b.transports.append(bridgedb.Bridges.PluggableTransport(b, random.choice(["obfs", "obfs2", "pt1"]), randomIP(), randomPort())) @@ -81,17 +81,17 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, transports=False): ip = randomIPv6() nn = "bridge-%s" % int(ip) - fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + fp = "".join([random.choice("0123456789ABCDEF") for _ in range(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) oraddrs = [] if or_addresses: - for i in xrange(8): + for i in range(8): b.orAddresses.append((randomValidIPv6(), randomPort(), 6)) if transports: - for i in xrange(0,8): + for i in range(0,8): b.transports.append(bridgedb.Bridges.PluggableTransport(b, random.choice(["obfs", "obfs2", "pt1"]), randomIP(), randomPort())) @@ -226,10 +226,10 @@ class BridgeStabilityTests(unittest.TestCase): db = self.db b = random.choice([fakeBridge,fakeBridge6])() def timestampSeries(x): - for i in xrange(61): + for i in range(61): yield (i+1)*60*30 + x # 30 minute intervals now = time.time() - time_on_address = long(60*30*60) # 30 hours + time_on_address = 60*30*60 # 30 hours downtime = 60*60*random.randint(0,4) # random hours of downtime for t in timestampSeries(now): @@ -244,22 +244,22 @@ class BridgeStabilityTests(unittest.TestCase): def testLastSeenWithDifferentAddressAndPort(self): db = self.db - for i in xrange(10): + for i in range(10): num_desc = 30 time_start = time.time() - ts = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + ts = [ 60*30*(i+1) + time_start for i in range(num_desc) ] b = random.choice([fakeBridge(), fakeBridge6()]) [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] # change the port b.orport = b.orport+1 last_seen = ts[-1] - ts = [ 60*30*(i+1) + last_seen for i in xrange(num_desc) ] + ts = [ 60*30*(i+1) + last_seen for i in range(num_desc) ] [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] b = db.getBridgeHistory(b.fingerprint) assert b.tosa == ts[-1] - last_seen - assert (long(last_seen*1000) == b.lastSeenWithDifferentAddressAndPort) - assert (long(ts[-1]*1000) == b.lastSeenWithThisAddressAndPort) + assert (last_seen*1000 == b.lastSeenWithDifferentAddressAndPort) + assert (ts[-1]*1000 == b.lastSeenWithThisAddressAndPort) def testFamiliar(self): # create some bridges @@ -267,11 +267,11 @@ class BridgeStabilityTests(unittest.TestCase): num_bridges = 10 num_desc = 4*48 # 30m intervals, 48 per day time_start = time.time() - bridges = [ fakeBridge() for x in xrange(num_bridges) ] + bridges = [ fakeBridge() for x in range(num_bridges) ] t = time.time() - ts = [ (i+1)*60*30+t for i in xrange(num_bridges) ] + ts = [ (i+1)*60*30+t for i in range(num_bridges) ] for b in bridges: - time_series = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + time_series = [ 60*30*(i+1) + time_start for i in range(num_desc) ] [ bridgedb.Stability.addOrUpdateBridgeHistory(b, i) for i in time_series ] assert None not in bridges # +1 to avoid rounding errors @@ -288,7 +288,7 @@ class BridgeStabilityTests(unittest.TestCase): num_bridges = 20 time_start = time.time() bridges = [random.choice([fakeBridge, fakeBridge6])() - for i in xrange(num_bridges)] + for i in range(num_bridges)] # run some of the bridges for the full time series running = bridges[:num_bridges/2] @@ -303,7 +303,7 @@ class BridgeStabilityTests(unittest.TestCase): # figure out how many intervals it will take for weightedUptime to # decay to < 1 num_desc = int(30*log(1/float(num_successful*30*60))/(-0.05)) - timeseries = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + timeseries = [ 60*30*(i+1) + time_start for i in range(num_desc) ] for i in timeseries: for b in running: diff --git a/bridgedb/test/moat_helpers.py b/bridgedb/test/moat_helpers.py index 07b44a0..236b529 100644 --- a/bridgedb/test/moat_helpers.py +++ b/bridgedb/test/moat_helpers.py @@ -48,7 +48,7 @@ MOAT_ROTATION_PERIOD = "3 hours" MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = 'moat_captcha_hmac_key' MOAT_GIMP_CAPTCHA_RSA_KEYFILE = 'moat_captcha_rsa_key' -TEST_CONFIG_FILE = io.StringIO(unicode("""\ +TEST_CONFIG_FILE = io.StringIO("""\ GIMP_CAPTCHA_DIR = %r SERVER_PUBLIC_FQDN = %r SUPPORTED_TRANSPORTS = %r @@ -88,13 +88,13 @@ MOAT_GIMP_CAPTCHA_RSA_KEYFILE = %r MOAT_N_IP_CLUSTERS, MOAT_ROTATION_PERIOD, MOAT_GIMP_CAPTCHA_HMAC_KEYFILE, - MOAT_GIMP_CAPTCHA_RSA_KEYFILE))) + MOAT_GIMP_CAPTCHA_RSA_KEYFILE)) def _createConfig(configFile=TEST_CONFIG_FILE): configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(configFile.read(), '<string>', 'exec') - exec compiled in configuration + exec(compiled, configuration) config = Conf(**configuration) return config diff --git a/bridgedb/test/test_Bridges.py b/bridgedb/test/test_Bridges.py index 5991952..77eff1c 100644 --- a/bridgedb/test/test_Bridges.py +++ b/bridgedb/test/test_Bridges.py @@ -62,7 +62,7 @@ class BridgeRingTests(unittest.TestCase): """ self.addBridgesFromSameSubnet() - chosen = self.ring.bridges.keys()[:10] + chosen = list(self.ring.bridges.keys())[:10] bridges = self.ring.filterDistinctSubnets(chosen) # Since they're all in the same /16, we should only get one @@ -75,7 +75,7 @@ class BridgeRingTests(unittest.TestCase): """ self.addRandomBridges() - chosen = self.ring.bridges.keys()[:3] + chosen = list(self.ring.bridges.keys())[:3] bridges = self.ring.filterDistinctSubnets(chosen) self.assertGreaterEqual(len(bridges), 1) @@ -92,7 +92,7 @@ class BridgeRingTests(unittest.TestCase): filtering by distinct subnets. """ self.addRandomBridges() - bridges = self.ring.getBridges('a' * Bridges.DIGEST_LEN, N=3, filterBySubnet=True) + bridges = self.ring.getBridges(b'a' * Bridges.DIGEST_LEN, N=3, filterBySubnet=True) self.assertEqual(len(bridges), 3) def test_dumpAssignments(self): @@ -107,7 +107,7 @@ class BridgeRingTests(unittest.TestCase): f.seek(0) data = f.read() - first = self.ring.bridges.values()[0].fingerprint + first = list(self.ring.bridges.values())[0].fingerprint # The first bridge's fingerprint should be within the data somewhere self.assertIn(first, data) @@ -149,7 +149,7 @@ class FixedBridgeSplitterTests(unittest.TestCase): f.seek(0) data = f.read() - first = self.splitter.rings[0].bridges.values()[0].fingerprint + first = list(self.splitter.rings[0].bridges.values())[0].fingerprint # The first bridge's fingerprint should be within the data somewhere self.assertIn(first, data) diff --git a/bridgedb/test/test_Storage.py b/bridgedb/test/test_Storage.py index dffba2b..ca10bb9 100644 --- a/bridgedb/test/test_Storage.py +++ b/bridgedb/test/test_Storage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unittests for the :mod:`bridgedb.Storage` module.""" import os diff --git a/bridgedb/test/test_bridgerequest.py b/bridgedb/test/test_bridgerequest.py index 681d54a..80b5534 100644 --- a/bridgedb/test/test_bridgerequest.py +++ b/bridgedb/test/test_bridgerequest.py @@ -48,11 +48,11 @@ class BridgeRequestBaseTests(unittest.TestCase): should use the default client identifier string. """ self.assertEqual(self.request.getHashringPlacement('AAAA'), - 3486762050L) + 3486762050) def test_BridgeRequestBase_getHashringPlacement_with_client(self): """BridgeRequestBase.getHashringPlacement() with a client parameter should use the client identifier string. """ self.assertEqual(self.request.getHashringPlacement('AAAA', client='you'), - 2870307088L) + 2870307088) diff --git a/bridgedb/test/test_bridges.py b/bridgedb/test/test_bridges.py index 709ae81..fea2d33 100644 --- a/bridgedb/test/test_bridges.py +++ b/bridgedb/test/test_bridges.py @@ -466,7 +466,7 @@ class BridgeAddressBaseTests(unittest.TestCase): cc = self.bab.country self.assertIsNotNone(cc) - self.assertIsInstance(cc, basestring) + self.assertIsInstance(cc, str) self.assertEqual(len(cc), 2) @@ -506,7 +506,7 @@ class PluggableTransportTests(unittest.TestCase): args = pt._parseArgumentsIntoDict(["sharedsecret=foobar", "publickey=1234"]) self.assertIsInstance(args, dict) - self.assertItemsEqual(args, {"sharedsecret": "foobar", + self.assertCountEqual(args, {"sharedsecret": "foobar", "publickey": "1234"}) def test_PluggableTransport_parseArgumentsIntoDict_valid_list_multi(self): @@ -517,7 +517,7 @@ class PluggableTransportTests(unittest.TestCase): args = pt._parseArgumentsIntoDict(["sharedsecret=foobar,password=baz", "publickey=1234"]) self.assertIsInstance(args, dict) - self.assertItemsEqual(args, {"sharedsecret": "foobar", + self.assertCountEqual(args, {"sharedsecret": "foobar", "password": "baz", "publickey": "1234"}) @@ -528,7 +528,7 @@ class PluggableTransportTests(unittest.TestCase): pt = bridges.PluggableTransport() args = pt._parseArgumentsIntoDict( ["sharedsecret=foobar,password,publickey=1234"]) - self.assertItemsEqual(args, {"sharedsecret": "foobar", + self.assertCountEqual(args, {"sharedsecret": "foobar", "publickey": "1234"}) def test_PluggableTransport_checkArguments_scramblesuit_missing_password(self): @@ -848,10 +848,10 @@ class BridgeTests(unittest.TestCase): self._networkstatusFile)[0] self.serverdescriptor = descriptors.parseServerDescriptorsFile( self._serverDescriptorFile)[0] - self.extrainfo = descriptors.parseExtraInfoFiles( - self._extrainfoFile).values()[0] - self.extrainfoNew = descriptors.parseExtraInfoFiles( - self._extrainfoNewFile).values()[0] + self.extrainfo = list(descriptors.parseExtraInfoFiles( + self._extrainfoFile).values())[0] + self.extrainfoNew = list(descriptors.parseExtraInfoFiles( + self._extrainfoNewFile).values())[0] def _writeNetworkstatus(self, networkstatus): with open(self._networkstatusFile, 'w') as fh: @@ -970,7 +970,7 @@ class BridgeTests(unittest.TestCase): self.assertEqual( identifier, ''.join(['$$', - hashlib.sha1(bridge.fingerprint).hexdigest().upper(), + hashlib.sha1(bridge.fingerprint.encode('utf-8')).hexdigest().upper(), '~', bridge.nickname])) def test_Bridge_str_without_fingerprint(self): @@ -1013,11 +1013,11 @@ class BridgeTests(unittest.TestCase): """ self.bridge.address = '1.1.1.1' self.bridge.orPort = 443 - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) def test_Bridge_allVanillaAddresses_idempotency_others(self): @@ -1027,17 +1027,17 @@ class BridgeTests(unittest.TestCase): """ self.bridge.address = '1.1.1.1' self.bridge.orPort = 443 - self.assertItemsEqual(self.bridge.orAddresses, []) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.orAddresses, []) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) def test_Bridge_allVanillaAddresses_reentrancy_all(self): """Bridge.allVanillaAddresses should be reentrant, i.e. updating the @@ -1045,34 +1045,34 @@ class BridgeTests(unittest.TestCase): returned by allVanillaAddresses. """ self.bridge.address = '1.1.1.1' - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), None, 4)]) self.assertEqual(self.bridge.address, ipaddr.IPv4Address('1.1.1.1')) self.assertEqual(self.bridge.orPort, None) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) self.bridge.orPort = 443 - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) self.assertEqual(self.bridge.address, ipaddr.IPv4Address('1.1.1.1')) self.assertEqual(self.bridge.orPort, 443) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) self.bridge.address = '2.2.2.2' - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('2.2.2.2'), 443, 4)]) self.assertEqual(self.bridge.address, ipaddr.IPv4Address('2.2.2.2')) self.assertEqual(self.bridge.orPort, 443) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) self.bridge.orAddresses.append( (ipaddr.IPv6Address('200::6ffb:11bb:a129'), 4443, 6)) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('2.2.2.2'), 443, 4), (ipaddr.IPv6Address('200::6ffb:11bb:a129'), 4443, 6)]) self.assertEqual(self.bridge.address, ipaddr.IPv4Address('2.2.2.2')) self.assertEqual(self.bridge.orPort, 443) - self.assertItemsEqual(self.bridge.orAddresses, + self.assertCountEqual(self.bridge.orAddresses, [(ipaddr.IPv6Address('200::6ffb:11bb:a129'), 4443, 6)]) def test_Bridge_allVanillaAddresses_reentrancy_orPort(self): @@ -1081,15 +1081,15 @@ class BridgeTests(unittest.TestCase): set, it should return the orPort. """ self.bridge.address = '1.1.1.1' - self.assertItemsEqual(self.bridge.orAddresses, []) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), None, 4)]) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) self.bridge.orPort = 443 - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) def test_Bridge_allVanillaAddresses_reentrancy_address(self): """Calling Bridge.allVanillaAddresses before Bridge.address is set @@ -1097,10 +1097,10 @@ class BridgeTests(unittest.TestCase): is set, it should return the address. """ self.bridge.orPort = 443 - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(None, 443, 4)]) self.bridge.address = '1.1.1.1' - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) def test_Bridge_allVanillaAddresses_reentrancy_orAddresses(self): @@ -1109,14 +1109,14 @@ class BridgeTests(unittest.TestCase): """ self.bridge.address = '1.1.1.1' self.bridge.orPort = 443 - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) - self.assertItemsEqual(self.bridge.orAddresses, []) + self.assertCountEqual(self.bridge.orAddresses, []) self.bridge.orAddresses.append( (ipaddr.IPv4Address('2.2.2.2'), 4443, 4)) - self.assertItemsEqual(self.bridge.orAddresses, + self.assertCountEqual(self.bridge.orAddresses, [(ipaddr.IPv4Address('2.2.2.2'), 4443, 4)]) - self.assertItemsEqual(self.bridge.allVanillaAddresses, + self.assertCountEqual(self.bridge.allVanillaAddresses, [(ipaddr.IPv4Address('2.2.2.2'), 4443, 4), (ipaddr.IPv4Address('1.1.1.1'), 443, 4)]) diff --git a/bridgedb/test/test_captcha.py b/bridgedb/test/test_captcha.py index 328a03c..b87251e 100644 --- a/bridgedb/test/test_captcha.py +++ b/bridgedb/test/test_captcha.py @@ -57,17 +57,17 @@ class ReCaptchaTests(unittest.TestCase): def test_get(self): """Test get() method.""" - # Force urllib2 to do anything less idiotic than the defaults: + # Force urllib.request to do anything less idiotic than the defaults: envkey = 'HTTPS_PROXY' oldkey = None - if os.environ.has_key(envkey): + if envkey in os.environ: oldkey = os.environ[envkey] os.environ[envkey] = '127.0.0.1:9150' # This stupid thing searches the environment for ``<protocol>_PROXY`` # variables, hence the above 'HTTPS_PROXY' env setting: - proxy = captcha.urllib2.ProxyHandler() - opener = captcha.urllib2.build_opener(proxy) - captcha.urllib2.install_opener(opener) + proxy = captcha.urllib.request.ProxyHandler() + opener = captcha.urllib.request.build_opener(proxy) + captcha.urllib.request.install_opener(opener) try: # There isn't really a reliable way to test this function! :( @@ -77,8 +77,8 @@ class ReCaptchaTests(unittest.TestCase): reason += "connection.\nThis test failed with: %s" % error raise unittest.SkipTest(reason) else: - self.assertIsInstance(self.c.image, basestring) - self.assertIsInstance(self.c.challenge, basestring) + self.assertIsInstance(self.c.image, bytes) + self.assertIsInstance(self.c.challenge, str) finally: # Replace the original environment variable if there was one: if oldkey: @@ -146,7 +146,7 @@ class GimpCaptchaTests(unittest.TestCase): c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey, self.cacheDir) challenge = c.createChallenge('w00t') - self.assertIsInstance(challenge, basestring) + self.assertIsInstance(challenge, str) def test_createChallenge_base64(self): """createChallenge() return value should be urlsafe base64-encoded.""" @@ -179,18 +179,18 @@ class GimpCaptchaTests(unittest.TestCase): self.assertEqual(hmac, correctHMAC) decrypted = self.sekrit.decrypt(orig) - timestamp = int(decrypted[:12].lstrip('0')) + timestamp = int(decrypted[:12].lstrip(b'0')) # The timestamp should be within 30 seconds of right now. self.assertApproximates(timestamp, int(time.time()), 30) - self.assertEqual('ThisAnswerShouldDecryptToThis', decrypted[12:]) + self.assertEqual(b'ThisAnswerShouldDecryptToThis', decrypted[12:]) def test_get(self): """GimpCaptcha.get() should return image and challenge strings.""" c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey, self.cacheDir) image, challenge = c.get() - self.assertIsInstance(image, basestring) - self.assertIsInstance(challenge, basestring) + self.assertIsInstance(image, bytes) + self.assertIsInstance(challenge, str) def test_get_emptyCacheDir(self): """An empty cacheDir should raise GimpCaptchaError.""" @@ -207,7 +207,7 @@ class GimpCaptchaTests(unittest.TestCase): with open(badFile, 'w') as fh: fh.write(' ') fh.flush() - os.chmod(badFile, 0266) + os.chmod(badFile, 0o266) c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey, self.badCacheDir) @@ -291,7 +291,7 @@ class GimpCaptchaTests(unittest.TestCase): c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey, self.cacheDir) image, challenge = c.get() - solution = unicode(c.answer) + solution = c.answer if isinstance(c.answer, str) else c.answer.decode('utf-8') self.assertEquals( c.check(challenge, solution, c.secretKey, c.hmacKey), True) diff --git a/bridgedb/test/test_crypto.py b/bridgedb/test/test_crypto.py index 7a34671..45c06ae 100644 --- a/bridgedb/test/test_crypto.py +++ b/bridgedb/test/test_crypto.py @@ -15,6 +15,7 @@ from __future__ import print_function from __future__ import unicode_literals import base64 +import binascii import io import logging import math @@ -63,15 +64,15 @@ class GetKeyTests(unittest.TestCase): """Test retrieving the secret_key from an empty file.""" filename = os.path.join(os.getcwd(), 'sekrit') key = crypto.getKey(filename) - self.failUnlessIsInstance(key, basestring, - "key isn't a string! type=%r" % type(key)) + self.failUnlessIsInstance(key, bytes, + "key isn't bytes! type=%r" % type(key)) def test_getKey_tmpfile(self): """Test retrieving the secret_key from a new tmpfile.""" filename = self.mktemp() key = crypto.getKey(filename) - self.failUnlessIsInstance(key, basestring, - "key isn't a string! type=%r" % type(key)) + self.failUnlessIsInstance(key, bytes, + "key isn't bytes! type=%r" % type(key)) def test_getKey_keyexists(self): """Write the example key to a file and test reading it back.""" @@ -81,13 +82,13 @@ class GetKeyTests(unittest.TestCase): fh.flush() key = crypto.getKey(filename) - self.failUnlessIsInstance(key, basestring, - "key isn't a string! type=%r" % type(key)) + self.failUnlessIsInstance(key, bytes, + "key isn't bytes! type=%r" % type(key)) self.assertEqual(SEKRIT_KEY, key, """The example key and the one read from file differ! key (in hex): %s SEKRIT_KEY (in hex): %s""" - % (key.encode('hex'), SEKRIT_KEY.encode('hex'))) + % (binascii.hexlify(key).decode('utf-8'), binascii.hexlify(SEKRIT_KEY).decode('utf-8'))) class InitializeGnuPGTests(unittest.TestCase): @@ -214,7 +215,7 @@ class InitializeGnuPGTests(unittest.TestCase): sig = signfunc("This is a test of the public broadcasting system.") print(sig) self.assertIsNotNone(sig) - self.assertTrue(sig.startswith('-----BEGIN PGP SIGNED MESSAGE-----')) + self.assertTrue(sig.startswith(b'-----BEGIN PGP SIGNED MESSAGE-----')) def test_crypto_initializeGnuPG_nonexistent_default_key(self): """When the key specified by EMAIL_GPG_PRIMARY_KEY_FINGERPRINT doesn't diff --git a/bridgedb/test/test_distributors_moat_request.py b/bridgedb/test/test_distributors_moat_request.py index 555b3eb..a5c22e0 100644 --- a/bridgedb/test/test_distributors_moat_request.py +++ b/bridgedb/test/test_distributors_moat_request.py @@ -30,7 +30,7 @@ class MoatBridgeRequest(unittest.TestCase): self.bridgeRequest.withoutBlockInCountry(data) self.bridgeRequest.generateFilters() - self.assertItemsEqual(['byTransportNotBlockedIn(vanilla,us,4)', + self.assertCountEqual(['byTransportNotBlockedIn(vanilla,us,4)', 'byTransportNotBlockedIn(vanilla,ir,4)', 'byTransportNotBlockedIn(vanilla,sy,4)', 'byProbingResistance(vanilla,4)'], diff --git a/bridgedb/test/test_distributors_moat_server.py b/bridgedb/test/test_distributors_moat_server.py index 33069ef..695116b 100644 --- a/bridgedb/test/test_distributors_moat_server.py +++ b/bridgedb/test/test_distributors_moat_server.py @@ -140,7 +140,7 @@ class MiscellaneousTests(unittest.TestCase): """ server.setSupportedTransports(self.config.SUPPORTED_TRANSPORTS) - self.assertItemsEqual(server.getSupportedTransports(), + self.assertCountEqual(server.getSupportedTransports(), ["obfs4", "obfs3", "scramblesuit", "fte", "vanilla"]) @@ -553,7 +553,7 @@ class CaptchaFetchResourceTests(unittest.TestCase): def test_getPreferredTransports_unknown_transport(self): preferred = self.resource.getPreferredTransports(['dinosaur']) - self.assertItemsEqual(preferred, + self.assertCountEqual(preferred, ['obfs4', 'obfs3', 'fte', 'scramblesuit', 'vanilla']) def assert_data_is_ok(self, decoded): @@ -884,7 +884,7 @@ class CaptchaCheckResourceTests(unittest.TestCase): response = self.resource.failureResponse(6, request, bridgeRequest) - self.assertIn("No bridges available", response) + self.assertIn(b"No bridges available", response) def test_render_POST_no_bridges(self): self.mock_getBridgeLines() @@ -892,7 +892,7 @@ class CaptchaCheckResourceTests(unittest.TestCase): request = self.create_valid_POST_make_new_challenge() response = self.resource.render(request) - self.assertIn("No bridges available", response) + self.assertIn(b"No bridges available", response) def test_render_POST_unexpired(self): request = self.create_valid_POST_make_new_challenge() diff --git a/bridgedb/test/test_email_autoresponder.py b/bridgedb/test/test_email_autoresponder.py index f59e5ef..c8e9624 100644 --- a/bridgedb/test/test_email_autoresponder.py +++ b/bridgedb/test/test_email_autoresponder.py @@ -235,7 +235,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" response = autoresponder.EmailResponse() response.write(self.body) response.rewind() - contents = str(response.read()).replace('\x00', '') + contents = response.read().replace(b'\x00', b'').decode('utf-8') # The newlines in the email body should have been replaced with # ``EmailResponse.delimiter``. delimited = self.body.replace('\n', response.delimiter) \ @@ -247,7 +247,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" response = autoresponder.EmailResponse() response.write(self.body) response.rewind() - contents = str(response.read(3)).replace('\x00', '') + contents = response.read(3).replace(b'\x00', b'').decode('utf-8') self.assertEqual(contents, self.body[:3]) def test_EmailResponse_write(self): @@ -370,7 +370,7 @@ class SMTPAutoresponderTests(unittest.TestCase): """ self._getIncomingLines() ours = Address(self.context.fromAddr) - plus = '@'.join([ours.local + '+zh_cn', ours.domain]) + plus = '@'.join([ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')]) self.message.lines[1] = 'To: {0}'.format(plus) self._setUpResponder() recipient = str(self.responder.getMailFrom()) @@ -383,7 +383,7 @@ class SMTPAutoresponderTests(unittest.TestCase): """ self._getIncomingLines() ours = Address(self.context.fromAddr) - plus = '@'.join(['get' + ours.local + '+zh_cn', ours.domain]) + plus = '@'.join(['get' + ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')]) self.message.lines[1] = 'To: {0}'.format(plus) self._setUpResponder() recipient = str(self.responder.getMailFrom()) diff --git a/bridgedb/test/test_email_distributor.py b/bridgedb/test/test_email_distributor.py index d95370f..716d190 100644 --- a/bridgedb/test/test_email_distributor.py +++ b/bridgedb/test/test_email_distributor.py @@ -234,8 +234,8 @@ class EmailDistributorTests(unittest.TestCase): # There should be an IPv4 subring and an IPv6 ring: ringnames = dist.hashring.filterRings.keys() - self.failUnlessIn("IPv4", "".join([str(ringname) for ringname in ringnames])) - self.failUnlessIn("IPv6", "".join([str(ringname) for ringname in ringnames])) + self.failUnlessIn("IPv", "".join([str(ringname) for ringname in ringnames])) + self.assertEqual(2, len(ringnames)) [dist.hashring.insert(bridge) for bridge in self.bridges] @@ -246,8 +246,8 @@ class EmailDistributorTests(unittest.TestCase): # Ugh, the hashring code is so gross looking. subrings = dist.hashring.filterRings - subring1 = subrings.values()[0][1] - subring2 = subrings.values()[1][1] + subring1 = list(subrings.values())[0][1] + subring2 = list(subrings.values())[1][1] # Each subring should have roughly the same number of bridges. # (Having ±10 bridges in either ring, out of 500 bridges total, should # be so bad.) diff --git a/bridgedb/test/test_email_dkim.py b/bridgedb/test/test_email_dkim.py index be34370..679575f 100644 --- a/bridgedb/test/test_email_dkim.py +++ b/bridgedb/test/test_email_dkim.py @@ -11,8 +11,9 @@ """Unittests for the :mod:`bridgedb.distributors.email.dkim` module.""" +import email +import email.message import io -import rfc822 from twisted.trial import unittest @@ -46,9 +47,8 @@ get bridges } def _createMessage(self, messageString): - """Create an ``rfc822.Message`` from a string.""" - messageIO = io.StringIO(unicode(messageString)) - return rfc822.Message(messageIO) + """Create an ``email.message.Message`` from a string.""" + return email.message_from_string(messageString) def test_checkDKIM_good(self): message = self._createMessage(self.goodMessage) diff --git a/bridgedb/test/test_email_server.py b/bridgedb/test/test_email_server.py index bc331f4..fb82cc0 100644 --- a/bridgedb/test/test_email_server.py +++ b/bridgedb/test/test_email_server.py @@ -13,10 +13,10 @@ from __future__ import print_function +import email.message import socket import string import types -import rfc822 from twisted.python import log from twisted.internet import defer @@ -95,10 +95,10 @@ class SMTPMessageTests(unittest.TestCase): defer.Deferred) def test_SMTPMessage_getIncomingMessage(self): - """``getIncomingMessage`` should return a ``rfc822.Message``.""" + """``getIncomingMessage`` should return a ``email.message.Message``.""" self.message.lineReceived(self.line) self.assertIsInstance(self.message.getIncomingMessage(), - rfc822.Message) + email.message.Message) class SMTPIncomingDeliveryTests(unittest.TestCase): @@ -154,7 +154,7 @@ class SMTPIncomingDeliveryTests(unittest.TestCase): """ self.helo = (domain, ipaddress) self._createProtocolWithHost(domain) - self.origin = Address('@'.join((username, domain,))) + self.origin = Address(b'@'.join((username, domain,))) self.user = User(username, self.helo, self.proto, self.origin) def _setUpMAILFROM(self): @@ -173,9 +173,9 @@ class SMTPIncomingDeliveryTests(unittest.TestCase): The default is to emulate sending: ``RCPT TO: bridges@localhost``. """ - name = username if username is not None else self.config.EMAIL_USERNAME - host = domain if domain is not None else 'localhost' - addr = ip if ip is not None else '127.0.0.1' + name = username if username is not None else self.config.EMAIL_USERNAME.encode('utf-8') + host = domain if domain is not None else b'localhost' + addr = ip if ip is not None else b'127.0.0.1' self._createUser(name, host, ip) self.delivery.setContext(self.context) @@ -203,7 +203,7 @@ class SMTPIncomingDeliveryTests(unittest.TestCase): ``'client@example.com'`` should contain a header stating: ``'Received: from example.com'``. """ - self._createUser('client', 'example.com', '127.0.0.1') + self._createUser(b'client', b'example.com', b'127.0.0.1') hdr = self.delivery.receivedHeader(self.helo, self.origin, [self.user,]) self.assertSubstring("Received: from example.com", hdr) @@ -237,14 +237,14 @@ class SMTPIncomingDeliveryTests(unittest.TestCase): def test_SMTPIncomingDelivery_validateTo_plusAddress(self): """Should return a callable that results in a SMTPMessage.""" - self._setUpRCPTTO('bridges+ar') + self._setUpRCPTTO(b'bridges+ar') validated = self.delivery.validateTo(self.user) self.assertIsInstance(validated, types.FunctionType) self.assertIsInstance(validated(), server.SMTPMessage) def test_SMTPIncomingDelivery_validateTo_badUsername_plusAddress(self): """'givemebridges+zh_cn@...' should raise an SMTPBadRcpt exception.""" - self._setUpRCPTTO('givemebridges+zh_cn') + self._setUpRCPTTO(b'givemebridges+zh_cn') self.assertRaises(SMTPBadRcpt, self.delivery.validateTo, self.user) def test_SMTPIncomingDelivery_validateTo_badUsername(self): @@ -252,20 +252,20 @@ class SMTPIncomingDeliveryTests(unittest.TestCase): ``RCPT TO: yo.mama@localhost`` should raise a ``twisted.mail.smtp.SMTPBadRcpt`` exception. """ - self._setUpRCPTTO('yo.mama') + self._setUpRCPTTO(b'yo.mama') self.assertRaises(SMTPBadRcpt, self.delivery.validateTo, self.user) def test_SMTPIncomingDelivery_validateTo_notOurDomain(self): """An SMTP ``RCPT TO: bridges@forealsi.es`` should raise an SMTPBadRcpt exception. """ - self._setUpRCPTTO('bridges', 'forealsi.es') + self._setUpRCPTTO(b'bridges', b'forealsi.es') self.assertRaises(SMTPBadRcpt, self.delivery.validateTo, self.user) def test_SMTPIncomingDelivery_validateTo_subdomain(self): """An SMTP ``RCPT TO: bridges@subdomain.localhost`` should be allowed. """ - self._setUpRCPTTO('bridges', 'subdomain.localhost') + self._setUpRCPTTO(b'bridges', b'subdomain.localhost') validated = self.delivery.validateTo(self.user) self.assertIsInstance(validated, types.FunctionType) self.assertIsInstance(validated(), server.SMTPMessage) @@ -339,7 +339,7 @@ class SMTPTestCaseMixin(util.TestCaseMixin): 'Subject: %s' % subject, '\r\n %s' % body, '.'] # SMTP DATA EOM command - emailText = self.proto.delimiter.join(contents) + emailText = self.proto.delimiter.decode('utf-8').join(contents) return emailText def _buildSMTP(self, commands): @@ -351,7 +351,7 @@ class SMTPTestCaseMixin(util.TestCaseMixin): :returns: The string for sending those **commands**, suitable for giving to a :api:`twisted.internet.Protocol.dataReceived` method. """ - data = self.proto.delimiter.join(commands) + self.proto.delimiter + data = self.proto.delimiter.decode('utf-8').join(commands) + self.proto.delimiter.decode('utf-8') return data def _test(self, commands, expected, noisy=False): @@ -369,8 +369,8 @@ class SMTPTestCaseMixin(util.TestCaseMixin): "client" and the "server" in a nicely formatted manner. """ data = self._buildSMTP(commands) - self.proto.dataReceived(data) - recv = self.transport.value() + self.proto.dataReceived(data.encode('utf-8')) + recv = self.transport.value().decode('utf-8') if noisy: client = data.replace('\r\n', '\r\n ') @@ -515,7 +515,12 @@ class EmailServerServiceTests(SMTPTestCaseMixin, unittest.TestCase): def tearDown(self): """Kill all connections with fire.""" if self.transport: + if not hasattr(self.transport, 'protocol'): + factory = server.addServer(self.config, self.dist) + self.transport.protocol = factory.buildProtocol(('127.0.0.1', 0)) + self.transport.loseConnection() + super(EmailServerServiceTests, self).tearDown() # FIXME: this is definitely not how we're supposed to do this, but it # kills the DirtyReactorAggregateErrors. diff --git a/bridgedb/test/test_email_templates.py b/bridgedb/test/test_email_templates.py index 8464fe9..4703019 100644 --- a/bridgedb/test/test_email_templates.py +++ b/bridgedb/test/test_email_templates.py @@ -27,7 +27,7 @@ class EmailTemplatesTests(unittest.TestCase): """Unittests for :func:`b.e.templates`.""" def setUp(self): - self.t = NullTranslations(StringIO(unicode('test'))) + self.t = NullTranslations(StringIO('test')) self.client = Address('blackhole@torproject.org') self.answer = 'obfs3 1.1.1.1:1111\nobfs3 2.2.2.2:2222' # This is the fingerprint of BridgeDB's offline, certification-only diff --git a/bridgedb/test/test_geo.py b/bridgedb/test/test_geo.py index 4ca6cbb..371882c 100644 --- a/bridgedb/test/test_geo.py +++ b/bridgedb/test/test_geo.py @@ -50,7 +50,7 @@ class GeoTests(unittest.TestCase): """Should return the CC since the IP is an ``ipaddr.IPAddress``.""" cc = geo.getCountryCode(self.ipv4) self.assertIsNotNone(cc) - self.assertIsInstance(cc, basestring) + self.assertIsInstance(cc, str) self.assertEqual(len(cc), 2) self.assertEqual(cc, self.expectedCC) @@ -77,7 +77,7 @@ class GeoTests(unittest.TestCase): """Should return the CC since the IP is an ``ipaddr.IPAddress``.""" cc = geo.getCountryCode(self.ipv6) self.assertIsNotNone(cc) - self.assertIsInstance(cc, basestring) + self.assertIsInstance(cc, str) self.assertEqual(len(cc), 2) self.assertEqual(cc, self.expectedCC) diff --git a/bridgedb/test/test_https.py b/bridgedb/test/test_https.py index ffd7281..8e3de1b 100644 --- a/bridgedb/test/test_https.py +++ b/bridgedb/test/test_https.py @@ -27,11 +27,11 @@ repository. from __future__ import print_function import gettext -import ipaddr +import ipaddress import mechanize import os -from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from twisted.trial import unittest from twisted.trial.reporter import TestResult @@ -137,7 +137,7 @@ class HTTPTests(unittest.TestCase): # ------------- Results # URL should be the same as last time self.assertEquals(self.br.response().geturl(), EXPECTED_URL) - soup = BeautifulSoup(captcha_response.read()) + soup = BeautifulSoup(captcha_response.read(), features="html5lib") return soup def getBridgeLinesFromSoup(self, soup, fieldsPerBridge): @@ -174,10 +174,10 @@ class HTTPTests(unittest.TestCase): self.br.set_debug_http(True) self.br.open(HTTP_ROOT) - headers = ''.join(self.br.response().info().headers) + headers = self.br.response().info() - self.assertIn("Content-Security-Policy", headers) - self.assertIn("default-src 'none';", headers) + self.assertIn("Content-Security-Policy", headers.keys()) + self.assertIn("default-src 'none';", ''.join(headers.values())) def test_404(self): """Asking for a non-existent resource should yield our custom 404 page, @@ -199,7 +199,7 @@ class HTTPTests(unittest.TestCase): for bridge in bridges: self.assertTrue(bridge != None) addr = bridge[0].rsplit(':', 1)[0] - self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(addr), ipaddress.IPv4Address) def test_get_vanilla_ipv6(self): self.openBrowser() @@ -212,7 +212,7 @@ class HTTPTests(unittest.TestCase): for bridge in bridges: self.assertTrue(bridge != None) addr = bridge[0].rsplit(':', 1)[0].strip('[]') - self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv6Address) + self.assertIsInstance(ipaddress.ip_address(addr), ipaddress.IPv6Address) def test_get_obfs4_ipv4(self): """Try asking for obfs4 bridges, and check that the PT arguments in the @@ -353,7 +353,7 @@ class _HTTPTranslationsTests(unittest.TestCase): localedir=self.i18n, languages=[locale,], fallback=True) - expected = language.gettext("What are bridges?") + expected = language.gettext("What are bridges?").encode("utf-8") if not locale.startswith('en'): self.assertNotEqual(expected, "What are bridges?") diff --git a/bridgedb/test/test_https_distributor.py b/bridgedb/test/test_https_distributor.py index c25c598..a96238d 100644 --- a/bridgedb/test/test_https_distributor.py +++ b/bridgedb/test/test_https_distributor.py @@ -261,7 +261,7 @@ class HTTPSDistributorTests(unittest.TestCase): [dist.insert(bridge) for bridge in bridges] - for i in xrange(5): + for i in range(5): bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') bridgeRequest1.transports.append('obfs2') bridgeRequest1.generateFilters() @@ -322,7 +322,7 @@ class HTTPSDistributorTests(unittest.TestCase): for i in range(5): responses[i] = dist.getBridges(bridgeRequest, 1) for i in range(4): - self.assertItemsEqual(responses[i], responses[i+1]) + self.assertCountEqual(responses[i], responses[i+1]) def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): param = BridgeRingParameters(needPorts=[(443, 1)]) @@ -335,7 +335,7 @@ class HTTPSDistributorTests(unittest.TestCase): [dist.insert(bridge) for bridge in bridges] [dist.insert(bridge) for bridge in self.bridges[:250]] - for _ in xrange(32): + for _ in range(32): bridgeRequest = self.randomClientRequest() answer = dist.getBridges(bridgeRequest, 1) count = 0 @@ -399,7 +399,7 @@ class HTTPSDistributorTests(unittest.TestCase): dist = distributor.HTTPSDistributor(3, self.key) [dist.insert(bridge) for bridge in self.bridges[:250]] - for i in xrange(500): + for i in range(500): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv6() bridgeRequest.generateFilters() @@ -421,7 +421,7 @@ class HTTPSDistributorTests(unittest.TestCase): dist = distributor.HTTPSDistributor(1, self.key) [dist.insert(bridge) for bridge in self.bridges[:250]] - for i in xrange(500): + for i in range(500): bridgeRequest = self.randomClientRequest() bridgeRequest.generateFilters() diff --git a/bridgedb/test/test_https_request.py b/bridgedb/test/test_https_request.py index 2ea3fe6..81f8e0b 100644 --- a/bridgedb/test/test_https_request.py +++ b/bridgedb/test/test_https_request.py @@ -64,7 +64,7 @@ class HTTPSBridgeRequestTests(unittest.TestCase): self.request = request.HTTPSBridgeRequest(addClientCountryCode=False) self.request.client = '5.5.5.5' self.request.withoutBlockInCountry(httprequest) - self.assertItemsEqual(['nl'], self.request.notBlockedIn) + self.assertCountEqual(['nl'], self.request.notBlockedIn) def test_HTTPSBridgeRequest_withoutBlockInCountry_bad_params(self): """HTTPSBridgeRequest.withoutBlockInCountry() should stop processing if diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 945ea06..ff7a1e3 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -17,9 +17,9 @@ import logging import os import shutil -import ipaddr +import ipaddress -from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from twisted.internet import reactor from twisted.internet import task @@ -27,7 +27,7 @@ from twisted.trial import unittest from twisted.web.resource import Resource from twisted.web.test import requesthelper -from bridgedb import translations +from bridgedb import _langs, translations from bridgedb.distributors.https import server from bridgedb.schedule import ScheduledInterval @@ -70,8 +70,8 @@ class ReplaceErrorPageTests(unittest.TestCase): request = DummyRequest(['']) exc = Exception("vegan gümmibären") errorPage = server.replaceErrorPage(request, exc) - self.assertSubstring("Bad News Bears", errorPage) - self.assertNotSubstring("vegan gümmibären", errorPage) + self.assertSubstring(b"Bad News Bears", errorPage) + self.assertNotSubstring("vegan gümmibären".encode("utf-8"), errorPage) def test_replaceErrorPage_matches_resource500(self): """``replaceErrorPage`` should return the error-500.html page.""" @@ -89,9 +89,9 @@ class ReplaceErrorPageTests(unittest.TestCase): exc = Exception("vegan gümmibären") server.resource500 = None errorPage = server.replaceErrorPage(request, exc) - self.assertNotSubstring("Bad News Bears", errorPage) - self.assertNotSubstring("vegan gümmibären", errorPage) - self.assertSubstring("Sorry! Something went wrong with your request.", + self.assertNotSubstring(b"Bad News Bears", errorPage) + self.assertNotSubstring("vegan gümmibären".encode("utf-8"), errorPage) + self.assertSubstring(b"Sorry! Something went wrong with your request.", errorPage) class ErrorResourceTests(unittest.TestCase): @@ -103,17 +103,17 @@ class ErrorResourceTests(unittest.TestCase): def test_resource404(self): """``server.resource404`` should display the error-404.html page.""" page = server.resource404.render(self.request) - self.assertSubstring('We dug around for the page you requested', page) + self.assertSubstring(b'We dug around for the page you requested', page) def test_resource500(self): """``server.resource500`` should display the error-500.html page.""" page = server.resource500.render(self.request) - self.assertSubstring('Bad News Bears', page) + self.assertSubstring(b'Bad News Bears', page) def test_maintenance(self): """``server.maintenance`` should display the error-503.html page.""" page = server.maintenance.render(self.request) - self.assertSubstring('Under Maintenance', page) + self.assertSubstring(b'Under Maintenance', page) class CustomErrorHandlingResourceTests(unittest.TestCase): @@ -214,15 +214,19 @@ class IndexResourceTests(unittest.TestCase): request = DummyRequest([self.pagename]) request.method = b'GET' page = self.indexResource.render_GET(request) - self.assertSubstring("add the bridges to Tor Browser", page) + self.assertSubstring(b"add the bridges to Tor Browser", page) def test_IndexResource_render_GET_lang_ta(self): """renderGet() with ?lang=ta should return the index page in Tamil.""" + + if 'ta' not in _langs.get_langs(): + self.skipTest("'ta' language unsupported") + request = DummyRequest([self.pagename]) request.method = b'GET' - request.addArg('lang', 'ta') + request.addArg(b'lang', 'ta') page = self.indexResource.render_GET(request) - self.assertSubstring("bridge-களை Tor Browser-உள்", page) + self.assertSubstring("bridge-களை Tor Browser-உள்".encode("utf-8"), page) class HowtoResourceTests(unittest.TestCase): @@ -239,15 +243,19 @@ class HowtoResourceTests(unittest.TestCase): request = DummyRequest([self.pagename]) request.method = b'GET' page = self.howtoResource.render_GET(request) - self.assertSubstring("the wizard", page) + self.assertSubstring(b"the wizard", page) def test_HowtoResource_render_GET_lang_ru(self): """renderGet() with ?lang=ru should return the howto page in Russian.""" + + if 'ru' not in _langs.get_langs(): + self.skipTest("'ru' language unsupported") + request = DummyRequest([self.pagename]) request.method = b'GET' - request.addArg('lang', 'ru') + request.addArg(b'lang', 'ru') page = self.howtoResource.render_GET(request) - self.assertSubstring("следовать инструкциям установщика", page) + self.assertSubstring("следовать инструкциям установщика".encode("utf-8"), page) class CaptchaProtectedResourceTests(unittest.TestCase): @@ -271,7 +279,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase): request.method = b'GET' page = self.captchaResource.render_GET(request) self.assertSubstring( - "Your browser is not displaying images properly", page) + b"Your browser is not displaying images properly", page) def test_render_GET_missingTemplate(self): """render_GET() with a missing template should raise an error and @@ -329,7 +337,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase): request = DummyRequest([self.pagename]) request.method = b'POST' page = self.captchaResource.render_POST(request) - self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], + self.assertEqual(BeautifulSoup(page, features="html5lib").find('meta')['http-equiv'], 'refresh') @@ -454,7 +462,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.addArg('captcha_response_field', '') page = self.captchaResource.render_POST(self.request) - self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], + self.assertEqual(BeautifulSoup(page, features="html5lib").find('meta')['http-equiv'], 'refresh') def test_render_POST_wrongSolution(self): @@ -469,7 +477,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.addArg('captcha_response_field', expectedResponse) page = self.captchaResource.render_POST(self.request) - self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], + self.assertEqual(BeautifulSoup(page, features="html5lib").find('meta')['http-equiv'], 'refresh') @@ -522,7 +530,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): def testCB(request): """Check the ``Request`` returned from ``_renderDeferred``.""" self.assertIsInstance(request, DummyRequest) - soup = BeautifulSoup(b''.join(request.written)).find('meta')['http-equiv'] + soup = BeautifulSoup(b''.join(request.written), features="html5lib").find(b'meta')['http-equiv'] self.assertEqual(soup, 'refresh') d = task.deferLater(reactor, 0, lambda x: x, (False, self.request)) @@ -541,7 +549,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """Check the ``Request`` returned from ``_renderDeferred``.""" self.assertIsInstance(request, DummyRequest) html = b''.join(request.written) - self.assertSubstring('Uh oh, spaghettios!', html) + self.assertSubstring(b'Uh oh, spaghettios!', html) d = task.deferLater(reactor, 0, lambda x: x, (True, self.request)) d.addCallback(self.captchaResource._renderDeferred) @@ -580,14 +588,14 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """Check that removing our remoteip setting produces a random IP.""" self.captchaResource.remoteIP = None ip = self.captchaResource.getRemoteIP() - realishIP = ipaddr.IPv4Address(ip).compressed + realishIP = ipaddress.IPv4Address(ip).compressed self.assertTrue(realishIP) self.assertNotEquals(realishIP, '111.111.111.111') def test_getRemoteIP_useConfiguredIP(self): """Check that our remoteip setting is used if configured.""" ip = self.captchaResource.getRemoteIP() - realishIP = ipaddr.IPv4Address(ip).compressed + realishIP = ipaddress.IPv4Address(ip).compressed self.assertTrue(realishIP) self.assertEquals(realishIP, '111.111.111.111') @@ -666,9 +674,9 @@ class BridgesResourceTests(unittest.TestCase): :returns: A list of the bridge lines contained on the **page**. """ # The bridge lines are contained in a <div class='bridges'> tag: - soup = BeautifulSoup(page) + soup = BeautifulSoup(page, features="html5lib") well = soup.find('div', {'class': 'bridge-lines'}) - content = well.renderContents().strip() + content = well.renderContents().decode('utf-8').strip() lines = content.splitlines() bridges = [] @@ -766,7 +774,7 @@ class BridgesResourceTests(unittest.TestCase): # Check that the IP and port seem okay: ip, port = fields[0].rsplit(':') - self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(ip), ipaddress.IPv4Address) self.assertIsInstance(int(port), int) self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) @@ -794,6 +802,10 @@ class BridgesResourceTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for plain bridges in Arabic.""" + + if 'ar' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + self.useBenignBridges() request = DummyRequest([b"bridges?transport=obfs3"]) @@ -804,10 +816,10 @@ class BridgesResourceTests(unittest.TestCase): request.headers.update({'accept-language': 'ar,en,en_US,'}) page = self.bridgesResource.render(request) - self.assertSubstring("rtl.css", page) + self.assertSubstring(b"rtl.css", page) self.assertSubstring( # "I need an alternative way to get bridges!" - "أحتاج إلى وسيلة بديلة للحصول على bridges", page) + "أحتاج إلى وسيلة بديلة للحصول على bridges".encode("utf-8"), page) for bridgeLine in self.parseBridgesFromHTMLPage(page): # Check that each bridge line had the expected number of fields: @@ -816,6 +828,10 @@ class BridgesResourceTests(unittest.TestCase): def test_render_GET_RTLlang_obfs3(self): """Test rendering a request for obfs3 bridges in Farsi.""" + + if 'fa' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + self.useBenignBridges() request = DummyRequest([b"bridges?transport=obfs3"]) @@ -827,12 +843,12 @@ class BridgesResourceTests(unittest.TestCase): request.args.update({'transport': ['obfs3']}) page = self.bridgesResource.render(request) - self.assertSubstring("rtl.css", page) + self.assertSubstring(b"rtl.css", page) self.assertSubstring( # "How to use the above bridge lines" (since there should be # bridges in this response, we don't tell them about alternative # mechanisms for getting bridges) - "چگونگی از پلهای خود استفاده کنید", page) + "چگونگی از پلهای خود استفاده کنید".encode("utf-8"), page) for bridgeLine in self.parseBridgesFromHTMLPage(page): # Check that each bridge line had the expected number of fields: @@ -842,7 +858,7 @@ class BridgesResourceTests(unittest.TestCase): # Check that the IP and port seem okay: ip, port = bridgeLine[1].rsplit(':') - self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(ip), ipaddress.IPv4Address) self.assertIsInstance(int(port), int) self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) @@ -868,15 +884,15 @@ class BridgesResourceTests(unittest.TestCase): # # (Yes, there are two leading spaces at the beginning of each line) # - bridgeLines = [line.strip() for line in page.strip().split('\n')] + bridgeLines = [line.strip() for line in page.strip().split(b'\n')] for bridgeLine in bridgeLines: - bridgeLine = bridgeLine.split(' ') + bridgeLine = bridgeLine.split(b' ') self.assertEqual(len(bridgeLine), 2) # Check that the IP and port seem okay: - ip, port = bridgeLine[0].rsplit(':') - self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address) + ip, port = bridgeLine[0].rsplit(b':') + self.assertIsInstance(ipaddress.ip_address(ip.decode("utf-8")), ipaddress.IPv4Address) self.assertIsInstance(int(port), int) self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) @@ -897,8 +913,8 @@ class BridgesResourceTests(unittest.TestCase): page = self.bridgesResource.renderAnswer(request, bridgeLines=None) # We don't want the fancy version: - self.assertNotSubstring("Bad News Bears", page) - self.assertSubstring("Sorry! Something went wrong with your request.", + self.assertNotSubstring(b"Bad News Bears", page) + self.assertSubstring(b"Sorry! Something went wrong with your request.", page) @@ -915,6 +931,10 @@ class OptionsResourceTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for obfs3 bridges in Hebrew.""" + + if 'he' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + request = DummyRequest(["bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' @@ -924,8 +944,8 @@ class OptionsResourceTests(unittest.TestCase): request.args.update({'transport': ['obfs2']}) page = self.optionsResource.render(request) - self.assertSubstring("rtl.css", page) - self.assertSubstring("מהם גשרים?", page) + self.assertSubstring(b"rtl.css", page) + self.assertSubstring("מהם גשרים?".encode("utf-8"), page) class HTTPSServerServiceTests(unittest.TestCase): diff --git a/bridgedb/test/test_main.py b/bridgedb/test/test_main.py index 52d98a2..82b609f 100644 --- a/bridgedb/test/test_main.py +++ b/bridgedb/test/test_main.py @@ -52,7 +52,7 @@ def mockUpdateBridgeHistory(bridges, timestamps): which doesn't access the database (so that we can test functions which call it, like :func:`bridgedb.main.load`). """ - for fingerprint, stamps in timestamps.items()[:]: + for fingerprint, stamps in timestamps.items(): for timestamp in stamps: print("Pretending to update Bridge %s with timestamp %s..." % (fingerprint, timestamp)) @@ -323,7 +323,7 @@ class BridgedbTests(unittest.TestCase): # a MoatDistributor ring, and an UnallocatedHolder ring: self.assertEqual(len(hashring.ringsByName.keys()), 4) self.assertGreater(len(httpsDist.proxies), 0) - self.assertItemsEqual(exitRelays, httpsDist.proxies) + self.assertCountEqual(exitRelays, httpsDist.proxies) def test_main_createBridgeRings_no_https_dist(self): """When HTTPS_DIST=False, main.createBridgeRings() should add only diff --git a/bridgedb/test/test_metrics.py b/bridgedb/test/test_metrics.py index ce0f63d..fcc9e4a 100644 --- a/bridgedb/test/test_metrics.py +++ b/bridgedb/test/test_metrics.py @@ -15,7 +15,7 @@ These tests are meant to ensure that the :mod:`bridgedb.metrics` module is functioning as expected. """ -import StringIO +import io import json import os @@ -104,7 +104,7 @@ class StateTest(unittest.TestCase): self.metrix.inc(self.key) self.metrix.coldMetrics = self.metrix.hotMetrics - pseudo_fh = StringIO.StringIO() + pseudo_fh = io.StringIO() metrics.export(pseudo_fh, 0) self.assertTrue(len(pseudo_fh.getvalue()) > 0) diff --git a/bridgedb/test/test_parse_addr.py b/bridgedb/test/test_parse_addr.py index 4659efb..b32d359 100644 --- a/bridgedb/test/test_parse_addr.py +++ b/bridgedb/test/test_parse_addr.py @@ -226,7 +226,7 @@ class ParseAddrIsIPAddressTests(unittest.TestCase): log.msg("addr.isIPAddress(%r) => %s" % (testAddress, result)) self.assertTrue(result is not None, "Got a None for testAddress: %r" % testAddress) - self.assertFalse(isinstance(result, basestring), + self.assertFalse(isinstance(result, str), "Expected %r result from isIPAddress(%r): %r %r" % (bool, testAddress, result, type(result))) @@ -669,11 +669,11 @@ class PortListTest(unittest.TestCase): ports. """ tooMany = addr.PortList.PORTSPEC_LEN + 1 - ports = [self.getRandomPort() for x in xrange(tooMany)] + ports = [self.getRandomPort() for x in range(tooMany)] log.msg("Testing addr.PortList(%s))" % ', '.join([type('')(port) for port in ports])) portList = addr.PortList(*ports) - self.assertEqual(len(portList), tooMany) + self.assertEqual(len(portList), len(set(ports))) def test_invalidPortNumber(self): """Test creating a :class:`addr.PortList` with an invalid port. @@ -699,8 +699,8 @@ class PortListTest(unittest.TestCase): ports = (443, 9001, 9030) portList = addr.PortList(*ports) iterator = iter(portList) - for x in xrange(len(ports)): - self.assertIn(iterator.next(), portList) + for x in range(len(ports)): + self.assertIn(next(iterator), portList) def test_str(self): """Test creating a :class:`addr.PortList` with valid ports. @@ -709,7 +709,7 @@ class PortListTest(unittest.TestCase): """ ports = (443, 9001, 9030) portList = addr.PortList(*ports) - self.assertTrue(isinstance(str(portList), basestring)) + self.assertTrue(isinstance(str(portList), str)) for port in ports: self.assertIn(str(port), str(portList)) @@ -735,7 +735,7 @@ class PortListTest(unittest.TestCase): """Test ``__getitem__`` with a string.""" ports = (443, 9001, 9030) portList = addr.PortList(*ports) - self.assertEqual(portList.__getitem__(long(0)), 9001) + self.assertEqual(portList.__getitem__(0), 9001) def test_mixedArgs(self): """Create a :class:`addr.PortList` with mixed type parameters.""" diff --git a/bridgedb/test/test_parse_descriptors.py b/bridgedb/test/test_parse_descriptors.py index df7f34f..eb86a0e 100644 --- a/bridgedb/test/test_parse_descriptors.py +++ b/bridgedb/test/test_parse_descriptors.py @@ -28,7 +28,7 @@ try: from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors -except (ImportError, NameError), error: +except (ImportError, NameError) as error: print("There was an error importing stem: %s" % error) else: HAS_STEM = True @@ -36,7 +36,7 @@ else: from bridgedb.test.util import Benchmarker -BRIDGE_NETWORKSTATUS_0 = '''\ +BRIDGE_NETWORKSTATUS_0 = b'''\ r MiserLandfalls 4IsyTSCtChPhFPAnq5rD8yymlqA /GMC4lz8RXT/62v6kZNdmzSmopk 2014-11-04 06:23:22 2.215.61.223 4056 0 a [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 s Fast Guard Running Stable Valid @@ -44,7 +44,7 @@ w Bandwidth=1678904 p reject 1-65535 ''' -BRIDGE_NETWORKSTATUS_1 = '''\ +BRIDGE_NETWORKSTATUS_1 = b'''\ r Unmentionable BgOrX0ViP5hNsK5ZvixAuPZ6EY0 NTg9NoE5ls9KjF96Dp/UdrabZ9Y 2014-11-04 12:23:37 80.44.173.87 51691 0 a [da14:7d1e:ba8e:60d0:b078:3f88:382b:5c70]:51690 s Fast Guard Running Stable Valid @@ -52,7 +52,7 @@ w Bandwidth=24361 p reject 1-65535 ''' -BRIDGE_SERVER_DESCRIPTOR = '''\ +BRIDGE_SERVER_DESCRIPTOR = b'''\ @purpose bridge router MiserLandfalls 2.215.61.223 4056 0 0 or-address [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 @@ -86,7 +86,7 @@ P2aB/+XQfzFBA5TaWF83coDng4OGodhwHaOx10Kn7Bg= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-11-04 06:23:22 write-history 2014-11-04 06:23:22 (900 s) 3188736,2226176,2866176 @@ -117,7 +117,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-11-04 08:10:25 write-history 2014-11-04 08:10:25 (900 s) 3188736,2226176,2866176,2226176 @@ -148,7 +148,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-12-04 03:10:25 write-history 2014-12-04 03:10:25 (900 s) 3188736,2226176,2866176,2226176 @@ -179,7 +179,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_SERVER_DESCRIPTOR_ED25519 = '''\ +BRIDGE_SERVER_DESCRIPTOR_ED25519 = b'''\ @purpose bridge router piratepartei 80.92.79.70 80 0 0 identity-ed25519 @@ -233,7 +233,7 @@ VC4FdHgFlAkXbiqkpWtD0ojJJjLlEeXbmGILjC1Ls2I= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = b'''\ extra-info piratepartei 312D64274C29156005843EECB19C6865FA3CC10C identity-ed25519 -----BEGIN ED25519 CERT----- @@ -307,7 +307,7 @@ class ParseDescriptorsTests(unittest.TestCase): :returns: The full path to the file which was written to. """ descFilename = os.path.join(os.getcwd(), filename) - with open(descFilename, 'w') as fh: + with open(descFilename, 'wb') as fh: for desc in descriptors: fh.write(desc) fh.flush() @@ -381,8 +381,8 @@ class ParseDescriptorsTests(unittest.TestCase): raise InvalidRouterNickname. """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - 'MiserLandfalls', - 'MiserLandfallsWaterfallsSnowfallsAvalanche') + b'MiserLandfalls', + b'MiserLandfallsWaterfallsSnowfallsAvalanche') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -399,8 +399,8 @@ class ParseDescriptorsTests(unittest.TestCase): See also: :trac:`16616` """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - 's Fast Guard Running Stable Valid', - 's Fast Guard Running Stable Valid HSDir') + b's Fast Guard Running Stable Valid', + b's Fast Guard Running Stable Valid HSDir') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -420,7 +420,7 @@ class ParseDescriptorsTests(unittest.TestCase): a ValueError. """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - '2.215.61.223', '[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]') + b'2.215.61.223', b'[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]') descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', unparseable) self.assertRaises(ValueError, @@ -434,9 +434,9 @@ class ParseDescriptorsTests(unittest.TestCase): expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' - with open(descFile, 'w') as fh: - fh.write('signature and stuff from the BridgeAuth would go here\n') - fh.write('some more annotations with parameters and stuff\n') + with open(descFile, 'wb') as fh: + fh.write(b'signature and stuff from the BridgeAuth would go here\n') + fh.write(b'some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() @@ -454,9 +454,9 @@ class ParseDescriptorsTests(unittest.TestCase): expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' - with open(descFile, 'w') as fh: - fh.write('signature and stuff from the BridgeAuth would go here\n') - fh.write('some more annotations with parameters and stuff\n') + with open(descFile, 'wb') as fh: + fh.write(b'signature and stuff from the BridgeAuth would go here\n') + fh.write(b'some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() @@ -479,7 +479,7 @@ class ParseDescriptorsTests(unittest.TestCase): """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseExtraInfoFiles(descFile) - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseExtraInfoFiles_one_file(self): @@ -488,12 +488,12 @@ class ParseDescriptorsTests(unittest.TestCase): """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseExtraInfoFiles(descFile) - bridge = routers.values()[0] + bridge = list(routers.values())[0] # The number of transports we parsed should be equal to the number of # 'transport' lines in the descriptor: self.assertEqual(len(bridge.transport), - BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) + BRIDGE_EXTRA_INFO_DESCRIPTOR.count(b'transport ')) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) @@ -522,7 +522,7 @@ class ParseDescriptorsTests(unittest.TestCase): "We shouldn't have any duplicate descriptors.") # We should only have the newest descriptor: - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), @@ -541,7 +541,7 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(len(routers), 1, "We shouldn't have any duplicate descriptors.") - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), @@ -564,7 +564,7 @@ class ParseDescriptorsTests(unittest.TestCase): "We shouldn't have any duplicate descriptors.") # We should only have the newest descriptor: - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-12-04 03:10:25", "%Y-%m-%d %H:%M:%S"), @@ -581,10 +581,10 @@ class ParseDescriptorsTests(unittest.TestCase): descFiles = [] # The timestamp and fingerprint from BRIDGE_EXTRA_INFO_DESCRIPTOR: - timestamp = "2014-11-04 06:23:22" - Y, M, rest = timestamp.split("-") - fpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0" - newerFpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0" + timestamp = b"2014-11-04 06:23:22" + Y, M, rest = timestamp.split(b"-") + fpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0" + newerFpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0" total = 0 needed = b * n @@ -592,15 +592,15 @@ class ParseDescriptorsTests(unittest.TestCase): if total >= needed: break # Re-digest the fingerprint to create a "new" bridge - newerFpr = hashlib.sha1(newerFpr).hexdigest().upper() + newerFpr = hashlib.sha1(newerFpr).hexdigest().upper().encode('utf-8') # Generate n extrainfos with different timestamps: count = 0 - for year in range(1, ((n + 1)/ 12) + 2): # Start from the next year + for year in range(1, ((n + 1)// 12) + 2): # Start from the next year if count >= n: break for month in range(1, 13): if count < n: - newerTimestamp = "-".join([str(int(Y) + year), "%02d" % month, rest]) + newerTimestamp = b"-".join([str(int(Y) + year).encode('utf-8'), b"%02d" % month, rest]) newerDuplicate = BRIDGE_EXTRA_INFO_DESCRIPTOR[:].replace( fpr, newerFpr).replace( timestamp, newerTimestamp) @@ -663,10 +663,10 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Give it a bad geoip-db-digest: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", - "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", + b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) @@ -684,7 +684,7 @@ class ParseDescriptorsTests(unittest.TestCase): "and one was unparseable, so that should only leave one " "descriptor remaining.")) - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.fingerprint, "E08B324D20AD0A13E114F027AB9AC3F32CA696A0", @@ -703,14 +703,14 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Mess up the bridge-ip-transports line: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "bridge-ip-transports <OR>=8", - "bridge-ip-transports <OR>") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"bridge-ip-transports <OR>=8", + b"bridge-ip-transports <OR>") parseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) @@ -750,8 +750,8 @@ class ParseDescriptorsTests(unittest.TestCase): zero parsed descriptors. """ unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - '-----END SIGNATURE-----', - '-----END SIGNATURE FOR REALZ-----') + b'-----END SIGNATURE-----', + b'-----END SIGNATURE FOR REALZ-----') # This must be a "real" file or _copyUnparseableDescriptorFile() will # raise an AttributeError saying: # '_io.BytesIO' object has no attribute 'rpartition'" @@ -766,7 +766,7 @@ class ParseDescriptorsTests(unittest.TestCase): missing the signature should return zero parsed descriptors. """ # Remove the signature - BEGIN_SIG = '-----BEGIN SIGNATURE-----' + BEGIN_SIG = b'-----BEGIN SIGNATURE-----' unparseable, _ = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG) # This must be a "real" file or _copyUnparseableDescriptorFile() will # raise an AttributeError saying: @@ -782,7 +782,7 @@ class ParseDescriptorsTests(unittest.TestCase): bad signature should raise an InvalidExtraInfoSignature exception. """ # Truncate the signature to 50 bytes - BEGIN_SIG = '-----BEGIN SIGNATURE-----' + BEGIN_SIG = b'-----BEGIN SIGNATURE-----' doc, sig = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG) unparseable = BEGIN_SIG.join([doc, sig[:50]]) # This must be a "real" file or _copyUnparseableDescriptorFile() will @@ -803,10 +803,10 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Give it a bad geoip-db-digest: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", - "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", + b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) @@ -817,7 +817,7 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseExtraInfoFiles_empty_file(self): """Test parsing an empty extrainfo descriptors file.""" - routers = descriptors.parseExtraInfoFiles(io.BytesIO('')) + routers = descriptors.parseExtraInfoFiles(io.BytesIO(b'')) self.assertIsInstance(routers, dict) self.assertEqual(len(routers), 0) @@ -846,7 +846,7 @@ class ParseDescriptorsTests(unittest.TestCase): True when the new file is successfully created. """ filename = "bridge-descriptors" - with open(filename, 'w') as fh: + with open(filename, 'wb') as fh: fh.write(BRIDGE_SERVER_DESCRIPTOR) fh.flush() @@ -858,7 +858,7 @@ class ParseDescriptorsTests(unittest.TestCase): copy of the bad file with a specific filename format. """ filename = "bridge-descriptors" - with open(filename, 'w') as fh: + with open(filename, 'wb') as fh: fh.write(BRIDGE_SERVER_DESCRIPTOR) fh.flush() diff --git a/bridgedb/test/test_persistent.py b/bridgedb/test/test_persistent.py index 858ad1a..21aa67f 100644 --- a/bridgedb/test/test_persistent.py +++ b/bridgedb/test/test_persistent.py @@ -17,10 +17,10 @@ functioning as expected. from __future__ import print_function +import io import os.path from copy import deepcopy -from io import StringIO from bridgedb import persistent from bridgedb.parse.options import MainOptions @@ -33,9 +33,9 @@ from sure import the from sure import expect -TEST_CONFIG_FILE = StringIO(unicode("""\ +TEST_CONFIG_FILE = io.StringIO("""\ BRIDGE_FILES = ['bridge-descriptors', 'bridge-descriptors.new'] -LOGFILE = 'bridgedb.log'""")) +LOGFILE = 'bridgedb.log'""") class StateTest(unittest.TestCase): @@ -47,10 +47,10 @@ class StateTest(unittest.TestCase): configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') - exec compiled in configuration + exec(compiled, configuration) config = persistent.Conf(**configuration) - fakeArgs = ['-c', os.path.join(os.getcwdu(), '..', 'bridgedb.conf')] + fakeArgs = ['-c', os.path.join(os.getcwd(), '..', 'bridgedb.conf')] options = MainOptions() options.parseOptions(fakeArgs) @@ -142,8 +142,8 @@ class StateTest(unittest.TestCase): setattr(thatConfig, 'BAR', 'all of the things') setattr(thatConfig, 'LOGFILE', 42) - this(thatConfig).should.have.property('FOO').being.a(basestring) - this(thatConfig).should.have.property('BAR').being.a(basestring) + this(thatConfig).should.have.property('FOO').being.a(str) + this(thatConfig).should.have.property('BAR').being.a(str) this(thatConfig).should.have.property('LOGFILE').being.an(int) this(thatConfig).should.have.property('BRIDGE_FILES').being.a(list) @@ -156,8 +156,8 @@ class StateTest(unittest.TestCase): thatState.useChangedSettings(thatConfig) the(thatState.FOO).should.equal('fuuuuu') - the(thatState).should.have.property('FOO').being.a(basestring) - the(thatState).should.have.property('BAR').being.a(basestring) + the(thatState).should.have.property('FOO').being.a(str) + the(thatState).should.have.property('BAR').being.a(str) the(thatState).should.have.property('LOGFILE').being.an(int) the(thatState.FOO).must.equal(thatConfig.FOO) the(thatState.BAR).must.equal(thatConfig.BAR) diff --git a/bridgedb/test/test_persistentSaveAndLoad.py b/bridgedb/test/test_persistentSaveAndLoad.py index 6d6584d..bcdcdd7 100644 --- a/bridgedb/test/test_persistentSaveAndLoad.py +++ b/bridgedb/test/test_persistentSaveAndLoad.py @@ -18,19 +18,19 @@ are all functioning as expected. This module should not import :mod:`sure`. """ +import io import os from copy import deepcopy -from io import StringIO from twisted.trial import unittest from bridgedb import persistent -TEST_CONFIG_FILE = StringIO(unicode("""\ +TEST_CONFIG_FILE = io.StringIO("""\ BRIDGE_FILES = ['bridge-descriptors', 'bridge-descriptors.new'] -LOGFILE = 'bridgedb.log'""")) +LOGFILE = 'bridgedb.log'""") class StateSaveAndLoadTests(unittest.TestCase): @@ -42,7 +42,7 @@ class StateSaveAndLoadTests(unittest.TestCase): configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') - exec compiled in configuration + exec(compiled, configuration) config = persistent.Conf(**configuration) self.config = config @@ -57,10 +57,9 @@ class StateSaveAndLoadTests(unittest.TestCase): self.assertIsInstance(loadedState, persistent.State) self.assertNotIdentical(self.state, loadedState) self.assertNotEqual(self.state, loadedState) - # For some reason, twisted.trial.unittest.TestCase in Python2.6 - # doesn't have an 'assertItemsEqual' attribute... - self.assertEqual(self.state.__dict__.keys().sort(), - loadedState.__dict__.keys().sort()) + + self.assertEqual(list(self.state.__dict__.keys()).sort(), + list(loadedState.__dict__.keys()).sort()) def savedStateAssertions(self, savedStatefile=None): self.assertTrue(os.path.isfile(str(self.state.statefile))) @@ -85,12 +84,12 @@ class StateSaveAndLoadTests(unittest.TestCase): def test_get_statefile(self): statefile = self.state._get_statefile() - self.assertIsInstance(statefile, basestring) + self.assertIsInstance(statefile, str) def test_set_statefile(self): self.state._set_statefile('bar.state') statefile = self.state._get_statefile() - self.assertIsInstance(statefile, basestring) + self.assertIsInstance(statefile, str) def test_set_statefile_new_dir(self): config = self.config @@ -149,7 +148,7 @@ class StateSaveAndLoadTests(unittest.TestCase): self.state.load, 'quux.state') def test_load_with_statefile_opened(self): - fh = open('quux.state', 'w+') + fh = open('quux.state', 'wb+') self.assertRaises(persistent.MissingState, self.state.load, fh) fh.close() diff --git a/bridgedb/test/test_proxy.py b/bridgedb/test/test_proxy.py index 0a69eb0..69b2b99 100644 --- a/bridgedb/test/test_proxy.py +++ b/bridgedb/test/test_proxy.py @@ -254,7 +254,7 @@ class ProxySetUnittests(unittest.TestCase): def test_ProxySet_proxies_getter(self): """ProxySet.proxies should list all proxies.""" - self.assertItemsEqual(self.proxyList.proxies, set(self.proxies)) + self.assertCountEqual(self.proxyList.proxies, set(self.proxies)) def test_ProxySet_proxies_setter(self): """``ProxySet.proxies = ['foo']`` should raise an ``AttributeError``.""" @@ -273,7 +273,7 @@ class ProxySetUnittests(unittest.TestCase): def test_ProxySet_exitRelays_getter(self): """ProxySet.exitRelays should list all exit relays.""" self.proxyList.addExitRelays(self.moarProxies) - self.assertItemsEqual(self.proxyList.exitRelays, set(self.moarProxies)) + self.assertCountEqual(self.proxyList.exitRelays, set(self.moarProxies)) def test_ProxySet_exitRelays_setter(self): """``ProxySet.exitRelays = ['foo']`` should raise an ``AttributeError``.""" @@ -350,7 +350,7 @@ class ProxySetUnittests(unittest.TestCase): whatever tags we want. """ tags = ['foo', 'bar', 'baz'] - extraProxies = zip(self.moarProxies, tags) + extraProxies = list(zip(self.moarProxies, tags)) self.proxyList.addProxies(extraProxies) self.assertEquals(len(self.proxyList), len(self.proxies) + len(extraProxies)) self.assertIn(extraProxies[0][0], self.proxyList) @@ -393,13 +393,13 @@ class ProxySetUnittests(unittest.TestCase): def test_ProxySet_addProxies_bad_type(self): """``ProxySet.addProxies()`` called with something which is neither an - iterable, a basestring, or an int should raise a ValueError. + iterable, a str, or an int should raise a ValueError. """ self.assertRaises(ValueError, self.proxyList.addProxies, object) def test_ProxySet_addProxies_list_of_bad_types(self): """``ProxySet.addProxies()`` called with something which is neither an - iterable, a basestring, or an int should raise a ValueError. + iterable, a str, or an int should raise a ValueError. """ self.assertRaises(ValueError, self.proxyList.addProxies, [object, object, object]) @@ -447,9 +447,9 @@ class ProxySetUnittests(unittest.TestCase): """ proxySetA = self.proxyList proxySetB = proxy.ProxySet(self.moarProxies) - self.assertItemsEqual(proxySetA.difference(proxySetB), + self.assertCountEqual(proxySetA.difference(proxySetB), set(self.proxies)) - self.assertItemsEqual(proxySetB.difference(proxySetA), + self.assertCountEqual(proxySetB.difference(proxySetA), set(self.moarProxies)) def test_ProxySet_firstSeen_returns_timestamp(self): @@ -493,7 +493,7 @@ class ProxySetUnittests(unittest.TestCase): a.extend(self.moarProxies) a = set(a) b = self.proxyList.intersection(set(self.moarProxies)) - self.assertItemsEqual(a, b) + self.assertCountEqual(a, b) def test_ProxySet_remove(self): """ProxySet.remove() should subtract proxies which were already added @@ -534,7 +534,7 @@ class ProxySetUnittests(unittest.TestCase): proxyListA = proxy.ProxySet(self.proxies) proxyListB = proxy.ProxySet(self.proxies) self.assertEqual(proxyListA, proxyListB) - self.assertItemsEqual(proxyListA, proxyListB) + self.assertCountEqual(proxyListA, proxyListB) self.assertEqual(hash(proxyListA), hash(proxyListB)) diff --git a/bridgedb/test/test_schedule.py b/bridgedb/test/test_schedule.py index 1bcec07..c95d6f9 100644 --- a/bridgedb/test/test_schedule.py +++ b/bridgedb/test/test_schedule.py @@ -168,7 +168,7 @@ class ScheduledIntervalTests(unittest.TestCase): ts = sched.getInterval(now) self.assertIsInstance(ts, str) secs = [int(x) for x in ts.replace('-', ' ').replace(':', ' ').split()] - [secs.append(0) for _ in xrange(6-len(secs))] + [secs.append(0) for _ in range(6-len(secs))] secs = schedule.calendar.timegm(secs) self.assertApproximates(now, secs, variance) diff --git a/bridgedb/test/test_smtp.py b/bridgedb/test/test_smtp.py index de443b3..f7261da 100644 --- a/bridgedb/test/test_smtp.py +++ b/bridgedb/test/test_smtp.py @@ -5,7 +5,7 @@ from __future__ import print_function import smtplib import asyncore import threading -import Queue +import queue import random import os @@ -44,7 +44,7 @@ LOCAL_SMTP_SERVER_PORT = 2525 # Must be the same as bridgedb's EMAIL_SMTP_PORT class EmailServer(SMTPServer): - def process_message(self, peer, mailfrom, rcpttos, data): + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ''' Overridden from SMTP server, called whenever a message is received''' self.message_queue.put(data) @@ -58,7 +58,7 @@ class EmailServer(SMTPServer): self.close() def start(self): - self.message_queue = Queue.Queue() + self.message_queue = queue.Queue() self._stop = threading.Event() self._thread = threading.Thread(target=self.thread_proc) # Ensures that if any tests do fail, then threads will exit when the @@ -93,7 +93,7 @@ class EmailServer(SMTPServer): # failures: # # https://travis-ci.org/isislovecruft/bridgedb/jobs/58996136#L3281 - except Queue.Empty: + except queue.Empty: pass else: assert message.find(text) != -1, ("Message did not contain text '%s'." @@ -103,7 +103,7 @@ class EmailServer(SMTPServer): def checkNoMessageReceived(self, timeoutInSecs=2.0): try: self.message_queue.get(block=True, timeout=timeoutInSecs) - except Queue.Empty: + except queue.Empty: return True assert False, "Found a message in the queue, but expected none" @@ -152,7 +152,7 @@ class SMTPTests(unittest.TestCase): # then check that our local SMTP server received a response # and that response contained some bridges - self.server.getAndCheckMessageContains("Here are your bridges") + self.server.getAndCheckMessageContains(b"Here are your bridges") def test_getBridges_rateLimitExceeded(self): if os.environ.get("CI"): @@ -168,14 +168,14 @@ class SMTPTests(unittest.TestCase): # then check that our local SMTP server received a response # and that response contained some bridges - self.server.getAndCheckMessageContains("Here are your bridges") + self.server.getAndCheckMessageContains(b"Here are your bridges") # send another request from the same email address sendMail(FROM_ADDRESS) # this time, the email response should not contain any bridges self.server.getAndCheckMessageContains( - "You have exceeded the rate limit. Please slow down!") + b"You have exceeded the rate limit. Please slow down!") # then we send another request from the same email address sendMail(FROM_ADDRESS) @@ -203,4 +203,4 @@ class SMTPTests(unittest.TestCase): % random.randint(MIN_FROM_ADDRESS, MAX_FROM_ADDRESS)) for i in range(NUM_MAILS): - self.server.getAndCheckMessageContains("Here are your bridges") + self.server.getAndCheckMessageContains(b"Here are your bridges") diff --git a/bridgedb/test/test_translations.py b/bridgedb/test/test_translations.py index 7d14cc0..4b504f1 100644 --- a/bridgedb/test/test_translations.py +++ b/bridgedb/test/test_translations.py @@ -10,7 +10,7 @@ from twisted.trial import unittest -from bridgedb import translations +from bridgedb import _langs, translations from bridgedb.test.https_helpers import DummyRequest @@ -40,15 +40,16 @@ class TranslationsMiscTests(unittest.TestCase): request = DummyRequest([b"bridges"]) request.headers.update(REALISH_HEADERS) request.args.update({ - b'transport': [b'obfs3',], - b'lang': [b'ar',], + 'transport': ['obfs3',], + 'lang': ['ar',], }) parsed = translations.getLocaleFromHTTPRequest(request) + + self.assertEqual(len(parsed), 3) self.assertEqual(parsed[0], 'ar') self.assertEqual(parsed[1], 'en') self.assertEqual(parsed[2], 'en_US') - self.assertEqual(len(parsed), 3) def test_getLocaleFromHTTPRequest_withLangParam_AcceptLanguage(self): """This request uses a '?lang=ar' param, with an 'Accept-Language' @@ -58,14 +59,15 @@ class TranslationsMiscTests(unittest.TestCase): """ request = DummyRequest([b"options"]) request.headers.update(ACCEPT_LANGUAGE_HEADER) - request.args.update({b'lang': [b'fa']}) + request.args.update({'lang': ['fa']}) parsed = translations.getLocaleFromHTTPRequest(request) + + self.assertEqual(len(parsed), 3) self.assertEqual(parsed[0], 'fa') self.assertEqual(parsed[1], 'en') self.assertEqual(parsed[2], 'en_US') #self.assertEqual(parsed[3], 'en-gb') - self.assertEqual(len(parsed), 3) def test_getLocaleFromPlusAddr(self): emailAddr = 'bridges@torproject.org' @@ -80,4 +82,6 @@ class TranslationsMiscTests(unittest.TestCase): def test_usingRTLLang(self): self.assertFalse(translations.usingRTLLang(['foo_BAR'])) self.assertFalse(translations.usingRTLLang(['en'])) - self.assertTrue(translations.usingRTLLang(['fa'])) + + if 'fa' in _langs.get_langs(): + self.assertTrue(translations.usingRTLLang(['fa'])) diff --git a/bridgedb/test/test_txrecaptcha.py b/bridgedb/test/test_txrecaptcha.py index 2550c3d..db37506 100644 --- a/bridgedb/test/test_txrecaptcha.py +++ b/bridgedb/test/test_txrecaptcha.py @@ -142,7 +142,7 @@ class BodyProducerTests(unittest.TestCase): def setUp(self): """Setup the tests.""" - self.content = 'Line 1\r\nLine 2\r\n' + self.content = b'Line 1\r\nLine 2\r\n' self.producer = txrecaptcha._BodyProducer(self.content) def test_interface(self): @@ -220,7 +220,7 @@ class SubmitTests(unittest.TestCase): """ self.assertIsInstance(response, txrecaptcha.RecaptchaResponse) self.assertIsInstance(response.is_valid, bool) - self.assertIsInstance(response.error_code, basestring) + self.assertIsInstance(response.error_code, str) d = txrecaptcha.submit(self.challenge, self.response, self.key, self.ip) @@ -265,12 +265,3 @@ class MiscTests(unittest.TestCase): result = txrecaptcha._ebRequest(fail) self.assertIsInstance(result, txrecaptcha.RecaptchaResponse) self.assertRegexpMatches(result.error_code, msg) - - def test_encodeIfNecessary(self): - """:func:`txrecapcha._encodeIfNecessary` should convert unicode objects - into strings. - """ - origString = unicode('abc') - self.assertIsInstance(origString, unicode) - newString = txrecaptcha._encodeIfNecessary(origString) - self.assertIsInstance(newString, str) diff --git a/bridgedb/test/test_util.py b/bridgedb/test/test_util.py index e858861..55ce35f 100644 --- a/bridgedb/test/test_util.py +++ b/bridgedb/test/test_util.py @@ -169,7 +169,7 @@ class JustifiedLogFormatterTests(unittest.TestCase): formatter = util.JustifiedLogFormatter() formatted = formatter.format(self.record) self.assertIsInstance(formatter, logging.Formatter) - self.assertIsInstance(formatted, basestring) + self.assertIsInstance(formatted, str) self.assertNotEqual(formatted, '') self.assertTrue('INFO' in formatted) self.assertTrue('This is a message' in formatted) diff --git a/bridgedb/test/util.py b/bridgedb/test/util.py index 9fc16b8..43219d5 100644 --- a/bridgedb/test/util.py +++ b/bridgedb/test/util.py @@ -135,7 +135,7 @@ def randomIPv6(): return ipaddr.IPv6Address(random.getrandbits(128)) def randomIP(): - if random.choice(xrange(2)): + if random.choice(range(2)): return randomIPv4() return randomIPv6() @@ -146,7 +146,7 @@ def randomIPv6String(): return bracketIPv6(randomIPv6().compressed) def randomIPString(): - if random.choice(xrange(2)): + if random.choice(range(2)): return randomIPv4String() return randomIPv6String() @@ -194,7 +194,7 @@ def generateFakeBridges(n=500): # Real tor currently only supports one extra ORAddress, and it can # only be IPv6. addrs = [(randomValidIPv6(), randomHighPort(), 6)] - fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) + fpr = "".join(random.choice('abcdef0123456789') for _ in range(40)) supported = ["obfs2", "obfs3", "fte"] transports = [] @@ -282,7 +282,7 @@ class DummyBridge(object): self.address = ipaddr.IPv4Address(ipv4) self.orPort = randomPort() self.fingerprint = "".join(random.choice('abcdef0123456789') - for _ in xrange(40)) + for _ in range(40)) self.orAddresses = [(randomIPv6(), randomPort(), 6)] def getBridgeLine(self, bridgeRequest, includeFingerprint=True): diff --git a/bridgedb/translations.py b/bridgedb/translations.py index 6d7d332..b6a9ef3 100644 --- a/bridgedb/translations.py +++ b/bridgedb/translations.py @@ -58,7 +58,10 @@ def getFirstSupportedLang(langs): if l in supported: lang = l break - return lang + + # crop locales (like 'en-US') to just the language + + return lang.split('-')[0] def getLocaleFromHTTPRequest(request): """Retrieve the languages from an HTTP ``Accept-Language:`` header. @@ -77,7 +80,7 @@ def getLocaleFromHTTPRequest(request): logging.debug("Client sent no 'Accept-Language' header. Using fallback.") header = 'en,en-US' - langs = headers.parseAcceptLanguage(header) + langs = list(headers.parseAcceptLanguage(header)) if not safelog.safe_logging: # pragma: no cover logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) @@ -87,6 +90,10 @@ def getLocaleFromHTTPRequest(request): logging.debug("Client requested language: %r" % chosenLang) langs.insert(0, chosenLang) + # normalize languages to be unicode + + langs = list(map(lambda l: l if isinstance(l, str) else l.decode('utf-8'), langs)) + installTranslations(langs) return langs @@ -124,9 +131,9 @@ def installTranslations(langs): gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, languages=langs, fallback=True)) except IOError as error: - logging.error(error.message) + logging.error(str(error)) - language.install(unicode=True) + language.install() return language def usingRTLLang(langs): @@ -140,11 +147,11 @@ def usingRTLLang(langs): :returns: ``True`` if the preferred language is right-to-left; ``False`` otherwise. """ + lang = getFirstSupportedLang(langs) - rtl = False try: - rtl = babel.core.Locale.parse(lang).text_direction == "rtl" + return babel.core.Locale.parse(lang).text_direction == "rtl" except ValueError as err: logging.warning("Couldn't parse locale %s: %s" % (lang, err)) - return rtl + return False diff --git a/bridgedb/txrecaptcha.py b/bridgedb/txrecaptcha.py index fc4ca2f..7292360 100644 --- a/bridgedb/txrecaptcha.py +++ b/bridgedb/txrecaptcha.py @@ -41,17 +41,17 @@ from twisted.web import client from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer -from zope.interface import implements +from zope.interface import implementer from bridgedb.crypto import SSLVerifyingContextFactory #: This was taken from :data:`recaptcha.client.captcha.API_SSL_SERVER`. -API_SSL_SERVER = API_SERVER = "https://www.google.com/recaptcha/api" -API_SSL_VERIFY_URL = "%s/verify" % API_SSL_SERVER +API_SSL_SERVER = API_SERVER = b"https://www.google.com/recaptcha/api" +API_SSL_VERIFY_URL = b"%s/verify" % API_SSL_SERVER #: (:class:`OpenSSL.crypto.X509`) Only trust certificate for the reCAPTCHA #: :data:`API_SSL_SERVER` which were signed by the Google Internet Authority CA. -GOOGLE_INTERNET_AUTHORITY_CA_CERT = load_certificate(FILETYPE_PEM, bytes("""\ +GOOGLE_INTERNET_AUTHORITY_CA_CERT = load_certificate(FILETYPE_PEM, b"""\ -----BEGIN CERTIFICATE----- MIICsDCCAhmgAwIBAgIDFXfhMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 @@ -68,7 +68,7 @@ Y3Jscy9zZWN1cmVjYS5jcmwwDQYJKoZIhvcNAQEFBQADgYEAvprjecFG+iJsxzEF ZUNgujFQodUovxOWZshcnDW7fZ7mTlk3zpeVJrGPZzhaDhvuJjIfKqHweFB7gwB+ ARlIjNvrPq86fpVg0NOTawALkSqOUMl3MynBQO+spR7EHcRbADQ/JemfTEh2Ycfl vZqhEFBfurZkX0eTANq98ZvVfpg= ------END CERTIFICATE-----""")) +-----END CERTIFICATE-----""") # `t.w.client.HTTPConnectionPool` isn't available in Twisted-12.0.0 # (see ticket #11219: https://bugs.torproject.org/11219): @@ -209,9 +209,9 @@ class RecaptchaResponseProtocol(protocol.Protocol): self.finished.callback(result) +@implementer(IBodyProducer) class _BodyProducer(object): """I write a string into the HTML body of an open request.""" - implements(IBodyProducer) def __init__(self, body): self.body = body @@ -253,12 +253,6 @@ def _ebRequest(fail): error = fail.getErrorMessage() or "possible problem in _ebRequest()" return RecaptchaResponse(is_valid=False, error_code=error) -def _encodeIfNecessary(string): - """Encode unicode objects in utf-8 if necessary.""" - if isinstance(string, unicode): - return string.encode('utf-8') - return string - def submit(recaptcha_challenge_field, recaptcha_response_field, private_key, remoteip, agent=_agent): """Submits a reCaptcha request for verification. This function is a patched @@ -291,14 +285,15 @@ def submit(recaptcha_challenge_field, recaptcha_response_field, d.errback(failure.Failure(ValueError('incorrect-captcha-sol'))) return d - params = urllib.urlencode({ - 'privatekey': _encodeIfNecessary(private_key), - 'remoteip': _encodeIfNecessary(remoteip), - 'challenge': _encodeIfNecessary(recaptcha_challenge_field), - 'response': _encodeIfNecessary(recaptcha_response_field)}) + params = urllib.parse.urlencode({ + 'privatekey': private_key, + 'remoteip': remoteip, + 'challenge': recaptcha_challenge_field, + 'response': recaptcha_response_field, + }).encode('utf-8') body = _BodyProducer(params) headers = Headers({"Content-type": ["application/x-www-form-urlencoded"], "User-agent": ["reCAPTCHA Python"]}) - d = agent.request('POST', API_SSL_VERIFY_URL, headers, body) + d = agent.request(b'POST', API_SSL_VERIFY_URL, headers, body) d.addCallbacks(_cbRequest, _ebRequest) return d diff --git a/bridgedb/util.py b/bridgedb/util.py index 63f508e..9572ad1 100644 --- a/bridgedb/util.py +++ b/bridgedb/util.py @@ -18,6 +18,7 @@ import logging import logging.config import logging.handlers import os +import re import time from twisted.python import components @@ -260,17 +261,15 @@ def replaceControlChars(text, replacement=None, encoding="utf-8"): :rtype: str :returns: The sanitized **text**. """ - escaped = bytearray() - for byte in bytearray(text, encoding): - if byte in range(0, 32) + [92, 127]: - if replacement: - byte = replacement - else: - continue - escaped += bytearray([byte]) + if replacement is None: + replacement = '' + + # the following replaces characters 0-31, 92, and 127 + + text = text.decode(encoding) if isinstance(text, bytes) else text + return re.sub(r'[\x00-\x1f\x5c\x7f]', '', text) - return str(escaped) def registerAdapter(adapter, adapted, interface): """Register a Zope interface adapter for global use. @@ -308,7 +307,6 @@ class JustifiedLogFormatter(logging.Formatter): :param bool logTrace: If ``True``, include information on the calling function in formatted log messages. """ - super(JustifiedLogFormatter, self).__init__(datefmt=datefmt) self.logThreads = logThreads self.logTrace = logTrace @@ -318,7 +316,7 @@ class JustifiedLogFormatter(logging.Formatter): _fmt.append("%(callingFunc)s") _fmt.append("%(message)s") - self._fmt = " ".join(_fmt) + super(JustifiedLogFormatter, self).__init__(fmt = " ".join(_fmt), datefmt=datefmt) def _formatCallingFuncName(self, record): """Format the combined module name and function name of the place where @@ -360,7 +358,7 @@ class JustifiedLogFormatter(logging.Formatter): return super(JustifiedLogFormatter, self).format(record) -class mixin: +class mixin(metaclass=abc.ABCMeta): """Subclasses of me can be used as a mixin class by registering another class, ``ClassA``, which should be mixed with the ``mixin`` subclass, in order to provide simple, less error-prone, multiple inheritance models:: @@ -407,4 +405,3 @@ class mixin: .. info:: This class' name is lowercased because pylint is hardcoded to expect mixin classes to end in ``'mixin'``. """ - __metaclass__ = abc.ABCMeta diff --git a/requirements.txt b/requirements.txt index 5769e11..eed4889 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ Babel==2.8.0 -BeautifulSoup==3.2.2 +beautifulsoup4==4.8.2 +html5lib==1.0b8 Mako==1.1.1 pycryptodome==3.9.6 Twisted==19.10.0 diff --git a/scripts/bridgedb b/scripts/bridgedb index 876fde9..5c92179 100644 --- a/scripts/bridgedb +++ b/scripts/bridgedb @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of BridgeDB, a Tor bridge distribution system. diff --git a/scripts/create_descriptors b/scripts/create_descriptors index ab6ea2e..60022db 100755 --- a/scripts/create_descriptors +++ b/scripts/create_descriptors @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # # This file is part of BridgeDB, a Tor bridge distribution system. @@ -6,7 +6,7 @@ import os import random import sys import time -import ipaddr +import ipaddress import math import argparse import hashlib @@ -117,10 +117,13 @@ def get_hex_string(size): return hexstr -def get_random_ipv6_addr(): +def get_random_addr(ip_version=4): valid_addr = None while not valid_addr: - maybe = ipaddr.IPv6Address(random.getrandbits(128)) + if ip_version == 4: + maybe = ipaddress.IPv4Address(random.getrandbits(32)) + else: + maybe = ipaddress.IPv6Address(random.getrandbits(128)) valid = check_ip_validity(maybe) if valid: valid_addr = maybe @@ -128,13 +131,6 @@ def get_random_ipv6_addr(): return str(valid_addr) -def get_random_ipv4_addr(): - return "%i.%i.%i.%i" % (random.randint(0, 255), - random.randint(0, 255), - random.randint(0, 255), - random.randint(0, 255)) - - def get_protocol(tor_version): line = "" if tor_version is not None: @@ -198,8 +194,8 @@ def create_server_desc(signing_key): timestamp = make_timestamp(variation=True, period=36) server_desc = RelayDescriptor.create({ - "router": "%s %s %s 0 0" % (nickname, get_random_ipv4_addr(), port), - "or-address": "[%s]:%s" % (get_random_ipv6_addr(), port-1), + "router": "%s %s %s 0 0" % (nickname, get_random_addr(ip_version=4), port), + "or-address": "[%s]:%s" % (get_random_addr(ip_version=6), port-1), "platform": "Tor %s on Linux" % tor_version, get_protocol(tor_version): "", "published": timestamp, diff --git a/scripts/get-tor-exits b/scripts/get-tor-exits index 6ab2201..c250dbe 100755 --- a/scripts/get-tor-exits +++ b/scripts/get-tor-exits @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of BridgeDB, a Tor bridge distribution system. @@ -17,8 +17,9 @@ from __future__ import print_function import os.path import socket import sys +import io -from ipaddr import IPAddress +from ipaddress import IPv4Address from OpenSSL import SSL @@ -57,10 +58,10 @@ def getSelfIPAddress(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('bridges.torproject.org', 443)) name = s.getsockname()[0] - ip = IPAddress(name) + ip = IPv4Address(name) if ip.is_link_local or ip.is_private or ip.is_reserved: name = s.getpeername()[0] - ip = IPAddress(name) + ip = IPv4Address(name) except ValueError as error: log.err("get-tor-exits: A socket gave us something that wasn't an IP: %s" % error) @@ -117,7 +118,7 @@ class FileWriter(protocol.Protocol): """Write a portion of the download with ``bytes`` size to disk.""" if self.remaining: display = bytes[:self.remaining] - self.fh.write(display) + self.fh.write(display.decode("utf-8")) self.fh.flush() self.remaining -= len(display) @@ -128,32 +129,11 @@ class FileWriter(protocol.Protocol): self.finished.callback(None) -class WebClientContextFactory(ssl.ClientContextFactory): - """An HTTPS client.""" - - def getContext(self, hostname, port): - """Get this connection's OpenSSL context. - - By default, :api:`twisted.internet.ssl.ClientContextFactory` uses - ``OpenSSL.SSL.SSLv23_METHOD``, which allows SSLv2, SSLv3, and TLSv1, - then they disable SSLv2 in the - :api:`twisted.internet.ssl.ClientContextFactory.getContext` method. - - We disable SSLv3 also. - - :rtype: ``OpenSSL.SSL.Context`` - :returns: An OpenSSL context with options set. - """ - ctx = self._contextFactory(self.method) - ctx.set_options(SSL.OP_NO_SSLv2 ^ SSL.OP_NO_SSLv3) - return ctx - - def main(filename=None, address=None, port=None): fh = filename if filename: - if (not isinstance(filename, file)) and (filename is not sys.stdout): + if (not isinstance(filename, io.TextIOBase)) and (filename is not sys.stdout): fh = open(filename, 'w') if not address: @@ -170,9 +150,8 @@ def main(filename=None, address=None, port=None): log.msg("get-tor-exits: Requesting %s..." % check) - contextFactory = WebClientContextFactory() - agent = client.Agent(reactor, contextFactory) - d = agent.request("GET", check) + agent = client.Agent(reactor) + d = agent.request(b"GET", check.encode("utf-8")) d.addCallback(writeToFile, fh) d.addErrback(handle) d.addCallbacks(log.msg, log.err) @@ -182,7 +161,7 @@ def main(filename=None, address=None, port=None): reactor.run() if filename: - if (not isinstance(filename, file)) and (filename is not sys.stdout): + if (not isinstance(filename, io.TextIOBase)) and (filename is not sys.stdout): fh.flush() fh.close() @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #_____________________________________________________________________________ # # This file is part of BridgeDB, a Tor bridge distribution system. |
