summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPhilipp Winter <phw@nymity.ch>2020-02-19 10:08:03 -0800
committerPhilipp Winter <phw@nymity.ch>2020-02-19 10:08:03 -0800
commitc3a820daeffa1acc02b9c0458ba8495ff387ecd4 (patch)
tree8ea30b1ab83aed095405c16f4a356be6697879bc
parent7b142b899766dea9c503fa3bdf999421ee4b7f37 (diff)
parent1f614896ed3442c8758e7ea10bef0c1fc366a4c1 (diff)
Merge branch 'defect/30946' into develop
-rw-r--r--.travis.requirements.txt3
-rw-r--r--.travis.yml2
-rw-r--r--CHANGELOG4
-rw-r--r--MANIFEST.in2
-rw-r--r--bridgedb/Bridges.py21
-rw-r--r--bridgedb/Stability.py38
-rw-r--r--bridgedb/Storage.py22
-rw-r--r--bridgedb/bridgerequest.py2
-rw-r--r--bridgedb/bridges.py16
-rw-r--r--bridgedb/captcha.py29
-rw-r--r--bridgedb/configure.py2
-rw-r--r--bridgedb/crypto.py45
-rw-r--r--bridgedb/distribute.py4
-rw-r--r--bridgedb/distributors/email/__init__.py6
-rw-r--r--bridgedb/distributors/email/autoresponder.py58
-rw-r--r--bridgedb/distributors/email/dkim.py6
-rw-r--r--bridgedb/distributors/email/server.py39
-rw-r--r--bridgedb/distributors/email/templates.py1
-rw-r--r--bridgedb/distributors/https/distributor.py5
-rw-r--r--bridgedb/distributors/https/server.py111
-rw-r--r--bridgedb/distributors/moat/server.py24
-rw-r--r--bridgedb/filters.py5
-rw-r--r--bridgedb/main.py6
-rw-r--r--bridgedb/metrics.py12
-rw-r--r--bridgedb/parse/addr.py13
-rw-r--r--bridgedb/parse/descriptors.py42
-rw-r--r--bridgedb/parse/headers.py7
-rw-r--r--bridgedb/parse/nickname.py2
-rw-r--r--bridgedb/parse/options.py6
-rw-r--r--bridgedb/persistent.py15
-rw-r--r--bridgedb/proxy.py20
-rw-r--r--bridgedb/qrcodes.py10
-rw-r--r--bridgedb/schedule.py9
-rw-r--r--bridgedb/test/deprecated.py2
-rw-r--r--bridgedb/test/email_helpers.py10
-rw-r--r--bridgedb/test/https_helpers.py6
-rw-r--r--bridgedb/test/legacy_Tests.py36
-rw-r--r--bridgedb/test/moat_helpers.py6
-rw-r--r--bridgedb/test/test_Bridges.py10
-rw-r--r--bridgedb/test/test_Storage.py2
-rw-r--r--bridgedb/test/test_bridgerequest.py4
-rw-r--r--bridgedb/test/test_bridges.py76
-rw-r--r--bridgedb/test/test_captcha.py28
-rw-r--r--bridgedb/test/test_crypto.py17
-rw-r--r--bridgedb/test/test_distributors_moat_request.py2
-rw-r--r--bridgedb/test/test_distributors_moat_server.py8
-rw-r--r--bridgedb/test/test_email_autoresponder.py8
-rw-r--r--bridgedb/test/test_email_distributor.py8
-rw-r--r--bridgedb/test/test_email_dkim.py8
-rw-r--r--bridgedb/test/test_email_server.py39
-rw-r--r--bridgedb/test/test_email_templates.py2
-rw-r--r--bridgedb/test/test_geo.py4
-rw-r--r--bridgedb/test/test_https.py18
-rw-r--r--bridgedb/test/test_https_distributor.py10
-rw-r--r--bridgedb/test/test_https_request.py2
-rw-r--r--bridgedb/test/test_https_server.py102
-rw-r--r--bridgedb/test/test_main.py4
-rw-r--r--bridgedb/test/test_metrics.py4
-rw-r--r--bridgedb/test/test_parse_addr.py14
-rw-r--r--bridgedb/test/test_parse_descriptors.py112
-rw-r--r--bridgedb/test/test_persistent.py18
-rw-r--r--bridgedb/test/test_persistentSaveAndLoad.py21
-rw-r--r--bridgedb/test/test_proxy.py18
-rw-r--r--bridgedb/test/test_schedule.py2
-rw-r--r--bridgedb/test/test_smtp.py18
-rw-r--r--bridgedb/test/test_translations.py18
-rw-r--r--bridgedb/test/test_txrecaptcha.py13
-rw-r--r--bridgedb/test/test_util.py2
-rw-r--r--bridgedb/test/util.py8
-rw-r--r--bridgedb/translations.py21
-rw-r--r--bridgedb/txrecaptcha.py31
-rw-r--r--bridgedb/util.py23
-rw-r--r--requirements.txt3
-rw-r--r--scripts/bridgedb2
-rwxr-xr-xscripts/create_descriptors22
-rwxr-xr-xscripts/get-tor-exits41
-rw-r--r--setup.py2
77 files changed, 693 insertions, 699 deletions
diff --git a/.travis.requirements.txt b/.travis.requirements.txt
index 6f3dec7..1a1d7ff 100644
--- a/.travis.requirements.txt
+++ b/.travis.requirements.txt
@@ -15,7 +15,7 @@
#------------------------------------------------------------------------------
attrs==19.2.0
Babel==2.8.0
-BeautifulSoup==3.2.2
+beautifulsoup4==4.8.2
Mako==1.1.1
pycryptodome==3.9.6
Twisted==19.10.0
@@ -25,6 +25,7 @@ gnupg==2.3.1
ipaddr==2.2.0
mechanize==0.4.5
Pillow==6.2.2
+pyOpenSSL==19.0.0
pygeoip==0.3.2
qrcode==6.1
service_identity==18.1.0
diff --git a/.travis.yml b/.travis.yml
index c0499b0..e57481d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ notifications:
on_failure: change
python:
- - "2.7"
+ - "3.7"
addons:
hosts:
diff --git a/CHANGELOG b/CHANGELOG
index bb1db03..6d4c402 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,7 @@
+ * FIXES https://bugs.torproject.org/30946
+ This patch ports BridgeDB to Python 3. Python 2 is no longer supported
+ since Jan 1, 2020.
+
Changes in version 0.9.3 - 2020-02-18
* FIXES <https://bugs.torproject.org/33299>
diff --git a/MANIFEST.in b/MANIFEST.in
index bbe1e74..bb16c8c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,4 @@
include versioneer.py
include bridgedb/_version.py
include requirements.txt
-recursice-include bridgedb/i18n *.po *.pot
+recursive-include bridgedb/i18n *.po *.pot
diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py
index 3c94fdf..c29a2bf 100644
--- a/bridgedb/Bridges.py
+++ b/bridgedb/Bridges.py
@@ -10,6 +10,7 @@
them into hashrings for distributors.
"""
+import binascii
import bisect
import logging
import re
@@ -28,12 +29,6 @@ from bridgedb.parse.fingerprint import isValidFingerprint
from bridgedb.parse.fingerprint import toHex
from bridgedb.safelog import logSafely
-try:
- from cStringIO import StringIO
-except ImportError:
- from io import StringIO
-
-
ID_LEN = 20 # XXX Only used in commented out line in Storage.py
DIGEST_LEN = 20
PORTSPEC_LEN = 16
@@ -335,7 +330,7 @@ class BridgeRing(object):
else:
logging.debug(
"Got duplicate bridge %r in main hashring for position %r."
- % (logSafely(k.encode('hex')), pos.encode('hex')))
+ % (logSafely(binascii.hexlify(k).decode('utf-8')), binascii.hexlify(pos).decode('utf-8')))
keys.sort()
if filterBySubnet:
@@ -361,7 +356,7 @@ class BridgeRing(object):
def dumpAssignments(self, f, description=""):
logging.info("Dumping bridge assignments for %s..." % self.name)
- for b in self.bridges.itervalues():
+ for b in self.bridges.values():
desc = [ description ]
for tp,val,_,subring in self.subrings:
if subring.getBridgeByID(b.identity):
@@ -380,7 +375,7 @@ class FixedBridgeSplitter(object):
def insert(self, bridge):
# Grab the first 4 bytes
digest = self.hmac(bridge.identity)
- pos = long( digest[:8], 16 )
+ pos = int( digest[:8], 16 )
which = pos % len(self.rings)
self.rings[which].insert(bridge)
@@ -405,7 +400,7 @@ class FixedBridgeSplitter(object):
description is ``"IPv6 obfs2 bridges"`` the line would read:
``"IPv6 obfs2 bridges ring=3"``.
"""
- for index, ring in zip(xrange(len(self.rings)), self.rings):
+ for index, ring in zip(range(len(self.rings)), self.rings):
ring.dumpAssignments(filename, "%s ring=%s" % (description, index))
@@ -544,7 +539,7 @@ class BridgeSplitter(object):
logging.info("Current rings: %s" % " ".join(self.ringsByName))
def dumpAssignments(self, f, description=""):
- for name,ring in self.ringsByName.iteritems():
+ for name,ring in self.ringsByName.items():
ring.dumpAssignments(f, "%s %s" % (description, name))
@@ -633,8 +628,8 @@ class FilteredBridgeSplitter(object):
"""
filterNames = []
- for filterName in [x.func_name for x in list(ringname)]:
- # Using `assignBridgesToSubring.func_name` gives us a messy
+ for filterName in [x.__name__ for x in list(ringname)]:
+ # Using `assignBridgesToSubring.__name__` gives us a messy
# string which includes all parameters and memory addresses. Get
# rid of this by partitioning at the first `(`:
realFilterName = filterName.partition('(')[0]
diff --git a/bridgedb/Stability.py b/bridgedb/Stability.py
index f2f99ce..34ea51d 100644
--- a/bridgedb/Stability.py
+++ b/bridgedb/Stability.py
@@ -33,7 +33,7 @@ from bridgedb.schedule import toUnixSeconds
# tunables
weighting_factor = float(19)/float(20)
-discountIntervalMillis = long(60*60*12*1000)
+discountIntervalMillis = 60*60*12*1000
class BridgeHistory(object):
@@ -76,15 +76,15 @@ class BridgeHistory(object):
self.fingerprint = fingerprint
self.ip = ip
self.port = port
- self.weightedUptime = long(weightedUptime)
- self.weightedTime = long(weightedTime)
- self.weightedRunLength = long(weightedRunLength)
+ self.weightedUptime = int(weightedUptime)
+ self.weightedTime = int(weightedTime)
+ self.weightedRunLength = int(weightedRunLength)
self.totalRunWeights = float(totalRunWeights)
self.lastSeenWithDifferentAddressAndPort = \
- long(lastSeenWithDifferentAddressAndPort)
- self.lastSeenWithThisAddressAndPort = long(lastSeenWithThisAddressAndPort)
- self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues)
- self.lastUpdatedWeightedTime = long(lastUpdatedWeightedTime)
+ int(lastSeenWithDifferentAddressAndPort)
+ self.lastSeenWithThisAddressAndPort = int(lastSeenWithThisAddressAndPort)
+ self.lastDiscountedHistoryValues = int(lastDiscountedHistoryValues)
+ self.lastUpdatedWeightedTime = int(lastUpdatedWeightedTime)
def discountWeightedFractionalUptimeAndWeightedTime(self, discountUntilMillis):
""" discount weighted times """
@@ -111,8 +111,8 @@ class BridgeHistory(object):
@property
def weightedFractionalUptime(self):
"""Weighted Fractional Uptime"""
- if self.weightedTime <0.0001: return long(0)
- return long(10000) * self.weightedUptime / self.weightedTime
+ if self.weightedTime <0.0001: return 0
+ return 10000 * self.weightedUptime / self.weightedTime
@property
def tosa(self):
@@ -127,7 +127,7 @@ class BridgeHistory(object):
more recently than it, or if it has been around for a Weighted Time of 8 days.
"""
# if this bridge has been around longer than 8 days
- if self.weightedTime >= long(8 * 24 * 60 * 60):
+ if self.weightedTime >= 8 * 24 * 60 * 60:
return True
# return True if self.weightedTime is greater than the weightedTime
@@ -146,10 +146,10 @@ class BridgeHistory(object):
"""Weighted Mean Time Between Address Change"""
totalRunLength = self.weightedRunLength + \
((self.lastSeenWithThisAddressAndPort -
- self.lastSeenWithDifferentAddressAndPort) / long(1000))
+ self.lastSeenWithDifferentAddressAndPort) / 1000)
totalWeights = self.totalRunWeights + 1.0
- if totalWeights < 0.0001: return long(0)
+ if totalWeights < 0.0001: return 0
assert(isinstance(long,totalRunLength))
assert(isinstance(long,totalWeights))
return totalRunlength / totalWeights
@@ -159,9 +159,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp):
bhe = db.getBridgeHistory(bridge.fingerprint)
if not bhe:
# This is the first status, assume 60 minutes.
- secondsSinceLastStatusPublication = long(60*60)
- lastSeenWithDifferentAddressAndPort = timestamp * long(1000)
- lastSeenWithThisAddressAndPort = timestamp * long(1000)
+ secondsSinceLastStatusPublication = 60*60
+ lastSeenWithDifferentAddressAndPort = timestamp * 1000
+ lastSeenWithThisAddressAndPort = timestamp * 1000
bhe = BridgeHistory(
bridge.fingerprint, bridge.address, bridge.orPort,
@@ -179,9 +179,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp):
# Calculate the seconds since the last parsed status. If this is
# the first status or we haven't seen a status for more than 60
# minutes, assume 60 minutes.
- statusPublicationMillis = long(timestamp * 1000)
+ statusPublicationMillis = timestamp * 1000
if (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort) > 60*60*1000:
- secondsSinceLastStatusPublication = long(60*60)
+ secondsSinceLastStatusPublication = 60*60
logging.debug("Capping secondsSinceLastStatusPublication to 1 hour")
# otherwise, roll with it
else:
@@ -278,7 +278,7 @@ def updateBridgeHistory(bridges, timestamps):
logging.debug("Beginning bridge stability calculations")
sortedTimestamps = {}
- for fingerprint, stamps in timestamps.items()[:]:
+ for fingerprint, stamps in timestamps.items():
stamps.sort()
bridge = bridges[fingerprint]
for timestamp in stamps:
diff --git a/bridgedb/Storage.py b/bridgedb/Storage.py
index 4182c4b..cfd60bb 100644
--- a/bridgedb/Storage.py
+++ b/bridgedb/Storage.py
@@ -8,7 +8,6 @@ import binascii
import sqlite3
import time
import hashlib
-from contextlib import GeneratorContextManager
from functools import wraps
from ipaddr import IPAddress
import sys
@@ -212,7 +211,7 @@ class Database(object):
cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,))
def getEmailTime(self, addr):
- addr = hashlib.sha1(addr).hexdigest()
+ addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
cur.execute("SELECT when_mailed FROM EmailedBridges WHERE email = ?", (addr,))
v = cur.fetchone()
@@ -221,7 +220,7 @@ class Database(object):
return strToTime(v[0])
def setEmailTime(self, addr, whenMailed):
- addr = hashlib.sha1(addr).hexdigest()
+ addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
t = timeToStr(whenMailed)
cur.execute("INSERT OR REPLACE INTO EmailedBridges "
@@ -262,7 +261,7 @@ class Database(object):
(distributor, hex_key))
def getWarnedEmail(self, addr):
- addr = hashlib.sha1(addr).hexdigest()
+ addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
cur.execute("SELECT * FROM WarnedEmails WHERE email = ?", (addr,))
v = cur.fetchone()
@@ -271,7 +270,7 @@ class Database(object):
return True
def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()):
- addr = hashlib.sha1(addr).hexdigest()
+ addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
t = timeToStr(whenWarned)
cur = self._cur
if warned == True:
@@ -345,11 +344,18 @@ def openDatabase(sqlite_file):
return conn
-class DBGeneratorContextManager(GeneratorContextManager):
+class DBGeneratorContextManager(object):
"""Helper for @contextmanager decorator.
Overload __exit__() so we can call the generator many times
"""
+
+ def __init__(self, gen):
+ self.gen = gen
+
+ def __enter__(self):
+ return next(self.gen)
+
def __exit__(self, type, value, traceback):
"""Handle exiting a with statement block
@@ -362,7 +368,7 @@ class DBGeneratorContextManager(GeneratorContextManager):
"""
if type is None:
try:
- self.gen.next()
+ next(self.gen)
except StopIteration:
return
return
@@ -374,7 +380,7 @@ class DBGeneratorContextManager(GeneratorContextManager):
try:
self.gen.throw(type, value, traceback)
raise RuntimeError("generator didn't stop after throw()")
- except StopIteration, exc:
+ except StopIteration as exc:
# Suppress the exception *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
# raised inside the "with" statement from being suppressed
diff --git a/bridgedb/bridgerequest.py b/bridgedb/bridgerequest.py
index 7bd7b64..821bd7c 100644
--- a/bridgedb/bridgerequest.py
+++ b/bridgedb/bridgerequest.py
@@ -164,7 +164,7 @@ class BridgeRequestBase(object):
# Get an HMAC with the key of the client identifier:
digest = getHMACFunc(key)(client)
# Take the lower 8 bytes of the digest and convert to a long:
- position = long(digest[:8], 16)
+ position = int(digest[:8], 16)
return position
def isValid(self, valid=None):
diff --git a/bridgedb/bridges.py b/bridgedb/bridges.py
index ca1330b..1dc32af 100644
--- a/bridgedb/bridges.py
+++ b/bridgedb/bridges.py
@@ -240,7 +240,7 @@ class BridgeAddressBase(object):
:param str value: The binary-encoded SHA-1 hash digest of the public
half of this Bridge's identity key.
"""
- self.fingerprint = toHex(value)
+ self.fingerprint = toHex(value).decode('utf-8')
@identity.deleter
def identity(self):
@@ -743,7 +743,7 @@ class BridgeBackwardsCompatibility(BridgeBase):
if not fingerprint:
if not len(idDigest) == 20:
raise TypeError("Bridge with invalid ID")
- self.fingerprint = toHex(idDigest)
+ self.fingerprint = toHex(idDigest).decode('utf-8')
elif fingerprint:
if not isValidFingerprint(fingerprint):
raise TypeError("Bridge with invalid fingerprint (%r)"
@@ -1037,7 +1037,7 @@ class Bridge(BridgeBackwardsCompatibility):
if safelog.safe_logging:
prefix = '$$'
if fingerprint:
- fingerprint = hashlib.sha1(fingerprint).hexdigest().upper()
+ fingerprint = hashlib.sha1(fingerprint.encode('utf-8')).hexdigest().upper()
if not fingerprint:
fingerprint = '0' * 40
@@ -1177,7 +1177,7 @@ class Bridge(BridgeBackwardsCompatibility):
# their ``methodname`` matches the requested transport:
transports = filter(lambda pt: pt.methodname == desired, self.transports)
# Filter again for whichever of IPv4 or IPv6 was requested:
- transports = filter(lambda pt: pt.address.version == ipVersion, transports)
+ transports = list(filter(lambda pt: pt.address.version == ipVersion, transports))
if not transports:
raise PluggableTransportUnavailable(
@@ -1377,7 +1377,7 @@ class Bridge(BridgeBackwardsCompatibility):
:meth:`_getBlockKey`.
:param str countryCode: A two-character country code specifier.
"""
- if self._blockedIn.has_key(key):
+ if key in self._blockedIn:
self._blockedIn[key].append(countryCode.lower())
else:
self._blockedIn[key] = [countryCode.lower(),]
@@ -1665,7 +1665,7 @@ class Bridge(BridgeBackwardsCompatibility):
logging.info("Verifying extrainfo signature for %s..." % self)
# Get the bytes of the descriptor signature without the headers:
- document, signature = descriptor.get_bytes().split(TOR_BEGIN_SIGNATURE)
+ document, signature = str(descriptor).split(TOR_BEGIN_SIGNATURE)
signature = signature.replace(TOR_END_SIGNATURE, '')
signature = signature.replace('\n', '')
signature = signature.strip()
@@ -1709,8 +1709,8 @@ class Bridge(BridgeBackwardsCompatibility):
# This is the hexadecimal SHA-1 hash digest of the descriptor document
# as it was signed:
- signedDigest = codecs.encode(unpadded, 'hex_codec')
- actualDigest = hashlib.sha1(document).hexdigest()
+ signedDigest = codecs.encode(unpadded, 'hex_codec').decode('utf-8')
+ actualDigest = hashlib.sha1(document.encode('utf-8')).hexdigest()
except Exception as error:
logging.debug("Error verifying extrainfo signature: %s" % error)
diff --git a/bridgedb/captcha.py b/bridgedb/captcha.py
index b66972c..2bdf6b9 100644
--- a/bridgedb/captcha.py
+++ b/bridgedb/captcha.py
@@ -63,11 +63,10 @@ import logging
import random
import os
import time
-import urllib2
+import urllib.request
-from BeautifulSoup import BeautifulSoup
-
-from zope.interface import Interface, Attribute, implements
+from bs4 import BeautifulSoup
+from zope.interface import Interface, Attribute, implementer
from bridgedb import crypto
from bridgedb import schedule
@@ -101,6 +100,7 @@ class ICaptcha(Interface):
"""Retrieve a new CAPTCHA image."""
+@implementer(ICaptcha)
class Captcha(object):
"""A generic CAPTCHA base class.
@@ -117,7 +117,6 @@ class Captcha(object):
:ivar secretKey: A private key used for decrypting challenge strings during
CAPTCHA solution verification.
"""
- implements(ICaptcha)
def __init__(self, publicKey=None, secretKey=None):
"""Obtain a new CAPTCHA for a client."""
@@ -185,14 +184,14 @@ class ReCaptcha(Captcha):
form = "/noscript?k=%s" % self.publicKey
# Extract and store image from recaptcha
- html = urllib2.urlopen(urlbase + form).read()
+ html = urllib.request.urlopen(urlbase + form).read()
# FIXME: The remaining lines currently cannot be reliably unit tested:
soup = BeautifulSoup(html) # pragma: no cover
imgurl = urlbase + "/" + soup.find('img')['src'] # pragma: no cover
cField = soup.find( # pragma: no cover
'input', {'name': 'recaptcha_challenge_field'}) # pragma: no cover
self.challenge = str(cField['value']) # pragma: no cover
- self.image = urllib2.urlopen(imgurl).read() # pragma: no cover
+ self.image = urllib.request.urlopen(imgurl).read() # pragma: no cover
class GimpCaptcha(Captcha):
@@ -271,6 +270,10 @@ class GimpCaptcha(Captcha):
:returns: ``True`` if the CAPTCHA solution was correct and not
stale. ``False`` otherwise.
"""
+
+ if isinstance(solution, bytes):
+ solution = solution.decode('utf-8')
+
hmacIsValid = False
if not solution:
@@ -290,12 +293,12 @@ class GimpCaptcha(Captcha):
if hmacIsValid:
try:
answerBlob = secretKey.decrypt(encBlob)
- timestamp = answerBlob[:12].lstrip('0')
+ timestamp = answerBlob[:12].lstrip(b'0')
then = cls.sched.nextIntervalStarts(int(timestamp))
now = int(time.time())
- answer = answerBlob[12:]
+ answer = answerBlob[12:].decode('utf-8')
except Exception as error:
- logging.warn(error.message)
+ logging.warn(str(error))
else:
# If the beginning of the 'next' interval (the interval
# after the one when the CAPTCHA timestamp was created)
@@ -367,10 +370,10 @@ class GimpCaptcha(Captcha):
"""
timestamp = str(int(time.time())).zfill(12)
blob = timestamp + answer
- encBlob = self.publicKey.encrypt(blob)
+ encBlob = self.publicKey.encrypt(blob.encode('utf-8'))
hmac = crypto.getHMAC(self.hmacKey, encBlob)
challenge = urlsafe_b64encode(hmac + encBlob)
- return challenge
+ return challenge.decode("utf-8")
def get(self):
"""Get a random CAPTCHA from the cache directory.
@@ -388,7 +391,7 @@ class GimpCaptcha(Captcha):
try:
imageFilename = random.choice(os.listdir(self.cacheDir))
imagePath = os.path.join(self.cacheDir, imageFilename)
- with open(imagePath) as imageFile:
+ with open(imagePath, 'rb') as imageFile:
self.image = imageFile.read()
except IndexError:
raise GimpCaptchaError("CAPTCHA cache dir appears empty: %r"
diff --git a/bridgedb/configure.py b/bridgedb/configure.py
index 0bc4dd1..e069ea3 100644
--- a/bridgedb/configure.py
+++ b/bridgedb/configure.py
@@ -82,7 +82,7 @@ def loadConfig(configFile=None, configCls=None):
if itsSafeToUseLogging:
logging.info("Loading settings from config file: '%s'" % conffile)
compiled = compile(open(conffile).read(), '<string>', 'exec')
- exec compiled in configuration
+ exec(compiled, configuration)
if itsSafeToUseLogging:
logging.debug("New configuration settings:")
diff --git a/bridgedb/crypto.py b/bridgedb/crypto.py
index ac2e3ec..db3d083 100644
--- a/bridgedb/crypto.py
+++ b/bridgedb/crypto.py
@@ -47,7 +47,7 @@ import io
import logging
import os
import re
-import urllib
+import urllib.parse
import OpenSSL
@@ -57,29 +57,9 @@ from Crypto.PublicKey import RSA
from twisted.internet import ssl
from twisted.python.procutils import which
-
#: The hash digest to use for HMACs.
DIGESTMOD = hashlib.sha1
-# Test to see if we have the old or new style buffer() interface. Trying
-# to use an old-style buffer on Python2.7 prior to version 2.7.5 will produce:
-#
-# TypeError: 'buffer' does not have the buffer interface
-#
-#: ``True`` if we have the new-style `buffer`_ interface; ``False`` otherwise.
-#:
-#: .. _buffer: https://docs.python.org/2/c-api/buffer.html
-NEW_BUFFER_INTERFACE = False
-try:
- io.BytesIO(buffer('test'))
-except TypeError: # pragma: no cover
- logging.warn(
- "This Python version is too old! "\
- "It doesn't support new-style buffer interfaces: "\
- "https://mail.python.org/pipermail/python-dev/2010-October/104917.html")
-else:
- NEW_BUFFER_INTERFACE = True
-
class PKCS1PaddingError(Exception):
"""Raised when there is a problem adding or removing PKCS#1 padding."""
@@ -89,7 +69,7 @@ class RSAKeyGenerationError(Exception):
def writeKeyToFile(key, filename):
- """Write **key** to **filename**, with ``0400`` permissions.
+ """Write **key** to **filename**, with ``400`` octal permissions.
If **filename** doesn't exist, it will be created. If it does exist
already, and is writable by the owner of the current process, then it will
@@ -102,7 +82,7 @@ def writeKeyToFile(key, filename):
"""
logging.info("Writing key to file: %r" % filename)
flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0)
- fd = os.open(filename, flags, 0400)
+ fd = os.open(filename, flags, 0o400)
os.write(fd, key)
os.fsync(fd)
os.close(fd)
@@ -210,6 +190,12 @@ def getKey(filename):
def getHMAC(key, value):
"""Return the HMAC of **value** using the **key**."""
+
+ # normalize inputs to be bytes
+
+ key = key.encode('utf-8') if isinstance(key, str) else key
+ value = value.encode('utf-8') if isinstance(value, str) else value
+
h = hmac.new(key, value, digestmod=DIGESTMOD)
return h.digest()
@@ -220,14 +206,19 @@ def getHMACFunc(key, hex=True):
:rtype: callable
:returns: A function which can be uses to generate HMACs.
"""
+
+ key = key.encode('utf-8') if isinstance(key, str) else key
h = hmac.new(key, digestmod=DIGESTMOD)
+
def hmac_fn(value):
+ value = value.encode('utf-8') if isinstance(value, str) else value
h_tmp = h.copy()
h_tmp.update(value)
if hex:
return h_tmp.hexdigest()
else:
return h_tmp.digest()
+
return hmac_fn
def removePKCS1Padding(message):
@@ -319,11 +310,12 @@ def initializeGnuPG(config):
logging.warn("No secret keys found in %s!" % gpg.secring)
return ret
- primarySK = filter(lambda key: key['fingerprint'] == primary, secrets)
- primaryPK = filter(lambda key: key['fingerprint'] == primary, publics)
+ primarySK = list(filter(lambda key: key['fingerprint'] == primary, secrets))
+ primaryPK = list(filter(lambda key: key['fingerprint'] == primary, publics))
if primarySK and primaryPK:
logging.info("Found GnuPG primary key with fingerprint: %s" % primary)
+
for sub in primaryPK[0]['subkeys']:
logging.info(" Subkey: %s Usage: %s" % (sub[0], sub[1].upper()))
else:
@@ -419,7 +411,8 @@ class SSLVerifyingContextFactory(ssl.CertificateOptions):
:rtype: str
:returns: The full hostname (including any subdomains).
"""
- hostname = urllib.splithost(urllib.splittype(url)[1])[0]
+
+ hostname = urllib.parse.urlparse(url).netloc
logging.debug("Parsed hostname %r for cert CN matching." % hostname)
return hostname
diff --git a/bridgedb/distribute.py b/bridgedb/distribute.py
index ec8fb26..42706c9 100644
--- a/bridgedb/distribute.py
+++ b/bridgedb/distribute.py
@@ -108,7 +108,7 @@ import math
from zope import interface
from zope.interface import Attribute
-from zope.interface import implements
+from zope.interface import implementer
# from bridgedb.hashring import IHashring
from bridgedb.interfaces import IName
@@ -155,12 +155,12 @@ class IDistribute(IName):
"""Get bridges based on a client's **bridgeRequest**."""
+@implementer(IDistribute)
class Distributor(Named):
"""A :class:`Distributor` distributes bridges to clients.
Inherit from me to create a new type of ``Distributor``.
"""
- implements(IDistribute)
_bridgesPerResponseMin = 1
_bridgesPerResponseMax = 3
diff --git a/bridgedb/distributors/email/__init__.py b/bridgedb/distributors/email/__init__.py
index 5c0ef50..16f439d 100644
--- a/bridgedb/distributors/email/__init__.py
+++ b/bridgedb/distributors/email/__init__.py
@@ -16,9 +16,3 @@
distributor.
'''
-import autoresponder
-import distributor
-import dkim
-import request
-import server
-import templates
diff --git a/bridgedb/distributors/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py
index e69f78a..3711eae 100644
--- a/bridgedb/distributors/email/autoresponder.py
+++ b/bridgedb/distributors/email/autoresponder.py
@@ -39,6 +39,7 @@ Functionality for autoresponding to incoming emails.
from __future__ import unicode_literals
from __future__ import print_function
+import email
import io
import logging
import time
@@ -51,7 +52,6 @@ from twisted.python import failure
from bridgedb import strings
from bridgedb import metrics
from bridgedb import safelog
-from bridgedb.crypto import NEW_BUFFER_INTERFACE
from bridgedb.distributors.email import dkim
from bridgedb.distributors.email import request
from bridgedb.distributors.email import templates
@@ -183,22 +183,12 @@ class EmailResponse(object):
the email.)
- :vartype _buff: :any:`unicode` or :any:`buffer`
- :var _buff: Used internally to write lines for the response email into the
- ``_mailfile``. The reason why both of these attributes have two
- possible types is for the same Python-buggy reasons which require
- :data:`~bridgedb.crypto.NEW_BUFFER_INTERFACE`.
- :vartype mailfile: :class:`io.StringIO` or :class:`io.BytesIO`
- :var mailfile: An in-memory file-like object for storing the formatted
- headers and body of the response email.
:var str delimiter: Delimiter between lines written to the
:data:`mailfile`.
:var bool closed: ``True`` if :meth:`close` has been called.
:vartype to: :api:`twisted.mail.smtp.Address`
:var to: The client's email address, to which this response should be sent.
"""
- _buff = buffer if NEW_BUFFER_INTERFACE else unicode
- mailfile = io.BytesIO if NEW_BUFFER_INTERFACE else io.StringIO
def __init__(self, gpgSignFunc=None):
"""Create a response to an email we have recieved.
@@ -212,7 +202,7 @@ class EmailResponse(object):
obtaining a pre-configured **gpgSignFunc**.
"""
self.gpgSign = gpgSignFunc
- self.mailfile = self.mailfile()
+ self.mailfile = io.StringIO()
self.delimiter = '\n'
self.closed = False
self.to = None
@@ -274,6 +264,9 @@ class EmailResponse(object):
:param str line: Something to append into the :data:`mailfile`.
"""
+
+ line = line.decode('utf-8') if isinstance(line, bytes) else line
+
if line.find('\r\n') != -1:
# If **line** contains newlines, send it to :meth:`writelines` to
# break it up so that we can replace them:
@@ -281,7 +274,7 @@ class EmailResponse(object):
self.writelines(line)
else:
line += self.delimiter
- self.mailfile.write(self._buff(line.encode('utf8')))
+ self.mailfile.write(line)
self.mailfile.flush()
def writelines(self, lines):
@@ -291,10 +284,11 @@ class EmailResponse(object):
(i.e. ``'\\n'``). See :api:`twisted.mail.smtp.SMTPClient.getMailData`
for the reason.
- :type lines: :any:`basestring` or :any:`list`
+ :type lines: :any:`str` or :any:`list`
:param lines: The lines to write to the :attr:`mailfile`.
"""
- if isinstance(lines, basestring):
+ if isinstance(lines, (str, bytes)):
+ lines = lines.decode('utf-8') if isinstance(lines, bytes) else lines
lines = lines.replace('\r\n', '\n')
for ln in lines.split('\n'):
self.write(ln)
@@ -321,20 +315,24 @@ class EmailResponse(object):
:kwargs: If given, the key will become the name of the header, and the
value will become the contents of that header.
"""
+
+ fromAddress = fromAddress.decode('utf-8') if isinstance(fromAddress, bytes) else fromAddress
+ toAddress = toAddress.decode('utf-8') if isinstance(toAddress, bytes) else toAddress
+
self.write("From: %s" % fromAddress)
self.write("To: %s" % toAddress)
if includeMessageID:
- self.write("Message-ID: %s" % smtp.messageid().encode('utf-8'))
+ self.write("Message-ID: %s" % smtp.messageid())
if inReplyTo:
- self.write("In-Reply-To: %s" % inReplyTo.encode('utf-8'))
- self.write("Content-Type: %s" % contentType.encode('utf-8'))
- self.write("Date: %s" % smtp.rfc822date().encode('utf-8'))
+ self.write("In-Reply-To: %s" % inReplyTo)
+ self.write("Content-Type: %s" % contentType)
+ self.write("Date: %s" % smtp.rfc822date().decode('utf-8'))
if not subject:
subject = '[no subject]'
if not subject.lower().startswith('re'):
subject = "Re: " + subject
- self.write("Subject: %s" % subject.encode('utf-8'))
+ self.write("Subject: %s" % subject)
if kwargs:
for headerName, headerValue in kwargs.items():
@@ -342,7 +340,7 @@ class EmailResponse(object):
headerName = headerName.replace(' ', '-')
headerName = headerName.replace('_', '-')
header = "%s: %s" % (headerName, headerValue)
- self.write(header.encode('utf-8'))
+ self.write(header)
# The first blank line designates that the headers have ended:
self.write(self.delimiter)
@@ -442,8 +440,8 @@ class SMTPAutoresponder(smtp.SMTPClient):
if not body: return # The client was already warned.
- messageID = self.incoming.message.getheader("Message-ID", None)
- subject = self.incoming.message.getheader("Subject", None)
+ messageID = self.incoming.message.get("Message-ID", None)
+ subject = self.incoming.message.get("Subject", None)
response = generateResponse(recipient, client,
body, subject, messageID,
self.incoming.context.gpgSignFunc)
@@ -464,13 +462,13 @@ class SMTPAutoresponder(smtp.SMTPClient):
"""
clients = []
addrHeader = None
- try: fromAddr = self.incoming.message.getaddr("From")[1]
+ try: fromAddr = email.utils.parseaddr(self.incoming.message.get("From"))[1]
except (IndexError, TypeError, AttributeError): pass
else: addrHeader = fromAddr
if not addrHeader:
logging.warn("No From header on incoming mail.")
- try: senderHeader = self.incoming.message.getaddr("Sender")[1]
+ try: senderHeader = email.utils.parseaddr(self.incoming.message.get("Sender"))[1]
except (IndexError, TypeError, AttributeError): pass
else: addrHeader = senderHeader
if not addrHeader:
@@ -512,10 +510,10 @@ class SMTPAutoresponder(smtp.SMTPClient):
try:
ourAddress = smtp.Address(self.incoming.context.fromAddr)
- allRecipients = self.incoming.message.getaddrlist("To")
+ allRecipients = self.incoming.message.get_all("To")
- for _, addr in allRecipients:
- recipient = smtp.Address(addr)
+ for address in allRecipients:
+ recipient = smtp.Address(address)
if not ourAddress.domain in recipient.domain:
logging.debug(("Not our domain (%s) or subdomain, skipping"
" email address: %s")
@@ -528,11 +526,11 @@ class SMTPAutoresponder(smtp.SMTPClient):
" email address: %s") % str(recipient))
continue
# Only check the username before the first '+':
- beforePlus = recipient.local.split('+', 1)[0]
+ beforePlus = recipient.local.split(b'+', 1)[0]
if beforePlus == ourAddress.local:
ours = str(recipient)
if not ours:
- raise addr.BadEmail(allRecipients)
+ raise addr.BadEmail('No email address accepted, please see log', allRecipients)
except Exception as error:
logging.error(("Couldn't find our email address in incoming email "
diff --git a/bridgedb/distributors/email/dkim.py b/bridgedb/distributors/email/dkim.py
index be33d59..5cee31d 100644
--- a/bridgedb/distributors/email/dkim.py
+++ b/bridgedb/distributors/email/dkim.py
@@ -63,10 +63,8 @@ def checkDKIM(message, rules):
if 'dkim' in rules:
# getheader() returns the last of a given kind of header; we want
# to get the first, so we use getheaders() instead.
- dkimHeaders = message.getheaders("X-DKIM-Authentication-Results")
- dkimHeader = "<no header>"
- if dkimHeaders:
- dkimHeader = dkimHeaders[0]
+ dkimHeaders = message.get("X-DKIM-Authentication-Results")
+ dkimHeader = dkimHeaders if dkimHeaders else "<no header>"
if not dkimHeader.startswith("pass"):
logging.info("Rejecting bad DKIM header on incoming email: %r "
% dkimHeader)
diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py
index 8bd4b36..063b640 100644
--- a/bridgedb/distributors/email/server.py
+++ b/bridgedb/distributors/email/server.py
@@ -49,10 +49,10 @@ Servers which interface with clients and distribute bridges over SMTP.
from __future__ import unicode_literals
+import email.message
import logging
import io
import socket
-import rfc822
from twisted.internet import defer
from twisted.internet import reactor
@@ -62,7 +62,7 @@ from twisted.mail import smtp
from twisted.mail.smtp import rfc822date
from twisted.python import failure
-from zope.interface import implements
+from zope.interface import implementer
from bridgedb import __version__
from bridgedb import safelog
@@ -126,7 +126,7 @@ class MailServerContext(object):
self.nBridges = config.EMAIL_N_BRIDGES_PER_ANSWER
self.username = (config.EMAIL_USERNAME or "bridges")
- self.hostname = socket.gethostname()
+ self.hostname = socket.gethostname().encode("utf-8")
self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges@torproject.org")
self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr)
self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25)
@@ -165,6 +165,7 @@ class MailServerContext(object):
return canon
+@implementer(smtp.IMessage)
class SMTPMessage(object):
"""Plugs into the Twisted Mail and receives an incoming message.
@@ -186,7 +187,6 @@ class SMTPMessage(object):
:meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.reply` email
and :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.send` it.
"""
- implements(smtp.IMessage)
def __init__(self, context, canonicalFromSMTP=None):
"""Create a new SMTPMessage.
@@ -222,7 +222,7 @@ class SMTPMessage(object):
if self.nBytes > self.context.maximumSize:
self.ignoring = True
else:
- self.lines.append(line)
+ self.lines.append(line.decode('utf-8') if isinstance(line, bytes) else line)
if not safelog.safe_logging:
try:
ln = line.rstrip("\r\n").encode('utf-8', 'replace')
@@ -251,13 +251,11 @@ class SMTPMessage(object):
:rtype: :api:`twisted.mail.smtp.rfc822.Message`
:returns: A ``Message`` comprised of all lines received thus far.
"""
- rawMessage = io.StringIO()
- for line in self.lines:
- rawMessage.writelines(unicode(line.decode('utf8')) + u'\n')
- rawMessage.seek(0)
- return rfc822.Message(rawMessage)
+ return email.message_from_string('\n'.join(self.lines))
+
+@implementer(smtp.IMessageDelivery)
class SMTPIncomingDelivery(smtp.SMTP):
"""Plugs into :class:`SMTPIncomingServerFactory` and handles SMTP commands
for incoming connections.
@@ -272,7 +270,6 @@ class SMTPIncomingDelivery(smtp.SMTP):
:var fromCanonicalSMTP: If set, this is the canonicalized domain name of
the address we received from incoming connection's ``MAIL FROM:``.
"""
- implements(smtp.IMessageDelivery)
context = None
deferred = defer.Deferred()
@@ -293,11 +290,11 @@ class SMTPIncomingDelivery(smtp.SMTP):
:type recipients: list
:param recipients: A list of :api:`twisted.mail.smtp.User` instances.
"""
- helo_ = ' helo={0}'.format(helo[0]) if helo[0] else ''
- from_ = 'from %s ([%s]%s)' % (helo[0], helo[1], helo_)
- by_ = 'by %s with BridgeDB (%s)' % (smtp.DNSNAME, __version__)
- for_ = 'for %s; %s ' % (' '.join(map(str, recipients)), rfc822date())
- return str('Received: %s\n\t%s\n\t%s' % (from_, by_, for_))
+ helo_ = b' helo=%s' % (helo[0] if helo[0] else '')
+ from_ = b'from %s ([%s]%s)' % (helo[0], helo[1], helo_)
+ by_ = b'by %s with BridgeDB (%s)' % (smtp.DNSNAME, __version__.encode('utf-8'))
+ for_ = b'for %s; %s ' % (b' '.join([str(r).encode('utf-8') for r in recipients]), rfc822date())
+ return 'Received: %s\n\t%s\n\t%s' % (from_.decode('utf-8'), by_.decode('utf-8'), for_.decode('utf-8'))
def validateFrom(self, helo, origin):
"""Validate the ``MAIL FROM:`` address on the incoming SMTP connection.
@@ -376,10 +373,10 @@ class SMTPIncomingDelivery(smtp.SMTP):
ourAddress = smtp.Address(self.context.smtpFromAddr)
if not ((ourAddress.domain in recipient.domain) or
- (recipient.domain == "bridgedb")):
+ (recipient.domain == b"bridgedb")):
logging.debug(("Not our domain (%s) or subdomain, skipping"
" SMTP 'RCPT TO' address: %s")
- % (ourAddress.domain, str(recipient)))
+ % (ourAddress.domain.decode('utf-8'), str(recipient)))
raise smtp.SMTPBadRcpt(str(recipient))
# The recipient's username should at least start with ours,
# but it still might be a '+' address.
@@ -388,13 +385,14 @@ class SMTPIncomingDelivery(smtp.SMTP):
" SMTP 'RCPT TO' address: %s") % str(recipient))
raise smtp.SMTPBadRcpt(str(recipient))
# Ignore everything after the first '+', if there is one.
- beforePlus = recipient.local.split('+', 1)[0]
+ beforePlus = recipient.local.split(b'+', 1)[0]
if beforePlus != ourAddress.local:
raise smtp.SMTPBadRcpt(str(recipient))
return lambda: SMTPMessage(self.context, self.fromCanonicalSMTP)
+@implementer(smtp.IMessageDeliveryFactory)
class SMTPIncomingDeliveryFactory(object):
"""Factory for :class:`SMTPIncomingDelivery` s.
@@ -408,7 +406,6 @@ class SMTPIncomingDeliveryFactory(object):
:var delivery: A :class:`SMTPIncomingDelivery` to deliver incoming
SMTP messages to.
"""
- implements(smtp.IMessageDeliveryFactory)
context = None
delivery = SMTPIncomingDelivery
@@ -499,7 +496,7 @@ def addServer(config, distributor):
reactor.listenTCP(port, factory, interface=addr)
except CannotListenError as error: # pragma: no cover
logging.fatal(error)
- raise SystemExit(error.message)
+ raise SystemExit(str(error))
# Set up a LoopingCall to run every 30 minutes and forget old email times.
lc = LoopingCall(distributor.cleanDatabase)
diff --git a/bridgedb/distributors/email/templates.py b/bridgedb/distributors/email/templates.py
index 85dd105..188f052 100644
--- a/bridgedb/distributors/email/templates.py
+++ b/bridgedb/distributors/email/templates.py
@@ -59,6 +59,7 @@ def addCommands(template):
def addGreeting(template, clientName=None, welcome=False):
greeting = ""
+ clientName = clientName.decode('utf-8') if isinstance(clientName, bytes) else clientName
if not clientName:
greeting = template.gettext(strings.EMAIL_MISC_TEXT[7])
diff --git a/bridgedb/distributors/https/distributor.py b/bridgedb/distributors/https/distributor.py
index 7bce8d9..f3e2e9b 100644
--- a/bridgedb/distributors/https/distributor.py
+++ b/bridgedb/distributors/https/distributor.py
@@ -20,6 +20,7 @@ A Distributor that hands out bridges through a web interface.
:parts: 1
"""
+import binascii
import ipaddr
import logging
@@ -324,9 +325,9 @@ class HTTPSDistributor(Distributor):
logging.debug("Client request within time interval: %s" % interval)
logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings))
- logging.debug("Assigned client to subhashring position: %s" % position.encode('hex'))
+ logging.debug("Assigned client to subhashring position: %s" % binascii.hexlify(position).decode('utf-8'))
logging.debug("Total bridges: %d" % len(self.hashring))
- logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters]))
+ logging.debug("Bridge filters: %s" % ' '.join([x.__name__ for x in filters]))
# Check wheth we have a cached copy of the hashring:
if filters in self.hashring.filterRings.keys():
diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py
index 660d34e..503744e 100644
--- a/bridgedb/distributors/https/server.py
+++ b/bridgedb/distributors/https/server.py
@@ -95,6 +95,25 @@ supported_langs = []
metrix = metrics.HTTPSMetrics()
+def stringifyRequestArgs(args):
+ """Turn the given HTTP request arguments from bytes to str.
+
+ :param dict args: A dictionary of request arguments.
+ :rtype: dict
+ :returns: A dictionary of request arguments.
+ """
+
+ # Convert all key/value pairs from bytes to str.
+ str_args = {}
+ for arg, values in args.items():
+ arg = arg if isinstance(arg, str) else arg.decode("utf-8")
+ values = [value.decode("utf-8") if isinstance(value, bytes)
+ else value for value in values]
+ str_args[arg] = values
+
+ return str_args
+
+
def replaceErrorPage(request, error, template_name=None, html=True):
"""Create a general error page for displaying in place of tracebacks.
@@ -114,9 +133,9 @@ def replaceErrorPage(request, error, template_name=None, html=True):
or if **html** is ``False``, then we return a very simple HTML page
(without CSS, Javascript, images, etc.) which simply says
``"Sorry! Something went wrong with your request."``
- :rtype: str
- :returns: A string containing some content to serve to the client (rather
- than serving a Twisted traceback).
+ :rtype: bytes
+ :returns: A bytes object containing some content to serve to the client
+ (rather than serving a Twisted traceback).
"""
logging.error("Error while attempting to render %s: %s"
% (template_name or 'template',
@@ -135,13 +154,13 @@ def replaceErrorPage(request, error, template_name=None, html=True):
errorMessage = _("Sorry! Something went wrong with your request.")
if not html:
- return bytes(errorMessage)
+ return errorMessage.encode("utf-8")
try:
rendered = resource500.render(request)
except Exception as err:
logging.exception(err)
- rendered = bytes(errorMessage)
+ rendered = errorMessage.encode("utf-8")
return rendered
@@ -336,6 +355,7 @@ class ErrorResource(CSPResource):
self.setCSPHeader(request)
request.setHeader("Content-Type", "text/html; charset=utf-8")
request.setResponseCode(self.code)
+ request.args = stringifyRequestArgs(request.args)
try:
template = lookup.get_template(self.template)
@@ -373,12 +393,13 @@ class TranslatedTemplateResource(CustomErrorHandlingResource, CSPResource):
"""Create a new :api:`Resource <twisted.web.resource.Resource>` for a
Mako-templated webpage.
"""
- gettext.install("bridgedb", unicode=True)
+ gettext.install("bridgedb")
CSPResource.__init__(self)
self.template = template
def render_GET(self, request):
self.setCSPHeader(request)
+ request.args = stringifyRequestArgs(request.args)
rtl = False
try:
langs = translations.getLocaleFromHTTPRequest(request)
@@ -452,11 +473,11 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
:rtype: tuple
:returns: A 2-tuple of ``(image, challenge)``, where ``image`` is a
- binary, JPEG-encoded image, and ``challenge`` is a unique
+ JPEG-encoded image of type bytes, and ``challenge`` is a unique
string. If unable to retrieve a CAPTCHA, returns a tuple
- containing two empty strings.
+ containing (b'', '').
"""
- return ('', '')
+ return (b'', '')
def extractClientSolution(self, request):
"""Extract the client's CAPTCHA solution from a POST request.
@@ -468,7 +489,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
:type request: :api:`twisted.web.http.Request`
:param request: A ``Request`` object for 'bridges.html'.
:returns: A redirect for a request for a new CAPTCHA if there was a
- problem. Otherwise, returns a 2-tuple of strings, the first is the
+ problem. Otherwise, returns a 2-tuple of bytes, the first is the
client's CAPTCHA solution from the text input area, and the second
is the challenge string.
"""
@@ -491,16 +512,17 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
return False
def render_GET(self, request):
- """Retrieve a ReCaptcha from the API server and serve it to the client.
+ """Retrieve a CAPTCHA and serve it to the client.
:type request: :api:`twisted.web.http.Request`
:param request: A ``Request`` object for a page which should be
protected by a CAPTCHA.
- :rtype: str
- :returns: A rendered HTML page containing a ReCaptcha challenge image
+ :rtype: bytes
+ :returns: A rendered HTML page containing a CAPTCHA challenge image
for the client to solve.
"""
self.setCSPHeader(request)
+ request.args = stringifyRequestArgs(request.args)
rtl = False
image, challenge = self.getCaptchaImage(request)
@@ -509,14 +531,14 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
langs = translations.getLocaleFromHTTPRequest(request)
rtl = translations.usingRTLLang(langs)
# TODO: this does not work for versions of IE < 8.0
- imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(image)
+ imgstr = b'data:image/jpeg;base64,%s' % base64.b64encode(image)
template = lookup.get_template('captcha.html')
rendered = template.render(strings,
getSortedLangList(),
rtl=rtl,
lang=langs[0],
langOverride=translations.isLangOverridden(request),
- imgstr=imgstr,
+ imgstr=imgstr.decode("utf-8"),
challenge_field=challenge)
except Exception as err:
rendered = replaceErrorPage(request, err, 'captcha.html')
@@ -543,15 +565,16 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
"""
self.setCSPHeader(request)
request.setHeader("Content-Type", "text/html; charset=utf-8")
+ request.args = stringifyRequestArgs(request.args)
try:
if self.checkSolution(request) is True:
metrix.recordValidHTTPSRequest(request)
return self.resource.render(request)
except ValueError as err:
- logging.debug(err.message)
+ logging.debug(str(err))
except MaliciousRequest as err:
- logging.debug(err.message)
+ logging.debug(str(err))
# Make them wait a bit, then redirect them to a "daring
# work of art" as pennance for their sins.
d = task.deferLater(reactor, 1, lambda: request)
@@ -559,7 +582,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
metrix.recordInvalidHTTPSRequest(request)
return NOT_DONE_YET
except Exception as err:
- logging.debug(err.message)
+ logging.debug(str(err))
metrix.recordInvalidHTTPSRequest(request)
return replaceErrorPage(request, err)
@@ -730,7 +753,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource):
rendered = redirectTo(request.uri, request)
try:
- request.write(rendered)
+ request.write(rendered.encode('utf-8') if isinstance(rendered, str) else rendered)
request.finish()
except Exception as err: # pragma: no cover
logging.exception(err)
@@ -862,6 +885,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource):
HTML to the client.
"""
self.setCSPHeader(request)
+ request.args = stringifyRequestArgs(request.args)
d = self.checkSolution(request)
d.addCallback(self._renderDeferred)
return NOT_DONE_YET
@@ -888,7 +912,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
:param bool includeFingerprints: Do we include the bridge's
fingerprint in the response?
"""
- gettext.install("bridgedb", unicode=True)
+ gettext.install("bridgedb")
CSPResource.__init__(self)
self.distributor = distributor
self.schedule = schedule
@@ -908,10 +932,11 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
:type request: :api:`twisted.web.http.Request`
:param request: A ``Request`` object containing the HTTP method, full
URI, and any URL/POST arguments and headers present.
- :rtype: str
+ :rtype: bytes
:returns: A plaintext or HTML response to serve.
"""
self.setCSPHeader(request)
+ request.args = stringifyRequestArgs(request.args)
try:
response = self.getBridgeRequestAnswer(request)
@@ -919,7 +944,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
logging.exception(err)
response = self.renderAnswer(request)
- return response
+ return response.encode('utf-8') if isinstance(response, str) else response
def getClientIP(self, request):
"""Get the client's IP address from the ``'X-Forwarded-For:'``
@@ -949,6 +974,14 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
logging.info("Replying to web request from %s. Parameters were %r"
% (ip, request.args))
+ # Convert all key/value pairs from bytes to str.
+ str_args = {}
+ for arg, values in request.args.items():
+ arg = arg if isinstance(arg, str) else arg.decode("utf-8")
+ values = [value.decode("utf-8") if isinstance(value, bytes) else value for value in values]
+ str_args[arg] = values
+ request.args = str_args
+
if ip:
bridgeRequest = HTTPSBridgeRequest()
bridgeRequest.client = ip
@@ -1015,7 +1048,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
to use a bridge. If ``None``, then the returned page will instead
explain that there were no bridges of the type they requested,
with instructions on how to proceed.
- :rtype: str
+ :rtype: bytes
:returns: A plaintext or HTML response to serve.
"""
rtl = False
@@ -1024,7 +1057,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
if format == 'plain':
request.setHeader("Content-Type", "text/plain")
try:
- rendered = bytes('\n'.join(bridgeLines))
+ rendered = '\n'.join(bridgeLines).encode('utf-8')
except Exception as err:
rendered = replaceErrorPage(request, err, html=False)
else:
@@ -1033,7 +1066,9 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
qrjpeg = generateQR(bridgeLines)
if qrjpeg:
- qrcode = 'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg)
+ qrcode = b'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg)
+ qrcode = qrcode.decode("utf-8")
+
try:
langs = translations.getLocaleFromHTTPRequest(request)
rtl = translations.usingRTLLang(langs)
@@ -1048,7 +1083,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
except Exception as err:
rendered = replaceErrorPage(request, err)
- return rendered
+ return rendered.encode("utf-8") if isinstance(rendered, str) else rendered
def addWebServer(config, distributor):
@@ -1100,21 +1135,21 @@ def addWebServer(config, distributor):
howto = HowtoResource()
robots = static.File(os.path.join(TEMPLATE_DIR, 'robots.txt'))
assets = static.File(os.path.join(TEMPLATE_DIR, 'assets/'))
- keys = static.Data(bytes(strings.BRIDGEDB_OPENPGP_KEY), 'text/plain')
+ keys = static.Data(strings.BRIDGEDB_OPENPGP_KEY.encode('utf-8'), 'text/plain')
csp = CSPResource(enabled=config.CSP_ENABLED,
includeSelf=config.CSP_INCLUDE_SELF,
reportViolations=config.CSP_REPORT_ONLY,
useForwardedHeader=fwdHeaders)
root = CustomErrorHandlingResource()
- root.putChild('', index)
- root.putChild('robots.txt', robots)
- root.putChild('keys', keys)
- root.putChild('assets', assets)
- root.putChild('options', options)
- root.putChild('howto', howto)
- root.putChild('maintenance', maintenance)
- root.putChild('error', resource500)
+ root.putChild(b'', index)
+ root.putChild(b'robots.txt', robots)
+ root.putChild(b'keys', keys)
+ root.putChild(b'assets', assets)
+ root.putChild(b'options', options)
+ root.putChild(b'howto', howto)
+ root.putChild(b'maintenance', maintenance)
+ root.putChild(b'error', resource500)
root.putChild(CSPResource.reportURI, csp)
if config.RECAPTCHA_ENABLED:
@@ -1135,7 +1170,7 @@ def addWebServer(config, distributor):
if config.HTTPS_ROTATION_PERIOD:
count, period = config.HTTPS_ROTATION_PERIOD.split()
- sched = ScheduledInterval(count, period)
+ sched = ScheduledInterval(int(count), period)
else:
sched = Unscheduled()
@@ -1147,10 +1182,10 @@ def addWebServer(config, distributor):
secretKey=secretKey,
useForwardedHeader=fwdHeaders,
protectedResource=bridges)
- root.putChild('bridges', protected)
+ root.putChild(b'bridges', protected)
logging.info("Protecting resources with %s." % captcha.func.__name__)
else:
- root.putChild('bridges', bridges)
+ root.putChild(b'bridges', bridges)
site = Site(root)
site.displayTracebacks = False
diff --git a/bridgedb/distributors/moat/server.py b/bridgedb/distributors/moat/server.py
index 97dfc42..cdf2022 100644
--- a/bridgedb/distributors/moat/server.py
+++ b/bridgedb/distributors/moat/server.py
@@ -178,11 +178,11 @@ class JsonAPIResource(resource.Resource):
:param request: A ``Request`` for a :api:`twisted.web.resource.Resource`.
:returns: The encoded data.
"""
- request.responseHeaders.addRawHeader(b"Content-Type", b"application/vnd.api+json")
- request.responseHeaders.addRawHeader(b"Server", b"moat/%s" % MOAT_API_VERSION)
+ request.responseHeaders.addRawHeader("Content-Type", "application/vnd.api+json")
+ request.responseHeaders.addRawHeader("Server", "moat/%s" % MOAT_API_VERSION)
if data:
- rendered = json.dumps(data)
+ rendered = json.dumps(data).encode("utf-8")
else:
rendered = b""
@@ -240,7 +240,7 @@ class CustomErrorHandlingResource(resource.Resource):
response = resource501
response.detail = "moat version %s does not implement %s %s" % \
- (MOAT_API_VERSION, request.method, request.uri)
+ (MOAT_API_VERSION, request.method.decode('utf-8'), request.uri.decode('utf-8'))
return response
@@ -368,7 +368,7 @@ class CaptchaFetchResource(CaptchaResource):
logging.error("Unhandled error while retrieving Gimp captcha!")
logging.error(impossible)
- return (capt.image, capt.challenge)
+ return (capt.image, capt.challenge.decode('utf-8') if isinstance(capt.challenge, bytes) else capt.challenge)
def getPreferredTransports(self, supportedTransports):
"""Choose which transport a client should request, based on their list
@@ -476,7 +476,7 @@ class CaptchaFetchResource(CaptchaResource):
}
try:
- data["data"][0]["image"] = base64.b64encode(image)
+ data["data"][0]["image"] = base64.b64encode(image).decode('utf-8')
except Exception as impossible:
logging.error("Could not construct or encode captcha!")
logging.error(impossible)
@@ -602,7 +602,7 @@ class CaptchaCheckResource(CaptchaResource):
logging.warn(("Error processing client POST request: "
"Client JSON API data missing '%s' field.") % err)
except ValueError as err:
- logging.warn("Error processing client POST request: %s" % err.message)
+ logging.warn("Error processing client POST request: %s" % err)
except Exception as impossible:
logging.error(impossible)
@@ -824,13 +824,13 @@ def addMoatServer(config, distributor):
hmacKey, publicKey, secretKey,
fwdHeaders, skipLoopback)
- moat.putChild("fetch", fetch)
- moat.putChild("check", check)
- meek.putChild("moat", moat)
+ moat.putChild(b"fetch", fetch)
+ moat.putChild(b"check", check)
+ meek.putChild(b"moat", moat)
root = CustomErrorHandlingResource()
- root.putChild("meek", meek)
- root.putChild("moat", moat)
+ root.putChild(b"meek", meek)
+ root.putChild(b"moat", moat)
site = Site(root)
site.displayTracebacks = False
diff --git a/bridgedb/filters.py b/bridgedb/filters.py
index a02661e..3ea0723 100644
--- a/bridgedb/filters.py
+++ b/bridgedb/filters.py
@@ -13,6 +13,7 @@
"""Functions for filtering :class:`Bridges <bridgedb.bridges.Bridge>`."""
+import binascii
import logging
from ipaddr import IPv4Address
@@ -43,7 +44,7 @@ def bySubring(hmac, assigned, total):
logging.debug(("Creating a filter for assigning bridges to subhashring "
"%s-of-%s...") % (assigned, total))
- name = "-".join([str(hmac("")[:8]).encode('hex'),
+ name = "-".join([binascii.hexlify(str(hmac("")[:8]).encode('utf-8')).decode('utf-8'),
str(assigned), "of", str(total)])
try:
return _cache[name]
@@ -119,7 +120,7 @@ def byIPv(ipVersion=None):
return False
setattr(_byIPv, "description", "ip=%d" % ipVersion)
_byIPv.__name__ = "byIPv%d()" % ipVersion
- _byIPv.func_doc = _byIPv.func_doc.format(ipVersion)
+ _byIPv.__doc__ = _byIPv.__doc__.format(ipVersion)
_byIPv.name = name
_cache[name] = _byIPv
return _byIPv
diff --git a/bridgedb/main.py b/bridgedb/main.py
index 7c2df6d..bc309c5 100644
--- a/bridgedb/main.py
+++ b/bridgedb/main.py
@@ -198,7 +198,7 @@ def load(state, hashring, clear=False):
inserted = 0
logging.info("Trying to insert %d bridges into hashring, %d of which "
"have the 'Running' flag..." % (len(bridges),
- len(filter(lambda b: b.flags.running, bridges.values()))))
+ len(list(filter(lambda b: b.flags.running, bridges.values())))))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
@@ -423,17 +423,17 @@ def run(options, reactor=reactor):
logging.info("Reloading decoy bridges...")
antibot.loadDecoyBridges(config.DECOY_BRIDGES_FILE)
- logging.info("Reparsing bridge descriptors...")
(hashring,
emailDistributorTmp,
ipDistributorTmp,
moatDistributorTmp) = createBridgeRings(cfg, proxies, key)
- logging.info("Bridges loaded: %d" % len(hashring))
# Initialize our DB.
bridgedb.Storage.initializeDBLock()
bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite")
+ logging.info("Reparsing bridge descriptors...")
load(state, hashring, clear=False)
+ logging.info("Bridges loaded: %d" % len(hashring))
if emailDistributorTmp is not None:
emailDistributorTmp.prepopulateRings() # create default rings
diff --git a/bridgedb/metrics.py b/bridgedb/metrics.py
index 6b2e5cb..bb888d9 100644
--- a/bridgedb/metrics.py
+++ b/bridgedb/metrics.py
@@ -184,16 +184,13 @@ class Singleton(type):
pass
-class Metrics(object):
+class Metrics(metaclass=Singleton):
"""Base class representing metrics.
This class provides functionality that our three distribution mechanisms
share.
"""
- # We're using a meta class to implement a singleton for Metrics.
- __metaclass__ = Singleton
-
def __init__(self, binSize=BIN_SIZE):
logging.debug("Instantiating metrics class.")
self.binSize = binSize
@@ -235,7 +232,7 @@ class Metrics(object):
:returns: A list of metric lines.
"""
lines = []
- for key, value in self.coldMetrics.iteritems():
+ for key, value in self.coldMetrics.items():
# Round up our value to the nearest multiple of self.binSize to
# reduce the accuracy of our real values.
if (value % self.binSize) > 0:
@@ -282,6 +279,9 @@ class Metrics(object):
combinations.
"""
+ if isinstance(countryOrProvider, bytes):
+ countryOrProvider = countryOrProvider.decode('utf-8')
+
countryOrProvider = countryOrProvider.lower()
bridgeType = bridgeType.lower()
success = "success" if success else "fail"
@@ -403,7 +403,7 @@ class EmailMetrics(Metrics):
logging.debug("Recording %svalid email request for %s from %s." %
("" if success else "in", bridgeType, emailAddr))
- sld = emailAddr.domain.split(".")[0]
+ sld = emailAddr.domain.split(b".")[0]
# Now update our metrics.
key = self.createKey(self.keyPrefix, bridgeType, sld, success,
diff --git a/bridgedb/parse/addr.py b/bridgedb/parse/addr.py
index 90bc1d5..1d59c9a 100644
--- a/bridgedb/parse/addr.py
+++ b/bridgedb/parse/addr.py
@@ -231,6 +231,7 @@ def canonicalizeEmailDomain(domain, domainmap):
:returns: The canonical domain name for the email address.
"""
permitted = None
+ domain = domain.decode('utf-8') if isinstance(domain, bytes) else domain
try:
permitted = domainmap.get(domain)
@@ -297,7 +298,7 @@ def extractEmailAddress(emailaddr):
def isIPAddress(ip, compressed=True):
"""Check if an arbitrary string is an IP address, and that it's valid.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to check.
:param boolean compressed: If True, return a string representing the
compressed form of the address. Otherwise, return an
@@ -347,7 +348,7 @@ def isIPv4(ip):
.. attention:: This does *not* check validity. See :func:`isValidIP`.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to check.
:rtype: boolean
:returns: True if the address is an IPv4 address.
@@ -359,7 +360,7 @@ def isIPv6(ip):
.. attention:: This does *not* check validity. See :func:`isValidIP`.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to check.
:rtype: boolean
:returns: True if the address is an IPv6 address.
@@ -407,7 +408,7 @@ def isValidIP(ip):
reasons = []
try:
- if isinstance(ip, basestring):
+ if isinstance(ip, str):
ip = ipaddr.IPAddress(ip)
if ip.is_link_local:
@@ -445,7 +446,7 @@ def isLoopback(ip):
otherwise.
"""
try:
- if isinstance(ip, basestring):
+ if isinstance(ip, str):
ip = ipaddr.IPAddress(ip)
if ip.is_loopback:
@@ -572,7 +573,7 @@ class PortList(object):
for arg in args:
portlist = []
try:
- if isinstance(arg, basestring):
+ if isinstance(arg, str):
ports = set([int(p)
for p in arg.split(',')][:self.PORTSPEC_LEN])
portlist.extend([self._sanitycheck(p) for p in ports])
diff --git a/bridgedb/parse/descriptors.py b/bridgedb/parse/descriptors.py
index 1b01023..f6dee83 100644
--- a/bridgedb/parse/descriptors.py
+++ b/bridgedb/parse/descriptors.py
@@ -122,10 +122,10 @@ def parseNetworkStatusFile(filename, validate=True, skipAnnotations=True,
routers = []
logging.info("Parsing networkstatus file: %s" % filename)
- with open(filename) as fh:
+ with open(filename, 'rb') as fh:
position = fh.tell()
if skipAnnotations:
- while not fh.readline().startswith('r '):
+ while not fh.readline().startswith(b'r '):
position = fh.tell()
logging.debug("Skipping %d bytes of networkstatus file." % position)
fh.seek(position)
@@ -161,42 +161,8 @@ def parseServerDescriptorsFile(filename, validate=True):
logging.info("Parsing server descriptors with Stem: %s" % filename)
descriptorType = 'server-descriptor 1.0'
document = parse_file(filename, descriptorType, validate=validate)
- routers = list()
+ return list(document)
- # Work around https://bugs.torproject.org/26023 by parsing each descriptor
- # at a time and catching any errors not handled in stem:
- while True:
- try:
- routers.append(document.next())
- except StopIteration:
- break
- except Exception as error:
- logging.debug("Error while parsing a bridge server descriptor: %s"
- % error)
-
- return routers
-
-def __cmp_published__(x, y):
- """A custom ``cmp()`` which sorts descriptors by published date.
-
- :rtype: int
- :returns: Return negative if x<y, zero if x==y, positive if x>y.
- """
- if x.published < y.published:
- return -1
- elif x.published == y.published:
- # This *shouldn't* happen. It would mean that two descriptors for
- # the same router had the same timestamps, probably meaning there
- # is a severely-messed up OR implementation out there. Let's log
- # its fingerprint (no matter what!) so that we can look up its
- # ``platform`` line in its server-descriptor and tell whoever
- # wrote that code that they're probably (D)DOSing the Tor network.
- logging.warn(("Duplicate descriptor with identical timestamp (%s) "
- "for bridge %s with fingerprint %s !") %
- (x.published, x.nickname, x.fingerprint))
- return 0
- elif x.published > y.published:
- return 1
def deduplicate(descriptors, statistics=False):
"""Deduplicate some descriptors, returning only the newest for each router.
@@ -227,7 +193,7 @@ def deduplicate(descriptors, statistics=False):
duplicates[fingerprint] = [descriptor,]
for fingerprint, dupes in duplicates.items():
- dupes.sort(cmp=__cmp_published__)
+ dupes.sort(key=lambda x: x.published)
first = dupes.pop()
newest[fingerprint] = first
duplicates[fingerprint] = dupes
diff --git a/bridgedb/parse/headers.py b/bridgedb/parse/headers.py
index 5c3597a..29dd091 100644
--- a/bridgedb/parse/headers.py
+++ b/bridgedb/parse/headers.py
@@ -64,8 +64,8 @@ def parseAcceptLanguage(header):
for only in langsOnly:
if only not in langs:
# Add the fallback after the other languages like it:
- insertAfter = filter(lambda x: x.startswith(only),
- [x for x in langs])
+ insertAfter = list(filter(lambda x: x.startswith(only),
+ [x for x in langs]))
if insertAfter:
placement = langs.index(insertAfter[0]) + 1
langs.insert(placement, only)
@@ -75,5 +75,4 @@ def parseAcceptLanguage(header):
# Gettext wants underderscores, because that is how it creates the
# directories under i18n/, not hyphens:
- langs = map(lambda x: x.replace('-', '_'), [x for x in langs])
- return langs
+ return list(map(lambda x: x.replace('-', '_'), [x for x in langs]))
diff --git a/bridgedb/parse/nickname.py b/bridgedb/parse/nickname.py
index dcc328b..54bcf86 100644
--- a/bridgedb/parse/nickname.py
+++ b/bridgedb/parse/nickname.py
@@ -38,7 +38,7 @@ def isValidRouterNickname(nickname):
:rtype: bool
:returns: ``True`` if the nickname is valid, ``False`` otherwise.
"""
- ALPHANUMERIC = string.letters + string.digits
+ ALPHANUMERIC = string.ascii_letters + string.digits
try:
if not (1 <= len(nickname) <= 19):
diff --git a/bridgedb/parse/options.py b/bridgedb/parse/options.py
index e5c4d1e..c32ada3 100644
--- a/bridgedb/parse/options.py
+++ b/bridgedb/parse/options.py
@@ -117,12 +117,12 @@ def parseOptions():
try:
options.parseOptions()
except usage.UsageError as uerr:
- print(uerr.message)
+ print(str(uerr))
print(options.getUsage())
sys.exit(1)
except Exception as error: # pragma: no cover
exc, value, tb = sys.exc_info()
- print("Unhandled Error: %s" % error.message)
+ print("Unhandled Error: %s" % error)
print(traceback.format_exc(tb))
return options
@@ -206,7 +206,7 @@ class BaseOptions(usage.Options):
if rundir is not None:
gRundir = os.path.abspath(os.path.expanduser(rundir))
else:
- gRundir = os.getcwdu()
+ gRundir = os.getcwd()
setRundir(gRundir)
if not os.path.isdir(gRundir): # pragma: no cover
diff --git a/bridgedb/persistent.py b/bridgedb/persistent.py
index c46c4a5..3d52ec5 100644
--- a/bridgedb/persistent.py
+++ b/bridgedb/persistent.py
@@ -128,6 +128,11 @@ class State(jelly.Jellyable):
:param string statefile: The filename of the statefile.
"""
+
+ if filename is None:
+ self._statefile = None
+ return
+
filename = os.path.abspath(os.path.expanduser(filename))
logging.debug("Setting statefile to '%s'" % filename)
self._statefile = filename
@@ -173,8 +178,8 @@ class State(jelly.Jellyable):
err = ''
try:
- if isinstance(statefile, basestring):
- fh = open(statefile, 'r')
+ if isinstance(statefile, (str, bytes)):
+ fh = open(statefile, 'rb')
elif not statefile.closed:
fh = statefile
except (IOError, OSError) as error: # pragma: no cover
@@ -183,7 +188,7 @@ class State(jelly.Jellyable):
except (AttributeError, TypeError) as error:
err += "Failed statefile.open() and statefile.closed:"
err += "\n\t{0}\nstatefile type = '{1}'".format(
- error.message, type(statefile))
+ error, type(statefile))
else:
try:
status = pickle.load(fh)
@@ -209,7 +214,7 @@ class State(jelly.Jellyable):
fh = None
try:
- fh = open(statefile, 'w')
+ fh = open(statefile, 'wb')
except (IOError, OSError) as error: # pragma: no cover
logging.warn("Error writing state file to '%s': %s"
% (statefile, error))
@@ -218,7 +223,7 @@ class State(jelly.Jellyable):
pickle.dump(jelly.jelly(self), fh)
except AttributeError as error:
logging.debug("Tried jellying an unjelliable object: %s"
- % error.message)
+ % error)
if fh is not None:
fh.flush()
diff --git a/bridgedb/proxy.py b/bridgedb/proxy.py
index 6d93c48..654b40c 100644
--- a/bridgedb/proxy.py
+++ b/bridgedb/proxy.py
@@ -161,7 +161,7 @@ class ProxySet(MutableSet):
only added if it passes the checks in
:func:`~bridgedb.parse.addr.isIPAddress`.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to add.
:param tag: An optional value to link to **ip**. If not given, it will
be a timestamp (seconds since epoch, as a float) for when **ip**
@@ -183,7 +183,7 @@ class ProxySet(MutableSet):
def __contains__(self, ip):
"""x.__contains__(y) <==> y in x.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to check.
:rtype: boolean
:returns: True if ``ip`` is in this set; False otherwise.
@@ -196,7 +196,7 @@ class ProxySet(MutableSet):
def __sub__(self, ip):
"""Entirely remove **ip** from this set.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to remove.
"""
try:
@@ -244,7 +244,7 @@ class ProxySet(MutableSet):
if len(x) == 2: self.add(x[0], x[1])
elif len(x) == 1: self.add(x, tag)
else: raise ValueError(self._getErrorMessage(x, proxies))
- elif isinstance(x, (basestring, int)):
+ elif isinstance(x, (str, int)):
self.add(x, tag)
else:
raise ValueError(self._getErrorMessage(x, proxies))
@@ -270,9 +270,9 @@ class ProxySet(MutableSet):
def getTag(self, ip):
"""Get the tag for an **ip** in this ``ProxySet``, if available.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to obtain the tag for.
- :rtype: ``None`` or basestring or int
+ :rtype: ``None`` or str or int
:returns: The tag for that **ip**, iff **ip** exists in this
``ProxySet`` and it has a tag.
"""
@@ -281,7 +281,7 @@ class ProxySet(MutableSet):
def getAllWithTag(self, tag):
"""Get all proxies in this ``ProxySet`` with a given tag.
- :param basestring tag: A tag to search for.
+ :param str tag: A tag to search for.
:rtype: set
:returns: A set of all proxies which are contained within this
:class:`~bridgedb.proxy.ProxySet` which are also tagged with
@@ -293,7 +293,7 @@ class ProxySet(MutableSet):
def firstSeen(self, ip):
"""Get the timestamp when **ip** was first seen, if available.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to obtain a timestamp for.
:rtype: float or None
:returns: The timestamp (in seconds since epoch) if available.
@@ -306,7 +306,7 @@ class ProxySet(MutableSet):
def isExitRelay(self, ip):
"""Check if ``ip`` is a known Tor exit relay.
- :type ip: basestring or int
+ :type ip: str or int
:param ip: The IP address to check.
:rtype: boolean
:returns: True if ``ip`` is a known Tor exit relay; False otherwise.
@@ -425,7 +425,7 @@ class ExitListProtocol(protocol.ProcessProtocol):
def outReceived(self, data):
"""Some data was received from stdout."""
- self.data.append(data)
+ self.data.append(data.decode("utf-8"))
def outConnectionLost(self):
"""This will be called when stdout is closed."""
diff --git a/bridgedb/qrcodes.py b/bridgedb/qrcodes.py
index b3c4546..3cb3a89 100644
--- a/bridgedb/qrcodes.py
+++ b/bridgedb/qrcodes.py
@@ -13,7 +13,7 @@
"""Utilities for working with QRCodes."""
-import cStringIO
+import io
import logging
try:
@@ -26,15 +26,15 @@ except ImportError: # pragma: no cover
"python-qrcode package."))
-def generateQR(bridgelines, imageFormat=u'JPEG', bridgeSchema=False):
+def generateQR(bridgelines, imageFormat='JPEG', bridgeSchema=False):
"""Generate a QRCode for the client's bridge lines.
:param str bridgelines: The Bridge Lines which we are distributing to the
client.
:param bool bridgeSchema: If ``True``, prepend ``'bridge://'`` to the
beginning of each bridge line before QR encoding.
- :rtype: str or ``None``
- :returns: The generated QRCode, as a string.
+ :rtype: bytes or ``None``
+ :returns: The generated QRCode, as a bytes.
"""
logging.debug("Attempting to encode bridge lines into a QRCode...")
@@ -60,7 +60,7 @@ def generateQR(bridgelines, imageFormat=u'JPEG', bridgeSchema=False):
qr = qrcode.QRCode()
qr.add_data(bridgelines)
- buf = cStringIO.StringIO()
+ buf = io.BytesIO()
img = qr.make_image().resize([350, 350])
img.save(buf, imageFormat)
buf.seek(0)
diff --git a/bridgedb/schedule.py b/bridgedb/schedule.py
index e962c39..a2ebf6a 100644
--- a/bridgedb/schedule.py
+++ b/bridgedb/schedule.py
@@ -49,7 +49,7 @@ def fromUnixSeconds(timestamp):
:param int timestamp: A timestamp in Unix Era seconds.
:rtype: :any:`datetime.datetime`
"""
- return datetime.fromtimestamp(timestamp)
+ return datetime.utcfromtimestamp(timestamp)
class ISchedule(interface.Interface):
@@ -200,13 +200,14 @@ class ScheduledInterval(Unscheduled):
:raises UnknownInterval: if the specified **count** was invalid.
"""
try:
- if not count > 0:
+ if count is None:
count = 1
count = int(count)
except (TypeError, ValueError):
- raise UnknownInterval("%s.intervalCount: %r ist not an integer."
+ raise UnknownInterval("%s.intervalCount: %r is not an integer."
% (self.__class__.__name__, count))
- self.intervalCount = count
+
+ self.intervalCount = max(1, count)
def _setIntervalPeriod(self, period=None):
"""Set our :attr:`intervalPeriod`.
diff --git a/bridgedb/test/deprecated.py b/bridgedb/test/deprecated.py
index ab7c502..1a71bf1 100644
--- a/bridgedb/test/deprecated.py
+++ b/bridgedb/test/deprecated.py
@@ -126,7 +126,7 @@ class Bridge(object):
if not request: request = 'default'
digest = getHMACFunc('Order-Or-Addresses')(request)
- pos = long(digest[:8], 16) # lower 8 bytes -> long
+ pos = int(digest[:8], 16) # lower 8 bytes -> int
# default address type
if not addressClass: addressClass = ipaddr.IPv4Address
diff --git a/bridgedb/test/email_helpers.py b/bridgedb/test/email_helpers.py
index 645fc93..edc8196 100644
--- a/bridgedb/test/email_helpers.py
+++ b/bridgedb/test/email_helpers.py
@@ -52,7 +52,7 @@ EMAIL_FROM_ADDR = "bridges@localhost"
EMAIL_BIND_IP = "127.0.0.1"
EMAIL_PORT = 5225
-TEST_CONFIG_FILE = io.StringIO(unicode("""\
+TEST_CONFIG_FILE = io.StringIO("""\
EMAIL_DIST = %s
EMAIL_ROTATION_PERIOD = %s
EMAIL_INCLUDE_FINGERPRINTS = %s
@@ -96,14 +96,14 @@ EMAIL_PORT = %s
repr(EMAIL_N_BRIDGES_PER_ANSWER),
repr(EMAIL_FROM_ADDR),
repr(EMAIL_BIND_IP),
- repr(EMAIL_PORT))))
+ repr(EMAIL_PORT)))
def _createConfig(configFile=TEST_CONFIG_FILE):
configuration = {}
TEST_CONFIG_FILE.seek(0)
compiled = compile(configFile.read(), '<string>', 'exec')
- exec compiled in configuration
+ exec(compiled, configuration)
config = Conf(**configuration)
return config
@@ -138,7 +138,7 @@ class DummyEmailDistributor(object):
self.answerParameters = answerParameters
def getBridges(self, bridgeRequest, epoch):
- return [util.DummyBridge() for _ in xrange(self._bridgesPerResponseMin)]
+ return [util.DummyBridge() for _ in range(self._bridgesPerResponseMin)]
def cleanDatabase(self):
pass
@@ -167,7 +167,7 @@ class DummyEmailDistributorWithState(DummyEmailDistributor):
self.alreadySeen[bridgeRequest.client] += 1
if self.alreadySeen[bridgeRequest.client] <= 1:
- return [util.DummyBridge() for _ in xrange(self._bridgesPerResponseMin)]
+ return [util.DummyBridge() for _ in range(self._bridgesPerResponseMin)]
elif self.alreadySeen[bridgeRequest.client] == 2:
raise TooSoonEmail(
"Seen client '%s' %d times"
diff --git a/bridgedb/test/https_helpers.py b/bridgedb/test/https_helpers.py
index fad841e..ca268c9 100644
--- a/bridgedb/test/https_helpers.py
+++ b/bridgedb/test/https_helpers.py
@@ -50,7 +50,7 @@ CSP_ENABLED = True
CSP_REPORT_ONLY = True
CSP_INCLUDE_SELF = True
-TEST_CONFIG_FILE = io.StringIO(unicode("""\
+TEST_CONFIG_FILE = io.StringIO("""\
SERVER_PUBLIC_FQDN = %r
SERVER_PUBLIC_EXTERNAL_IP = %r
HTTPS_DIST = %r
@@ -100,14 +100,14 @@ CSP_INCLUDE_SELF = %r
GIMP_CAPTCHA_RSA_KEYFILE,
CSP_ENABLED,
CSP_REPORT_ONLY,
- CSP_INCLUDE_SELF)))
+ CSP_INCLUDE_SELF))
def _createConfig(configFile=TEST_CONFIG_FILE):
configuration = {}
TEST_CONFIG_FILE.seek(0)
compiled = compile(configFile.read(), '<string>', 'exec')
- exec compiled in configuration
+ exec(compiled, configuration)
config = Conf(**configuration)
return config
diff --git a/bridgedb/test/legacy_Tests.py b/bridgedb/test/legacy_Tests.py
index 00e1d12..93b52a7 100644
--- a/bridgedb/test/legacy_Tests.py
+++ b/bridgedb/test/legacy_Tests.py
@@ -61,17 +61,17 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False,
transports=False):
ip = randomIPv4()
nn = "bridge-%s" % int(ip)
- fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)])
+ fp = "".join([random.choice("0123456789ABCDEF") for _ in range(40)])
b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp)
b.setStatus(running, stable)
oraddrs = []
if or_addresses:
- for i in xrange(8):
+ for i in range(8):
b.orAddresses.append((randomValidIPv6(), randomPort(), 6))
if transports:
- for i in xrange(0,8):
+ for i in range(0,8):
b.transports.append(bridgedb.Bridges.PluggableTransport(b,
random.choice(["obfs", "obfs2", "pt1"]),
randomIP(), randomPort()))
@@ -81,17 +81,17 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False,
transports=False):
ip = randomIPv6()
nn = "bridge-%s" % int(ip)
- fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)])
+ fp = "".join([random.choice("0123456789ABCDEF") for _ in range(40)])
b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp)
b.setStatus(running, stable)
oraddrs = []
if or_addresses:
- for i in xrange(8):
+ for i in range(8):
b.orAddresses.append((randomValidIPv6(), randomPort(), 6))
if transports:
- for i in xrange(0,8):
+ for i in range(0,8):
b.transports.append(bridgedb.Bridges.PluggableTransport(b,
random.choice(["obfs", "obfs2", "pt1"]),
randomIP(), randomPort()))
@@ -226,10 +226,10 @@ class BridgeStabilityTests(unittest.TestCase):
db = self.db
b = random.choice([fakeBridge,fakeBridge6])()
def timestampSeries(x):
- for i in xrange(61):
+ for i in range(61):
yield (i+1)*60*30 + x # 30 minute intervals
now = time.time()
- time_on_address = long(60*30*60) # 30 hours
+ time_on_address = 60*30*60 # 30 hours
downtime = 60*60*random.randint(0,4) # random hours of downtime
for t in timestampSeries(now):
@@ -244,22 +244,22 @@ class BridgeStabilityTests(unittest.TestCase):
def testLastSeenWithDifferentAddressAndPort(self):
db = self.db
- for i in xrange(10):
+ for i in range(10):
num_desc = 30
time_start = time.time()
- ts = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ]
+ ts = [ 60*30*(i+1) + time_start for i in range(num_desc) ]
b = random.choice([fakeBridge(), fakeBridge6()])
[ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ]
# change the port
b.orport = b.orport+1
last_seen = ts[-1]
- ts = [ 60*30*(i+1) + last_seen for i in xrange(num_desc) ]
+ ts = [ 60*30*(i+1) + last_seen for i in range(num_desc) ]
[ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ]
b = db.getBridgeHistory(b.fingerprint)
assert b.tosa == ts[-1] - last_seen
- assert (long(last_seen*1000) == b.lastSeenWithDifferentAddressAndPort)
- assert (long(ts[-1]*1000) == b.lastSeenWithThisAddressAndPort)
+ assert (last_seen*1000 == b.lastSeenWithDifferentAddressAndPort)
+ assert (ts[-1]*1000 == b.lastSeenWithThisAddressAndPort)
def testFamiliar(self):
# create some bridges
@@ -267,11 +267,11 @@ class BridgeStabilityTests(unittest.TestCase):
num_bridges = 10
num_desc = 4*48 # 30m intervals, 48 per day
time_start = time.time()
- bridges = [ fakeBridge() for x in xrange(num_bridges) ]
+ bridges = [ fakeBridge() for x in range(num_bridges) ]
t = time.time()
- ts = [ (i+1)*60*30+t for i in xrange(num_bridges) ]
+ ts = [ (i+1)*60*30+t for i in range(num_bridges) ]
for b in bridges:
- time_series = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ]
+ time_series = [ 60*30*(i+1) + time_start for i in range(num_desc) ]
[ bridgedb.Stability.addOrUpdateBridgeHistory(b, i) for i in time_series ]
assert None not in bridges
# +1 to avoid rounding errors
@@ -288,7 +288,7 @@ class BridgeStabilityTests(unittest.TestCase):
num_bridges = 20
time_start = time.time()
bridges = [random.choice([fakeBridge, fakeBridge6])()
- for i in xrange(num_bridges)]
+ for i in range(num_bridges)]
# run some of the bridges for the full time series
running = bridges[:num_bridges/2]
@@ -303,7 +303,7 @@ class BridgeStabilityTests(unittest.TestCase):
# figure out how many intervals it will take for weightedUptime to
# decay to < 1
num_desc = int(30*log(1/float(num_successful*30*60))/(-0.05))
- timeseries = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ]
+ timeseries = [ 60*30*(i+1) + time_start for i in range(num_desc) ]
for i in timeseries:
for b in running:
diff --git a/bridgedb/test/moat_helpers.py b/bridgedb/test/moat_helpers.py
index 07b44a0..236b529 100644
--- a/bridgedb/test/moat_helpers.py
+++ b/bridgedb/test/moat_helpers.py
@@ -48,7 +48,7 @@ MOAT_ROTATION_PERIOD = "3 hours"
MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = 'moat_captcha_hmac_key'
MOAT_GIMP_CAPTCHA_RSA_KEYFILE = 'moat_captcha_rsa_key'
-TEST_CONFIG_FILE = io.StringIO(unicode("""\
+TEST_CONFIG_FILE = io.StringIO("""\
GIMP_CAPTCHA_DIR = %r
SERVER_PUBLIC_FQDN = %r
SUPPORTED_TRANSPORTS = %r
@@ -88,13 +88,13 @@ MOAT_GIMP_CAPTCHA_RSA_KEYFILE = %r
MOAT_N_IP_CLUSTERS,
MOAT_ROTATION_PERIOD,
MOAT_GIMP_CAPTCHA_HMAC_KEYFILE,
- MOAT_GIMP_CAPTCHA_RSA_KEYFILE)))
+ MOAT_GIMP_CAPTCHA_RSA_KEYFILE))
def _createConfig(configFile=TEST_CONFIG_FILE):
configuration = {}
TEST_CONFIG_FILE.seek(0)
compiled = compile(configFile.read(), '<string>', 'exec')
- exec compiled in configuration
+ exec(compiled, configuration)
config = Conf(**configuration)
return config
diff --git a/bridgedb/test/test_Bridges.py b/bridgedb/test/test_Bridges.py
index 5991952..77eff1c 100644
--- a/bridgedb/test/test_Bridges.py
+++ b/bridgedb/test/test_Bridges.py
@@ -62,7 +62,7 @@ class BridgeRingTests(unittest.TestCase):
"""
self.addBridgesFromSameSubnet()
- chosen = self.ring.bridges.keys()[:10]
+ chosen = list(self.ring.bridges.keys())[:10]
bridges = self.ring.filterDistinctSubnets(chosen)
# Since they're all in the same /16, we should only get one
@@ -75,7 +75,7 @@ class BridgeRingTests(unittest.TestCase):
"""
self.addRandomBridges()
- chosen = self.ring.bridges.keys()[:3]
+ chosen = list(self.ring.bridges.keys())[:3]
bridges = self.ring.filterDistinctSubnets(chosen)
self.assertGreaterEqual(len(bridges), 1)
@@ -92,7 +92,7 @@ class BridgeRingTests(unittest.TestCase):
filtering by distinct subnets.
"""
self.addRandomBridges()
- bridges = self.ring.getBridges('a' * Bridges.DIGEST_LEN, N=3, filterBySubnet=True)
+ bridges = self.ring.getBridges(b'a' * Bridges.DIGEST_LEN, N=3, filterBySubnet=True)
self.assertEqual(len(bridges), 3)
def test_dumpAssignments(self):
@@ -107,7 +107,7 @@ class BridgeRingTests(unittest.TestCase):
f.seek(0)
data = f.read()
- first = self.ring.bridges.values()[0].fingerprint
+ first = list(self.ring.bridges.values())[0].fingerprint
# The first bridge's fingerprint should be within the data somewhere
self.assertIn(first, data)
@@ -149,7 +149,7 @@ class FixedBridgeSplitterTests(unittest.TestCase):
f.seek(0)
data = f.read()
- first = self.splitter.rings[0].bridges.values()[0].fingerprint
+ first = list(self.splitter.rings[0].bridges.values())[0].fingerprint
# The first bridge's fingerprint should be within the data somewhere
self.assertIn(first, data)
diff --git a/bridgedb/test/test_Storage.py b/bridgedb/test/test_Storage.py
index dffba2b..ca10bb9 100644
--- a/bridgedb/test/test_Storage.py
+++ b/bridgedb/test/test_Storage.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
"""Unittests for the :mod:`bridgedb.Storage` module."""
import os
diff --git a/bridgedb/test/test_bridgerequest.py b/bridgedb/test/test_bridgerequest.py
index 681d54a..80b5534 100644
--- a/bridgedb/test/test_bridgerequest.py
+++ b/bridgedb/test/test_bridgerequest.py
@@ -48,11 +48,11 @@ class BridgeRequestBaseTests(unittest.TestCase):
should use the default client identifier string.
"""
self.assertEqual(self.request.getHashringPlacement('AAAA'),
- 3486762050L)
+ 3486762050)
def test_BridgeRequestBase_getHashringPlacement_with_client(self):
"""BridgeRequestBase.getHashringPlacement() with a client parameter
should use the client identifier string.
"""
self.assertEqual(self.request.getHashringPlacement('AAAA', client='you'),
- 2870307088L)
+ 2870307088)
diff --git a/bridgedb/test/test_bridges.py b/bridgedb/test/test_bridges.py
index 709ae81..fea2d33 100644
--- a/bridgedb/test/test_bridges.py
+++ b/bridgedb/test/test_bridges.py
@@ -466,7 +466,7 @@ class BridgeAddressBaseTests(unittest.TestCase):
cc = self.bab.country
self.assertIsNotNone(cc)
- self.assertIsInstance(cc, basestring)
+ self.assertIsInstance(cc, str)
self.assertEqual(len(cc), 2)
@@ -506,7 +506,7 @@ class PluggableTransportTests(unittest.TestCase):
args = pt._parseArgumentsIntoDict(["sharedsecret=foobar",
"publickey=1234"])
self.assertIsInstance(args, dict)
- self.assertItemsEqual(args, {"sharedsecret": "foobar",
+ self.assertCountEqual(args, {"sharedsecret": "foobar",
"publickey": "1234"})
def test_PluggableTransport_parseArgumentsIntoDict_valid_list_multi(self):
@@ -517,7 +517,7 @@ class PluggableTransportTests(unittest.TestCase):
args = pt._parseArgumentsIntoDict(["sharedsecret=foobar,password=baz",
"publickey=1234"])
self.assertIsInstance(args, dict)
- self.assertItemsEqual(args, {"sharedsecret": "foobar",
+ self.assertCountEqual(args, {"sharedsecret": "foobar",
"password": "baz",
"publickey": "1234"})
@@ -528,7 +528,7 @@ class PluggableTransportTests(unittest.TestCase):
pt = bridges.PluggableTransport()
args = pt._parseArgumentsIntoDict(
["sharedsecret=foobar,password,publickey=1234"])
- self.assertItemsEqual(args, {"sharedsecret": "foobar",
+ self.assertCountEqual(args, {"sharedsecret": "foobar",
"publickey": "1234"})
def test_PluggableTransport_checkArguments_scramblesuit_missing_password(self):
@@ -848,10 +848,10 @@ class BridgeTests(unittest.TestCase):
self._networkstatusFile)[0]
self.serverdescriptor = descriptors.parseServerDescriptorsFile(
self._serverDescriptorFile)[0]
- self.extrainfo = descriptors.parseExtraInfoFiles(
- self._extrainfoFile).values()[0]
- self.extrainfoNew = descriptors.parseExtraInfoFiles(
- self._extrainfoNewFile).values()[0]
+ self.extrainfo = list(descriptors.parseExtraInfoFiles(
+ self._extrainfoFile).values())[0]
+ self.extrainfoNew = list(descriptors.parseExtraInfoFiles(
+ self._extrainfoNewFile).values())[0]
def _writeNetworkstatus(self, networkstatus):
with open(self._networkstatusFile, 'w') as fh:
@@ -970,7 +970,7 @@ class BridgeTests(unittest.TestCase):
self.assertEqual(
identifier,
''.join(['$$',
- hashlib.sha1(bridge.fingerprint).hexdigest().upper(),
+ hashlib.sha1(bridge.fingerprint.encode('utf-8')).hexdigest().upper(),
'~', bridge.nickname]))
def test_Bridge_str_without_fingerprint(self):
@@ -1013,11 +1013,11 @@ class BridgeTests(unittest.TestCase):
"""
self.bridge.address = '1.1.1.1'
self.bridge.orPort = 443
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
def test_Bridge_allVanillaAddresses_idempotency_others(self):
@@ -1027,17 +1027,17 @@ class BridgeTests(unittest.TestCase):
"""
self.bridge.address = '1.1.1.1'
self.bridge.orPort = 443
- self.assertItemsEqual(self.bridge.orAddresses, [])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.orAddresses, [])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
def test_Bridge_allVanillaAddresses_reentrancy_all(self):
"""Bridge.allVanillaAddresses should be reentrant, i.e. updating the
@@ -1045,34 +1045,34 @@ class BridgeTests(unittest.TestCase):
returned by allVanillaAddresses.
"""
self.bridge.address = '1.1.1.1'
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), None, 4)])
self.assertEqual(self.bridge.address, ipaddr.IPv4Address('1.1.1.1'))
self.assertEqual(self.bridge.orPort, None)
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
self.bridge.orPort = 443
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
self.assertEqual(self.bridge.address, ipaddr.IPv4Address('1.1.1.1'))
self.assertEqual(self.bridge.orPort, 443)
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
self.bridge.address = '2.2.2.2'
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('2.2.2.2'), 443, 4)])
self.assertEqual(self.bridge.address, ipaddr.IPv4Address('2.2.2.2'))
self.assertEqual(self.bridge.orPort, 443)
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
self.bridge.orAddresses.append(
(ipaddr.IPv6Address('200::6ffb:11bb:a129'), 4443, 6))
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('2.2.2.2'), 443, 4),
(ipaddr.IPv6Address('200::6ffb:11bb:a129'), 4443, 6)])
self.assertEqual(self.bridge.address, ipaddr.IPv4Address('2.2.2.2'))
self.assertEqual(self.bridge.orPort, 443)
- self.assertItemsEqual(self.bridge.orAddresses,
+ self.assertCountEqual(self.bridge.orAddresses,
[(ipaddr.IPv6Address('200::6ffb:11bb:a129'), 4443, 6)])
def test_Bridge_allVanillaAddresses_reentrancy_orPort(self):
@@ -1081,15 +1081,15 @@ class BridgeTests(unittest.TestCase):
set, it should return the orPort.
"""
self.bridge.address = '1.1.1.1'
- self.assertItemsEqual(self.bridge.orAddresses, [])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), None, 4)])
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
self.bridge.orPort = 443
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
def test_Bridge_allVanillaAddresses_reentrancy_address(self):
"""Calling Bridge.allVanillaAddresses before Bridge.address is set
@@ -1097,10 +1097,10 @@ class BridgeTests(unittest.TestCase):
is set, it should return the address.
"""
self.bridge.orPort = 443
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(None, 443, 4)])
self.bridge.address = '1.1.1.1'
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
def test_Bridge_allVanillaAddresses_reentrancy_orAddresses(self):
@@ -1109,14 +1109,14 @@ class BridgeTests(unittest.TestCase):
"""
self.bridge.address = '1.1.1.1'
self.bridge.orPort = 443
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
- self.assertItemsEqual(self.bridge.orAddresses, [])
+ self.assertCountEqual(self.bridge.orAddresses, [])
self.bridge.orAddresses.append(
(ipaddr.IPv4Address('2.2.2.2'), 4443, 4))
- self.assertItemsEqual(self.bridge.orAddresses,
+ self.assertCountEqual(self.bridge.orAddresses,
[(ipaddr.IPv4Address('2.2.2.2'), 4443, 4)])
- self.assertItemsEqual(self.bridge.allVanillaAddresses,
+ self.assertCountEqual(self.bridge.allVanillaAddresses,
[(ipaddr.IPv4Address('2.2.2.2'), 4443, 4),
(ipaddr.IPv4Address('1.1.1.1'), 443, 4)])
diff --git a/bridgedb/test/test_captcha.py b/bridgedb/test/test_captcha.py
index 328a03c..b87251e 100644
--- a/bridgedb/test/test_captcha.py
+++ b/bridgedb/test/test_captcha.py
@@ -57,17 +57,17 @@ class ReCaptchaTests(unittest.TestCase):
def test_get(self):
"""Test get() method."""
- # Force urllib2 to do anything less idiotic than the defaults:
+ # Force urllib.request to do anything less idiotic than the defaults:
envkey = 'HTTPS_PROXY'
oldkey = None
- if os.environ.has_key(envkey):
+ if envkey in os.environ:
oldkey = os.environ[envkey]
os.environ[envkey] = '127.0.0.1:9150'
# This stupid thing searches the environment for ``<protocol>_PROXY``
# variables, hence the above 'HTTPS_PROXY' env setting:
- proxy = captcha.urllib2.ProxyHandler()
- opener = captcha.urllib2.build_opener(proxy)
- captcha.urllib2.install_opener(opener)
+ proxy = captcha.urllib.request.ProxyHandler()
+ opener = captcha.urllib.request.build_opener(proxy)
+ captcha.urllib.request.install_opener(opener)
try:
# There isn't really a reliable way to test this function! :(
@@ -77,8 +77,8 @@ class ReCaptchaTests(unittest.TestCase):
reason += "connection.\nThis test failed with: %s" % error
raise unittest.SkipTest(reason)
else:
- self.assertIsInstance(self.c.image, basestring)
- self.assertIsInstance(self.c.challenge, basestring)
+ self.assertIsInstance(self.c.image, bytes)
+ self.assertIsInstance(self.c.challenge, str)
finally:
# Replace the original environment variable if there was one:
if oldkey:
@@ -146,7 +146,7 @@ class GimpCaptchaTests(unittest.TestCase):
c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey,
self.cacheDir)
challenge = c.createChallenge('w00t')
- self.assertIsInstance(challenge, basestring)
+ self.assertIsInstance(challenge, str)
def test_createChallenge_base64(self):
"""createChallenge() return value should be urlsafe base64-encoded."""
@@ -179,18 +179,18 @@ class GimpCaptchaTests(unittest.TestCase):
self.assertEqual(hmac, correctHMAC)
decrypted = self.sekrit.decrypt(orig)
- timestamp = int(decrypted[:12].lstrip('0'))
+ timestamp = int(decrypted[:12].lstrip(b'0'))
# The timestamp should be within 30 seconds of right now.
self.assertApproximates(timestamp, int(time.time()), 30)
- self.assertEqual('ThisAnswerShouldDecryptToThis', decrypted[12:])
+ self.assertEqual(b'ThisAnswerShouldDecryptToThis', decrypted[12:])
def test_get(self):
"""GimpCaptcha.get() should return image and challenge strings."""
c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey,
self.cacheDir)
image, challenge = c.get()
- self.assertIsInstance(image, basestring)
- self.assertIsInstance(challenge, basestring)
+ self.assertIsInstance(image, bytes)
+ self.assertIsInstance(challenge, str)
def test_get_emptyCacheDir(self):
"""An empty cacheDir should raise GimpCaptchaError."""
@@ -207,7 +207,7 @@ class GimpCaptchaTests(unittest.TestCase):
with open(badFile, 'w') as fh:
fh.write(' ')
fh.flush()
- os.chmod(badFile, 0266)
+ os.chmod(badFile, 0o266)
c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey,
self.badCacheDir)
@@ -291,7 +291,7 @@ class GimpCaptchaTests(unittest.TestCase):
c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey,
self.cacheDir)
image, challenge = c.get()
- solution = unicode(c.answer)
+ solution = c.answer if isinstance(c.answer, str) else c.answer.decode('utf-8')
self.assertEquals(
c.check(challenge, solution, c.secretKey, c.hmacKey),
True)
diff --git a/bridgedb/test/test_crypto.py b/bridgedb/test/test_crypto.py
index 7a34671..45c06ae 100644
--- a/bridgedb/test/test_crypto.py
+++ b/bridgedb/test/test_crypto.py
@@ -15,6 +15,7 @@ from __future__ import print_function
from __future__ import unicode_literals
import base64
+import binascii
import io
import logging
import math
@@ -63,15 +64,15 @@ class GetKeyTests(unittest.TestCase):
"""Test retrieving the secret_key from an empty file."""
filename = os.path.join(os.getcwd(), 'sekrit')
key = crypto.getKey(filename)
- self.failUnlessIsInstance(key, basestring,
- "key isn't a string! type=%r" % type(key))
+ self.failUnlessIsInstance(key, bytes,
+ "key isn't bytes! type=%r" % type(key))
def test_getKey_tmpfile(self):
"""Test retrieving the secret_key from a new tmpfile."""
filename = self.mktemp()
key = crypto.getKey(filename)
- self.failUnlessIsInstance(key, basestring,
- "key isn't a string! type=%r" % type(key))
+ self.failUnlessIsInstance(key, bytes,
+ "key isn't bytes! type=%r" % type(key))
def test_getKey_keyexists(self):
"""Write the example key to a file and test reading it back."""
@@ -81,13 +82,13 @@ class GetKeyTests(unittest.TestCase):
fh.flush()
key = crypto.getKey(filename)
- self.failUnlessIsInstance(key, basestring,
- "key isn't a string! type=%r" % type(key))
+ self.failUnlessIsInstance(key, bytes,
+ "key isn't bytes! type=%r" % type(key))
self.assertEqual(SEKRIT_KEY, key,
"""The example key and the one read from file differ!
key (in hex): %s
SEKRIT_KEY (in hex): %s"""
- % (key.encode('hex'), SEKRIT_KEY.encode('hex')))
+ % (binascii.hexlify(key).decode('utf-8'), binascii.hexlify(SEKRIT_KEY).decode('utf-8')))
class InitializeGnuPGTests(unittest.TestCase):
@@ -214,7 +215,7 @@ class InitializeGnuPGTests(unittest.TestCase):
sig = signfunc("This is a test of the public broadcasting system.")
print(sig)
self.assertIsNotNone(sig)
- self.assertTrue(sig.startswith('-----BEGIN PGP SIGNED MESSAGE-----'))
+ self.assertTrue(sig.startswith(b'-----BEGIN PGP SIGNED MESSAGE-----'))
def test_crypto_initializeGnuPG_nonexistent_default_key(self):
"""When the key specified by EMAIL_GPG_PRIMARY_KEY_FINGERPRINT doesn't
diff --git a/bridgedb/test/test_distributors_moat_request.py b/bridgedb/test/test_distributors_moat_request.py
index 555b3eb..a5c22e0 100644
--- a/bridgedb/test/test_distributors_moat_request.py
+++ b/bridgedb/test/test_distributors_moat_request.py
@@ -30,7 +30,7 @@ class MoatBridgeRequest(unittest.TestCase):
self.bridgeRequest.withoutBlockInCountry(data)
self.bridgeRequest.generateFilters()
- self.assertItemsEqual(['byTransportNotBlockedIn(vanilla,us,4)',
+ self.assertCountEqual(['byTransportNotBlockedIn(vanilla,us,4)',
'byTransportNotBlockedIn(vanilla,ir,4)',
'byTransportNotBlockedIn(vanilla,sy,4)',
'byProbingResistance(vanilla,4)'],
diff --git a/bridgedb/test/test_distributors_moat_server.py b/bridgedb/test/test_distributors_moat_server.py
index 33069ef..695116b 100644
--- a/bridgedb/test/test_distributors_moat_server.py
+++ b/bridgedb/test/test_distributors_moat_server.py
@@ -140,7 +140,7 @@ class MiscellaneousTests(unittest.TestCase):
"""
server.setSupportedTransports(self.config.SUPPORTED_TRANSPORTS)
- self.assertItemsEqual(server.getSupportedTransports(),
+ self.assertCountEqual(server.getSupportedTransports(),
["obfs4", "obfs3", "scramblesuit", "fte", "vanilla"])
@@ -553,7 +553,7 @@ class CaptchaFetchResourceTests(unittest.TestCase):
def test_getPreferredTransports_unknown_transport(self):
preferred = self.resource.getPreferredTransports(['dinosaur'])
- self.assertItemsEqual(preferred,
+ self.assertCountEqual(preferred,
['obfs4', 'obfs3', 'fte', 'scramblesuit', 'vanilla'])
def assert_data_is_ok(self, decoded):
@@ -884,7 +884,7 @@ class CaptchaCheckResourceTests(unittest.TestCase):
response = self.resource.failureResponse(6, request, bridgeRequest)
- self.assertIn("No bridges available", response)
+ self.assertIn(b"No bridges available", response)
def test_render_POST_no_bridges(self):
self.mock_getBridgeLines()
@@ -892,7 +892,7 @@ class CaptchaCheckResourceTests(unittest.TestCase):
request = self.create_valid_POST_make_new_challenge()
response = self.resource.render(request)
- self.assertIn("No bridges available", response)
+ self.assertIn(b"No bridges available", response)
def test_render_POST_unexpired(self):
request = self.create_valid_POST_make_new_challenge()
diff --git a/bridgedb/test/test_email_autoresponder.py b/bridgedb/test/test_email_autoresponder.py
index f59e5ef..c8e9624 100644
--- a/bridgedb/test/test_email_autoresponder.py
+++ b/bridgedb/test/test_email_autoresponder.py
@@ -235,7 +235,7 @@ a ball of timey-wimey, wibbly-warbly... stuff."""
response = autoresponder.EmailResponse()
response.write(self.body)
response.rewind()
- contents = str(response.read()).replace('\x00', '')
+ contents = response.read().replace(b'\x00', b'').decode('utf-8')
# The newlines in the email body should have been replaced with
# ``EmailResponse.delimiter``.
delimited = self.body.replace('\n', response.delimiter) \
@@ -247,7 +247,7 @@ a ball of timey-wimey, wibbly-warbly... stuff."""
response = autoresponder.EmailResponse()
response.write(self.body)
response.rewind()
- contents = str(response.read(3)).replace('\x00', '')
+ contents = response.read(3).replace(b'\x00', b'').decode('utf-8')
self.assertEqual(contents, self.body[:3])
def test_EmailResponse_write(self):
@@ -370,7 +370,7 @@ class SMTPAutoresponderTests(unittest.TestCase):
"""
self._getIncomingLines()
ours = Address(self.context.fromAddr)
- plus = '@'.join([ours.local + '+zh_cn', ours.domain])
+ plus = '@'.join([ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')])
self.message.lines[1] = 'To: {0}'.format(plus)
self._setUpResponder()
recipient = str(self.responder.getMailFrom())
@@ -383,7 +383,7 @@ class SMTPAutoresponderTests(unittest.TestCase):
"""
self._getIncomingLines()
ours = Address(self.context.fromAddr)
- plus = '@'.join(['get' + ours.local + '+zh_cn', ours.domain])
+ plus = '@'.join(['get' + ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')])
self.message.lines[1] = 'To: {0}'.format(plus)
self._setUpResponder()
recipient = str(self.responder.getMailFrom())
diff --git a/bridgedb/test/test_email_distributor.py b/bridgedb/test/test_email_distributor.py
index d95370f..716d190 100644
--- a/bridgedb/test/test_email_distributor.py
+++ b/bridgedb/test/test_email_distributor.py
@@ -234,8 +234,8 @@ class EmailDistributorTests(unittest.TestCase):
# There should be an IPv4 subring and an IPv6 ring:
ringnames = dist.hashring.filterRings.keys()
- self.failUnlessIn("IPv4", "".join([str(ringname) for ringname in ringnames]))
- self.failUnlessIn("IPv6", "".join([str(ringname) for ringname in ringnames]))
+ self.failUnlessIn("IPv", "".join([str(ringname) for ringname in ringnames]))
+ self.assertEqual(2, len(ringnames))
[dist.hashring.insert(bridge) for bridge in self.bridges]
@@ -246,8 +246,8 @@ class EmailDistributorTests(unittest.TestCase):
# Ugh, the hashring code is so gross looking.
subrings = dist.hashring.filterRings
- subring1 = subrings.values()[0][1]
- subring2 = subrings.values()[1][1]
+ subring1 = list(subrings.values())[0][1]
+ subring2 = list(subrings.values())[1][1]
# Each subring should have roughly the same number of bridges.
# (Having ±10 bridges in either ring, out of 500 bridges total, should
# be so bad.)
diff --git a/bridgedb/test/test_email_dkim.py b/bridgedb/test/test_email_dkim.py
index be34370..679575f 100644
--- a/bridgedb/test/test_email_dkim.py
+++ b/bridgedb/test/test_email_dkim.py
@@ -11,8 +11,9 @@
"""Unittests for the :mod:`bridgedb.distributors.email.dkim` module."""
+import email
+import email.message
import io
-import rfc822
from twisted.trial import unittest
@@ -46,9 +47,8 @@ get bridges
}
def _createMessage(self, messageString):
- """Create an ``rfc822.Message`` from a string."""
- messageIO = io.StringIO(unicode(messageString))
- return rfc822.Message(messageIO)
+ """Create an ``email.message.Message`` from a string."""
+ return email.message_from_string(messageString)
def test_checkDKIM_good(self):
message = self._createMessage(self.goodMessage)
diff --git a/bridgedb/test/test_email_server.py b/bridgedb/test/test_email_server.py
index bc331f4..fb82cc0 100644
--- a/bridgedb/test/test_email_server.py
+++ b/bridgedb/test/test_email_server.py
@@ -13,10 +13,10 @@
from __future__ import print_function
+import email.message
import socket
import string
import types
-import rfc822
from twisted.python import log
from twisted.internet import defer
@@ -95,10 +95,10 @@ class SMTPMessageTests(unittest.TestCase):
defer.Deferred)
def test_SMTPMessage_getIncomingMessage(self):
- """``getIncomingMessage`` should return a ``rfc822.Message``."""
+ """``getIncomingMessage`` should return a ``email.message.Message``."""
self.message.lineReceived(self.line)
self.assertIsInstance(self.message.getIncomingMessage(),
- rfc822.Message)
+ email.message.Message)
class SMTPIncomingDeliveryTests(unittest.TestCase):
@@ -154,7 +154,7 @@ class SMTPIncomingDeliveryTests(unittest.TestCase):
"""
self.helo = (domain, ipaddress)
self._createProtocolWithHost(domain)
- self.origin = Address('@'.join((username, domain,)))
+ self.origin = Address(b'@'.join((username, domain,)))
self.user = User(username, self.helo, self.proto, self.origin)
def _setUpMAILFROM(self):
@@ -173,9 +173,9 @@ class SMTPIncomingDeliveryTests(unittest.TestCase):
The default is to emulate sending: ``RCPT TO: bridges@localhost``.
"""
- name = username if username is not None else self.config.EMAIL_USERNAME
- host = domain if domain is not None else 'localhost'
- addr = ip if ip is not None else '127.0.0.1'
+ name = username if username is not None else self.config.EMAIL_USERNAME.encode('utf-8')
+ host = domain if domain is not None else b'localhost'
+ addr = ip if ip is not None else b'127.0.0.1'
self._createUser(name, host, ip)
self.delivery.setContext(self.context)
@@ -203,7 +203,7 @@ class SMTPIncomingDeliveryTests(unittest.TestCase):
``'client@example.com'`` should contain a header stating:
``'Received: from example.com'``.
"""
- self._createUser('client', 'example.com', '127.0.0.1')
+ self._createUser(b'client', b'example.com', b'127.0.0.1')
hdr = self.delivery.receivedHeader(self.helo, self.origin, [self.user,])
self.assertSubstring("Received: from example.com", hdr)
@@ -237,14 +237,14 @@ class SMTPIncomingDeliveryTests(unittest.TestCase):
def test_SMTPIncomingDelivery_validateTo_plusAddress(self):
"""Should return a callable that results in a SMTPMessage."""
- self._setUpRCPTTO('bridges+ar')
+ self._setUpRCPTTO(b'bridges+ar')
validated = self.delivery.validateTo(self.user)
self.assertIsInstance(validated, types.FunctionType)
self.assertIsInstance(validated(), server.SMTPMessage)
def test_SMTPIncomingDelivery_validateTo_badUsername_plusAddress(self):
"""'givemebridges+zh_cn@...' should raise an SMTPBadRcpt exception."""
- self._setUpRCPTTO('givemebridges+zh_cn')
+ self._setUpRCPTTO(b'givemebridges+zh_cn')
self.assertRaises(SMTPBadRcpt, self.delivery.validateTo, self.user)
def test_SMTPIncomingDelivery_validateTo_badUsername(self):
@@ -252,20 +252,20 @@ class SMTPIncomingDeliveryTests(unittest.TestCase):
``RCPT TO: yo.mama@localhost`` should raise a
``twisted.mail.smtp.SMTPBadRcpt`` exception.
"""
- self._setUpRCPTTO('yo.mama')
+ self._setUpRCPTTO(b'yo.mama')
self.assertRaises(SMTPBadRcpt, self.delivery.validateTo, self.user)
def test_SMTPIncomingDelivery_validateTo_notOurDomain(self):
"""An SMTP ``RCPT TO: bridges@forealsi.es`` should raise an SMTPBadRcpt
exception.
"""
- self._setUpRCPTTO('bridges', 'forealsi.es')
+ self._setUpRCPTTO(b'bridges', b'forealsi.es')
self.assertRaises(SMTPBadRcpt, self.delivery.validateTo, self.user)
def test_SMTPIncomingDelivery_validateTo_subdomain(self):
"""An SMTP ``RCPT TO: bridges@subdomain.localhost`` should be allowed.
"""
- self._setUpRCPTTO('bridges', 'subdomain.localhost')
+ self._setUpRCPTTO(b'bridges', b'subdomain.localhost')
validated = self.delivery.validateTo(self.user)
self.assertIsInstance(validated, types.FunctionType)
self.assertIsInstance(validated(), server.SMTPMessage)
@@ -339,7 +339,7 @@ class SMTPTestCaseMixin(util.TestCaseMixin):
'Subject: %s' % subject,
'\r\n %s' % body,
'.'] # SMTP DATA EOM command
- emailText = self.proto.delimiter.join(contents)
+ emailText = self.proto.delimiter.decode('utf-8').join(contents)
return emailText
def _buildSMTP(self, commands):
@@ -351,7 +351,7 @@ class SMTPTestCaseMixin(util.TestCaseMixin):
:returns: The string for sending those **commands**, suitable for
giving to a :api:`twisted.internet.Protocol.dataReceived` method.
"""
- data = self.proto.delimiter.join(commands) + self.proto.delimiter
+ data = self.proto.delimiter.decode('utf-8').join(commands) + self.proto.delimiter.decode('utf-8')
return data
def _test(self, commands, expected, noisy=False):
@@ -369,8 +369,8 @@ class SMTPTestCaseMixin(util.TestCaseMixin):
"client" and the "server" in a nicely formatted manner.
"""
data = self._buildSMTP(commands)
- self.proto.dataReceived(data)
- recv = self.transport.value()
+ self.proto.dataReceived(data.encode('utf-8'))
+ recv = self.transport.value().decode('utf-8')
if noisy:
client = data.replace('\r\n', '\r\n ')
@@ -515,7 +515,12 @@ class EmailServerServiceTests(SMTPTestCaseMixin, unittest.TestCase):
def tearDown(self):
"""Kill all connections with fire."""
if self.transport:
+ if not hasattr(self.transport, 'protocol'):
+ factory = server.addServer(self.config, self.dist)
+ self.transport.protocol = factory.buildProtocol(('127.0.0.1', 0))
+
self.transport.loseConnection()
+
super(EmailServerServiceTests, self).tearDown()
# FIXME: this is definitely not how we're supposed to do this, but it
# kills the DirtyReactorAggregateErrors.
diff --git a/bridgedb/test/test_email_templates.py b/bridgedb/test/test_email_templates.py
index 8464fe9..4703019 100644
--- a/bridgedb/test/test_email_templates.py
+++ b/bridgedb/test/test_email_templates.py
@@ -27,7 +27,7 @@ class EmailTemplatesTests(unittest.TestCase):
"""Unittests for :func:`b.e.templates`."""
def setUp(self):
- self.t = NullTranslations(StringIO(unicode('test')))
+ self.t = NullTranslations(StringIO('test'))
self.client = Address('blackhole@torproject.org')
self.answer = 'obfs3 1.1.1.1:1111\nobfs3 2.2.2.2:2222'
# This is the fingerprint of BridgeDB's offline, certification-only
diff --git a/bridgedb/test/test_geo.py b/bridgedb/test/test_geo.py
index 4ca6cbb..371882c 100644
--- a/bridgedb/test/test_geo.py
+++ b/bridgedb/test/test_geo.py
@@ -50,7 +50,7 @@ class GeoTests(unittest.TestCase):
"""Should return the CC since the IP is an ``ipaddr.IPAddress``."""
cc = geo.getCountryCode(self.ipv4)
self.assertIsNotNone(cc)
- self.assertIsInstance(cc, basestring)
+ self.assertIsInstance(cc, str)
self.assertEqual(len(cc), 2)
self.assertEqual(cc, self.expectedCC)
@@ -77,7 +77,7 @@ class GeoTests(unittest.TestCase):
"""Should return the CC since the IP is an ``ipaddr.IPAddress``."""
cc = geo.getCountryCode(self.ipv6)
self.assertIsNotNone(cc)
- self.assertIsInstance(cc, basestring)
+ self.assertIsInstance(cc, str)
self.assertEqual(len(cc), 2)
self.assertEqual(cc, self.expectedCC)
diff --git a/bridgedb/test/test_https.py b/bridgedb/test/test_https.py
index ffd7281..8e3de1b 100644
--- a/bridgedb/test/test_https.py
+++ b/bridgedb/test/test_https.py
@@ -27,11 +27,11 @@ repository.
from __future__ import print_function
import gettext
-import ipaddr
+import ipaddress
import mechanize
import os
-from BeautifulSoup import BeautifulSoup
+from bs4 import BeautifulSoup
from twisted.trial import unittest
from twisted.trial.reporter import TestResult
@@ -137,7 +137,7 @@ class HTTPTests(unittest.TestCase):
# ------------- Results
# URL should be the same as last time
self.assertEquals(self.br.response().geturl(), EXPECTED_URL)
- soup = BeautifulSoup(captcha_response.read())
+ soup = BeautifulSoup(captcha_response.read(), features="html5lib")
return soup
def getBridgeLinesFromSoup(self, soup, fieldsPerBridge):
@@ -174,10 +174,10 @@ class HTTPTests(unittest.TestCase):
self.br.set_debug_http(True)
self.br.open(HTTP_ROOT)
- headers = ''.join(self.br.response().info().headers)
+ headers = self.br.response().info()
- self.assertIn("Content-Security-Policy", headers)
- self.assertIn("default-src 'none';", headers)
+ self.assertIn("Content-Security-Policy", headers.keys())
+ self.assertIn("default-src 'none';", ''.join(headers.values()))
def test_404(self):
"""Asking for a non-existent resource should yield our custom 404 page,
@@ -199,7 +199,7 @@ class HTTPTests(unittest.TestCase):
for bridge in bridges:
self.assertTrue(bridge != None)
addr = bridge[0].rsplit(':', 1)[0]
- self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv4Address)
+ self.assertIsInstance(ipaddress.ip_address(addr), ipaddress.IPv4Address)
def test_get_vanilla_ipv6(self):
self.openBrowser()
@@ -212,7 +212,7 @@ class HTTPTests(unittest.TestCase):
for bridge in bridges:
self.assertTrue(bridge != None)
addr = bridge[0].rsplit(':', 1)[0].strip('[]')
- self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv6Address)
+ self.assertIsInstance(ipaddress.ip_address(addr), ipaddress.IPv6Address)
def test_get_obfs4_ipv4(self):
"""Try asking for obfs4 bridges, and check that the PT arguments in the
@@ -353,7 +353,7 @@ class _HTTPTranslationsTests(unittest.TestCase):
localedir=self.i18n,
languages=[locale,],
fallback=True)
- expected = language.gettext("What are bridges?")
+ expected = language.gettext("What are bridges?").encode("utf-8")
if not locale.startswith('en'):
self.assertNotEqual(expected, "What are bridges?")
diff --git a/bridgedb/test/test_https_distributor.py b/bridgedb/test/test_https_distributor.py
index c25c598..a96238d 100644
--- a/bridgedb/test/test_https_distributor.py
+++ b/bridgedb/test/test_https_distributor.py
@@ -261,7 +261,7 @@ class HTTPSDistributorTests(unittest.TestCase):
[dist.insert(bridge) for bridge in bridges]
- for i in xrange(5):
+ for i in range(5):
bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn')
bridgeRequest1.transports.append('obfs2')
bridgeRequest1.generateFilters()
@@ -322,7 +322,7 @@ class HTTPSDistributorTests(unittest.TestCase):
for i in range(5):
responses[i] = dist.getBridges(bridgeRequest, 1)
for i in range(4):
- self.assertItemsEqual(responses[i], responses[i+1])
+ self.assertCountEqual(responses[i], responses[i+1])
def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self):
param = BridgeRingParameters(needPorts=[(443, 1)])
@@ -335,7 +335,7 @@ class HTTPSDistributorTests(unittest.TestCase):
[dist.insert(bridge) for bridge in bridges]
[dist.insert(bridge) for bridge in self.bridges[:250]]
- for _ in xrange(32):
+ for _ in range(32):
bridgeRequest = self.randomClientRequest()
answer = dist.getBridges(bridgeRequest, 1)
count = 0
@@ -399,7 +399,7 @@ class HTTPSDistributorTests(unittest.TestCase):
dist = distributor.HTTPSDistributor(3, self.key)
[dist.insert(bridge) for bridge in self.bridges[:250]]
- for i in xrange(500):
+ for i in range(500):
bridgeRequest = self.randomClientRequest()
bridgeRequest.withIPv6()
bridgeRequest.generateFilters()
@@ -421,7 +421,7 @@ class HTTPSDistributorTests(unittest.TestCase):
dist = distributor.HTTPSDistributor(1, self.key)
[dist.insert(bridge) for bridge in self.bridges[:250]]
- for i in xrange(500):
+ for i in range(500):
bridgeRequest = self.randomClientRequest()
bridgeRequest.generateFilters()
diff --git a/bridgedb/test/test_https_request.py b/bridgedb/test/test_https_request.py
index 2ea3fe6..81f8e0b 100644
--- a/bridgedb/test/test_https_request.py
+++ b/bridgedb/test/test_https_request.py
@@ -64,7 +64,7 @@ class HTTPSBridgeRequestTests(unittest.TestCase):
self.request = request.HTTPSBridgeRequest(addClientCountryCode=False)
self.request.client = '5.5.5.5'
self.request.withoutBlockInCountry(httprequest)
- self.assertItemsEqual(['nl'], self.request.notBlockedIn)
+ self.assertCountEqual(['nl'], self.request.notBlockedIn)
def test_HTTPSBridgeRequest_withoutBlockInCountry_bad_params(self):
"""HTTPSBridgeRequest.withoutBlockInCountry() should stop processing if
diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py
index 945ea06..ff7a1e3 100644
--- a/bridgedb/test/test_https_server.py
+++ b/bridgedb/test/test_https_server.py
@@ -17,9 +17,9 @@ import logging
import os
import shutil
-import ipaddr
+import ipaddress
-from BeautifulSoup import BeautifulSoup
+from bs4 import BeautifulSoup
from twisted.internet import reactor
from twisted.internet import task
@@ -27,7 +27,7 @@ from twisted.trial import unittest
from twisted.web.resource import Resource
from twisted.web.test import requesthelper
-from bridgedb import translations
+from bridgedb import _langs, translations
from bridgedb.distributors.https import server
from bridgedb.schedule import ScheduledInterval
@@ -70,8 +70,8 @@ class ReplaceErrorPageTests(unittest.TestCase):
request = DummyRequest([''])
exc = Exception("vegan gümmibären")
errorPage = server.replaceErrorPage(request, exc)
- self.assertSubstring("Bad News Bears", errorPage)
- self.assertNotSubstring("vegan gümmibären", errorPage)
+ self.assertSubstring(b"Bad News Bears", errorPage)
+ self.assertNotSubstring("vegan gümmibären".encode("utf-8"), errorPage)
def test_replaceErrorPage_matches_resource500(self):
"""``replaceErrorPage`` should return the error-500.html page."""
@@ -89,9 +89,9 @@ class ReplaceErrorPageTests(unittest.TestCase):
exc = Exception("vegan gümmibären")
server.resource500 = None
errorPage = server.replaceErrorPage(request, exc)
- self.assertNotSubstring("Bad News Bears", errorPage)
- self.assertNotSubstring("vegan gümmibären", errorPage)
- self.assertSubstring("Sorry! Something went wrong with your request.",
+ self.assertNotSubstring(b"Bad News Bears", errorPage)
+ self.assertNotSubstring("vegan gümmibären".encode("utf-8"), errorPage)
+ self.assertSubstring(b"Sorry! Something went wrong with your request.",
errorPage)
class ErrorResourceTests(unittest.TestCase):
@@ -103,17 +103,17 @@ class ErrorResourceTests(unittest.TestCase):
def test_resource404(self):
"""``server.resource404`` should display the error-404.html page."""
page = server.resource404.render(self.request)
- self.assertSubstring('We dug around for the page you requested', page)
+ self.assertSubstring(b'We dug around for the page you requested', page)
def test_resource500(self):
"""``server.resource500`` should display the error-500.html page."""
page = server.resource500.render(self.request)
- self.assertSubstring('Bad News Bears', page)
+ self.assertSubstring(b'Bad News Bears', page)
def test_maintenance(self):
"""``server.maintenance`` should display the error-503.html page."""
page = server.maintenance.render(self.request)
- self.assertSubstring('Under Maintenance', page)
+ self.assertSubstring(b'Under Maintenance', page)
class CustomErrorHandlingResourceTests(unittest.TestCase):
@@ -214,15 +214,19 @@ class IndexResourceTests(unittest.TestCase):
request = DummyRequest([self.pagename])
request.method = b'GET'
page = self.indexResource.render_GET(request)
- self.assertSubstring("add the bridges to Tor Browser", page)
+ self.assertSubstring(b"add the bridges to Tor Browser", page)
def test_IndexResource_render_GET_lang_ta(self):
"""renderGet() with ?lang=ta should return the index page in Tamil."""
+
+ if 'ta' not in _langs.get_langs():
+ self.skipTest("'ta' language unsupported")
+
request = DummyRequest([self.pagename])
request.method = b'GET'
- request.addArg('lang', 'ta')
+ request.addArg(b'lang', 'ta')
page = self.indexResource.render_GET(request)
- self.assertSubstring("bridge-களை Tor Browser-உள்", page)
+ self.assertSubstring("bridge-களை Tor Browser-உள்".encode("utf-8"), page)
class HowtoResourceTests(unittest.TestCase):
@@ -239,15 +243,19 @@ class HowtoResourceTests(unittest.TestCase):
request = DummyRequest([self.pagename])
request.method = b'GET'
page = self.howtoResource.render_GET(request)
- self.assertSubstring("the wizard", page)
+ self.assertSubstring(b"the wizard", page)
def test_HowtoResource_render_GET_lang_ru(self):
"""renderGet() with ?lang=ru should return the howto page in Russian."""
+
+ if 'ru' not in _langs.get_langs():
+ self.skipTest("'ru' language unsupported")
+
request = DummyRequest([self.pagename])
request.method = b'GET'
- request.addArg('lang', 'ru')
+ request.addArg(b'lang', 'ru')
page = self.howtoResource.render_GET(request)
- self.assertSubstring("следовать инструкциям установщика", page)
+ self.assertSubstring("следовать инструкциям установщика".encode("utf-8"), page)
class CaptchaProtectedResourceTests(unittest.TestCase):
@@ -271,7 +279,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase):
request.method = b'GET'
page = self.captchaResource.render_GET(request)
self.assertSubstring(
- "Your browser is not displaying images properly", page)
+ b"Your browser is not displaying images properly", page)
def test_render_GET_missingTemplate(self):
"""render_GET() with a missing template should raise an error and
@@ -329,7 +337,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase):
request = DummyRequest([self.pagename])
request.method = b'POST'
page = self.captchaResource.render_POST(request)
- self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'],
+ self.assertEqual(BeautifulSoup(page, features="html5lib").find('meta')['http-equiv'],
'refresh')
@@ -454,7 +462,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase):
self.request.addArg('captcha_response_field', '')
page = self.captchaResource.render_POST(self.request)
- self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'],
+ self.assertEqual(BeautifulSoup(page, features="html5lib").find('meta')['http-equiv'],
'refresh')
def test_render_POST_wrongSolution(self):
@@ -469,7 +477,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase):
self.request.addArg('captcha_response_field', expectedResponse)
page = self.captchaResource.render_POST(self.request)
- self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'],
+ self.assertEqual(BeautifulSoup(page, features="html5lib").find('meta')['http-equiv'],
'refresh')
@@ -522,7 +530,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase):
def testCB(request):
"""Check the ``Request`` returned from ``_renderDeferred``."""
self.assertIsInstance(request, DummyRequest)
- soup = BeautifulSoup(b''.join(request.written)).find('meta')['http-equiv']
+ soup = BeautifulSoup(b''.join(request.written), features="html5lib").find(b'meta')['http-equiv']
self.assertEqual(soup, 'refresh')
d = task.deferLater(reactor, 0, lambda x: x, (False, self.request))
@@ -541,7 +549,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase):
"""Check the ``Request`` returned from ``_renderDeferred``."""
self.assertIsInstance(request, DummyRequest)
html = b''.join(request.written)
- self.assertSubstring('Uh oh, spaghettios!', html)
+ self.assertSubstring(b'Uh oh, spaghettios!', html)
d = task.deferLater(reactor, 0, lambda x: x, (True, self.request))
d.addCallback(self.captchaResource._renderDeferred)
@@ -580,14 +588,14 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase):
"""Check that removing our remoteip setting produces a random IP."""
self.captchaResource.remoteIP = None
ip = self.captchaResource.getRemoteIP()
- realishIP = ipaddr.IPv4Address(ip).compressed
+ realishIP = ipaddress.IPv4Address(ip).compressed
self.assertTrue(realishIP)
self.assertNotEquals(realishIP, '111.111.111.111')
def test_getRemoteIP_useConfiguredIP(self):
"""Check that our remoteip setting is used if configured."""
ip = self.captchaResource.getRemoteIP()
- realishIP = ipaddr.IPv4Address(ip).compressed
+ realishIP = ipaddress.IPv4Address(ip).compressed
self.assertTrue(realishIP)
self.assertEquals(realishIP, '111.111.111.111')
@@ -666,9 +674,9 @@ class BridgesResourceTests(unittest.TestCase):
:returns: A list of the bridge lines contained on the **page**.
"""
# The bridge lines are contained in a <div class='bridges'> tag:
- soup = BeautifulSoup(page)
+ soup = BeautifulSoup(page, features="html5lib")
well = soup.find('div', {'class': 'bridge-lines'})
- content = well.renderContents().strip()
+ content = well.renderContents().decode('utf-8').strip()
lines = content.splitlines()
bridges = []
@@ -766,7 +774,7 @@ class BridgesResourceTests(unittest.TestCase):
# Check that the IP and port seem okay:
ip, port = fields[0].rsplit(':')
- self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address)
+ self.assertIsInstance(ipaddress.ip_address(ip), ipaddress.IPv4Address)
self.assertIsInstance(int(port), int)
self.assertGreater(int(port), 0)
self.assertLessEqual(int(port), 65535)
@@ -794,6 +802,10 @@ class BridgesResourceTests(unittest.TestCase):
def test_render_GET_RTLlang(self):
"""Test rendering a request for plain bridges in Arabic."""
+
+ if 'ar' not in _langs.get_langs():
+ self.skipTest("'ar' language unsupported")
+
self.useBenignBridges()
request = DummyRequest([b"bridges?transport=obfs3"])
@@ -804,10 +816,10 @@ class BridgesResourceTests(unittest.TestCase):
request.headers.update({'accept-language': 'ar,en,en_US,'})
page = self.bridgesResource.render(request)
- self.assertSubstring("rtl.css", page)
+ self.assertSubstring(b"rtl.css", page)
self.assertSubstring(
# "I need an alternative way to get bridges!"
- "أحتاج إلى وسيلة بديلة للحصول على bridges", page)
+ "أحتاج إلى وسيلة بديلة للحصول على bridges".encode("utf-8"), page)
for bridgeLine in self.parseBridgesFromHTMLPage(page):
# Check that each bridge line had the expected number of fields:
@@ -816,6 +828,10 @@ class BridgesResourceTests(unittest.TestCase):
def test_render_GET_RTLlang_obfs3(self):
"""Test rendering a request for obfs3 bridges in Farsi."""
+
+ if 'fa' not in _langs.get_langs():
+ self.skipTest("'ar' language unsupported")
+
self.useBenignBridges()
request = DummyRequest([b"bridges?transport=obfs3"])
@@ -827,12 +843,12 @@ class BridgesResourceTests(unittest.TestCase):
request.args.update({'transport': ['obfs3']})
page = self.bridgesResource.render(request)
- self.assertSubstring("rtl.css", page)
+ self.assertSubstring(b"rtl.css", page)
self.assertSubstring(
# "How to use the above bridge lines" (since there should be
# bridges in this response, we don't tell them about alternative
# mechanisms for getting bridges)
- "چگونگی از پل‌های خود استفاده کنید", page)
+ "چگونگی از پل‌های خود استفاده کنید".encode("utf-8"), page)
for bridgeLine in self.parseBridgesFromHTMLPage(page):
# Check that each bridge line had the expected number of fields:
@@ -842,7 +858,7 @@ class BridgesResourceTests(unittest.TestCase):
# Check that the IP and port seem okay:
ip, port = bridgeLine[1].rsplit(':')
- self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address)
+ self.assertIsInstance(ipaddress.ip_address(ip), ipaddress.IPv4Address)
self.assertIsInstance(int(port), int)
self.assertGreater(int(port), 0)
self.assertLessEqual(int(port), 65535)
@@ -868,15 +884,15 @@ class BridgesResourceTests(unittest.TestCase):
#
# (Yes, there are two leading spaces at the beginning of each line)
#
- bridgeLines = [line.strip() for line in page.strip().split('\n')]
+ bridgeLines = [line.strip() for line in page.strip().split(b'\n')]
for bridgeLine in bridgeLines:
- bridgeLine = bridgeLine.split(' ')
+ bridgeLine = bridgeLine.split(b' ')
self.assertEqual(len(bridgeLine), 2)
# Check that the IP and port seem okay:
- ip, port = bridgeLine[0].rsplit(':')
- self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address)
+ ip, port = bridgeLine[0].rsplit(b':')
+ self.assertIsInstance(ipaddress.ip_address(ip.decode("utf-8")), ipaddress.IPv4Address)
self.assertIsInstance(int(port), int)
self.assertGreater(int(port), 0)
self.assertLessEqual(int(port), 65535)
@@ -897,8 +913,8 @@ class BridgesResourceTests(unittest.TestCase):
page = self.bridgesResource.renderAnswer(request, bridgeLines=None)
# We don't want the fancy version:
- self.assertNotSubstring("Bad News Bears", page)
- self.assertSubstring("Sorry! Something went wrong with your request.",
+ self.assertNotSubstring(b"Bad News Bears", page)
+ self.assertSubstring(b"Sorry! Something went wrong with your request.",
page)
@@ -915,6 +931,10 @@ class OptionsResourceTests(unittest.TestCase):
def test_render_GET_RTLlang(self):
"""Test rendering a request for obfs3 bridges in Hebrew."""
+
+ if 'he' not in _langs.get_langs():
+ self.skipTest("'ar' language unsupported")
+
request = DummyRequest(["bridges?transport=obfs3"])
request.method = b'GET'
request.getClientIP = lambda: '3.3.3.3'
@@ -924,8 +944,8 @@ class OptionsResourceTests(unittest.TestCase):
request.args.update({'transport': ['obfs2']})
page = self.optionsResource.render(request)
- self.assertSubstring("rtl.css", page)
- self.assertSubstring("מהם גשרים?", page)
+ self.assertSubstring(b"rtl.css", page)
+ self.assertSubstring("מהם גשרים?".encode("utf-8"), page)
class HTTPSServerServiceTests(unittest.TestCase):
diff --git a/bridgedb/test/test_main.py b/bridgedb/test/test_main.py
index 52d98a2..82b609f 100644
--- a/bridgedb/test/test_main.py
+++ b/bridgedb/test/test_main.py
@@ -52,7 +52,7 @@ def mockUpdateBridgeHistory(bridges, timestamps):
which doesn't access the database (so that we can test functions which
call it, like :func:`bridgedb.main.load`).
"""
- for fingerprint, stamps in timestamps.items()[:]:
+ for fingerprint, stamps in timestamps.items():
for timestamp in stamps:
print("Pretending to update Bridge %s with timestamp %s..." %
(fingerprint, timestamp))
@@ -323,7 +323,7 @@ class BridgedbTests(unittest.TestCase):
# a MoatDistributor ring, and an UnallocatedHolder ring:
self.assertEqual(len(hashring.ringsByName.keys()), 4)
self.assertGreater(len(httpsDist.proxies), 0)
- self.assertItemsEqual(exitRelays, httpsDist.proxies)
+ self.assertCountEqual(exitRelays, httpsDist.proxies)
def test_main_createBridgeRings_no_https_dist(self):
"""When HTTPS_DIST=False, main.createBridgeRings() should add only
diff --git a/bridgedb/test/test_metrics.py b/bridgedb/test/test_metrics.py
index ce0f63d..fcc9e4a 100644
--- a/bridgedb/test/test_metrics.py
+++ b/bridgedb/test/test_metrics.py
@@ -15,7 +15,7 @@ These tests are meant to ensure that the :mod:`bridgedb.metrics` module is
functioning as expected.
"""
-import StringIO
+import io
import json
import os
@@ -104,7 +104,7 @@ class StateTest(unittest.TestCase):
self.metrix.inc(self.key)
self.metrix.coldMetrics = self.metrix.hotMetrics
- pseudo_fh = StringIO.StringIO()
+ pseudo_fh = io.StringIO()
metrics.export(pseudo_fh, 0)
self.assertTrue(len(pseudo_fh.getvalue()) > 0)
diff --git a/bridgedb/test/test_parse_addr.py b/bridgedb/test/test_parse_addr.py
index 4659efb..b32d359 100644
--- a/bridgedb/test/test_parse_addr.py
+++ b/bridgedb/test/test_parse_addr.py
@@ -226,7 +226,7 @@ class ParseAddrIsIPAddressTests(unittest.TestCase):
log.msg("addr.isIPAddress(%r) => %s" % (testAddress, result))
self.assertTrue(result is not None,
"Got a None for testAddress: %r" % testAddress)
- self.assertFalse(isinstance(result, basestring),
+ self.assertFalse(isinstance(result, str),
"Expected %r result from isIPAddress(%r): %r %r"
% (bool, testAddress, result, type(result)))
@@ -669,11 +669,11 @@ class PortListTest(unittest.TestCase):
ports.
"""
tooMany = addr.PortList.PORTSPEC_LEN + 1
- ports = [self.getRandomPort() for x in xrange(tooMany)]
+ ports = [self.getRandomPort() for x in range(tooMany)]
log.msg("Testing addr.PortList(%s))"
% ', '.join([type('')(port) for port in ports]))
portList = addr.PortList(*ports)
- self.assertEqual(len(portList), tooMany)
+ self.assertEqual(len(portList), len(set(ports)))
def test_invalidPortNumber(self):
"""Test creating a :class:`addr.PortList` with an invalid port.
@@ -699,8 +699,8 @@ class PortListTest(unittest.TestCase):
ports = (443, 9001, 9030)
portList = addr.PortList(*ports)
iterator = iter(portList)
- for x in xrange(len(ports)):
- self.assertIn(iterator.next(), portList)
+ for x in range(len(ports)):
+ self.assertIn(next(iterator), portList)
def test_str(self):
"""Test creating a :class:`addr.PortList` with valid ports.
@@ -709,7 +709,7 @@ class PortListTest(unittest.TestCase):
"""
ports = (443, 9001, 9030)
portList = addr.PortList(*ports)
- self.assertTrue(isinstance(str(portList), basestring))
+ self.assertTrue(isinstance(str(portList), str))
for port in ports:
self.assertIn(str(port), str(portList))
@@ -735,7 +735,7 @@ class PortListTest(unittest.TestCase):
"""Test ``__getitem__`` with a string."""
ports = (443, 9001, 9030)
portList = addr.PortList(*ports)
- self.assertEqual(portList.__getitem__(long(0)), 9001)
+ self.assertEqual(portList.__getitem__(0), 9001)
def test_mixedArgs(self):
"""Create a :class:`addr.PortList` with mixed type parameters."""
diff --git a/bridgedb/test/test_parse_descriptors.py b/bridgedb/test/test_parse_descriptors.py
index df7f34f..eb86a0e 100644
--- a/bridgedb/test/test_parse_descriptors.py
+++ b/bridgedb/test/test_parse_descriptors.py
@@ -28,7 +28,7 @@ try:
from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor
from stem.descriptor.router_status_entry import RouterStatusEntryV3
from bridgedb.parse import descriptors
-except (ImportError, NameError), error:
+except (ImportError, NameError) as error:
print("There was an error importing stem: %s" % error)
else:
HAS_STEM = True
@@ -36,7 +36,7 @@ else:
from bridgedb.test.util import Benchmarker
-BRIDGE_NETWORKSTATUS_0 = '''\
+BRIDGE_NETWORKSTATUS_0 = b'''\
r MiserLandfalls 4IsyTSCtChPhFPAnq5rD8yymlqA /GMC4lz8RXT/62v6kZNdmzSmopk 2014-11-04 06:23:22 2.215.61.223 4056 0
a [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055
s Fast Guard Running Stable Valid
@@ -44,7 +44,7 @@ w Bandwidth=1678904
p reject 1-65535
'''
-BRIDGE_NETWORKSTATUS_1 = '''\
+BRIDGE_NETWORKSTATUS_1 = b'''\
r Unmentionable BgOrX0ViP5hNsK5ZvixAuPZ6EY0 NTg9NoE5ls9KjF96Dp/UdrabZ9Y 2014-11-04 12:23:37 80.44.173.87 51691 0
a [da14:7d1e:ba8e:60d0:b078:3f88:382b:5c70]:51690
s Fast Guard Running Stable Valid
@@ -52,7 +52,7 @@ w Bandwidth=24361
p reject 1-65535
'''
-BRIDGE_SERVER_DESCRIPTOR = '''\
+BRIDGE_SERVER_DESCRIPTOR = b'''\
@purpose bridge
router MiserLandfalls 2.215.61.223 4056 0 0
or-address [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055
@@ -86,7 +86,7 @@ P2aB/+XQfzFBA5TaWF83coDng4OGodhwHaOx10Kn7Bg=
-----END SIGNATURE-----
'''
-BRIDGE_EXTRA_INFO_DESCRIPTOR = '''\
+BRIDGE_EXTRA_INFO_DESCRIPTOR = b'''\
extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0
published 2014-11-04 06:23:22
write-history 2014-11-04 06:23:22 (900 s) 3188736,2226176,2866176
@@ -117,7 +117,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY=
-----END SIGNATURE-----
'''
-BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = '''\
+BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = b'''\
extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0
published 2014-11-04 08:10:25
write-history 2014-11-04 08:10:25 (900 s) 3188736,2226176,2866176,2226176
@@ -148,7 +148,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY=
-----END SIGNATURE-----
'''
-BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = '''\
+BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = b'''\
extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0
published 2014-12-04 03:10:25
write-history 2014-12-04 03:10:25 (900 s) 3188736,2226176,2866176,2226176
@@ -179,7 +179,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY=
-----END SIGNATURE-----
'''
-BRIDGE_SERVER_DESCRIPTOR_ED25519 = '''\
+BRIDGE_SERVER_DESCRIPTOR_ED25519 = b'''\
@purpose bridge
router piratepartei 80.92.79.70 80 0 0
identity-ed25519
@@ -233,7 +233,7 @@ VC4FdHgFlAkXbiqkpWtD0ojJJjLlEeXbmGILjC1Ls2I=
-----END SIGNATURE-----
'''
-BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = '''\
+BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = b'''\
extra-info piratepartei 312D64274C29156005843EECB19C6865FA3CC10C
identity-ed25519
-----BEGIN ED25519 CERT-----
@@ -307,7 +307,7 @@ class ParseDescriptorsTests(unittest.TestCase):
:returns: The full path to the file which was written to.
"""
descFilename = os.path.join(os.getcwd(), filename)
- with open(descFilename, 'w') as fh:
+ with open(descFilename, 'wb') as fh:
for desc in descriptors:
fh.write(desc)
fh.flush()
@@ -381,8 +381,8 @@ class ParseDescriptorsTests(unittest.TestCase):
raise InvalidRouterNickname.
"""
unparseable = BRIDGE_NETWORKSTATUS_0.replace(
- 'MiserLandfalls',
- 'MiserLandfallsWaterfallsSnowfallsAvalanche')
+ b'MiserLandfalls',
+ b'MiserLandfallsWaterfallsSnowfallsAvalanche')
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
@@ -399,8 +399,8 @@ class ParseDescriptorsTests(unittest.TestCase):
See also: :trac:`16616`
"""
unparseable = BRIDGE_NETWORKSTATUS_0.replace(
- 's Fast Guard Running Stable Valid',
- 's Fast Guard Running Stable Valid HSDir')
+ b's Fast Guard Running Stable Valid',
+ b's Fast Guard Running Stable Valid HSDir')
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
@@ -420,7 +420,7 @@ class ParseDescriptorsTests(unittest.TestCase):
a ValueError.
"""
unparseable = BRIDGE_NETWORKSTATUS_0.replace(
- '2.215.61.223', '[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]')
+ b'2.215.61.223', b'[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]')
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
unparseable)
self.assertRaises(ValueError,
@@ -434,9 +434,9 @@ class ParseDescriptorsTests(unittest.TestCase):
expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1]
descFile = 'networkstatus-bridges'
- with open(descFile, 'w') as fh:
- fh.write('signature and stuff from the BridgeAuth would go here\n')
- fh.write('some more annotations with parameters and stuff\n')
+ with open(descFile, 'wb') as fh:
+ fh.write(b'signature and stuff from the BridgeAuth would go here\n')
+ fh.write(b'some more annotations with parameters and stuff\n')
fh.write(BRIDGE_NETWORKSTATUS_0)
fh.write(BRIDGE_NETWORKSTATUS_1)
fh.flush()
@@ -454,9 +454,9 @@ class ParseDescriptorsTests(unittest.TestCase):
expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1]
descFile = 'networkstatus-bridges'
- with open(descFile, 'w') as fh:
- fh.write('signature and stuff from the BridgeAuth would go here\n')
- fh.write('some more annotations with parameters and stuff\n')
+ with open(descFile, 'wb') as fh:
+ fh.write(b'signature and stuff from the BridgeAuth would go here\n')
+ fh.write(b'some more annotations with parameters and stuff\n')
fh.write(BRIDGE_NETWORKSTATUS_0)
fh.write(BRIDGE_NETWORKSTATUS_1)
fh.flush()
@@ -479,7 +479,7 @@ class ParseDescriptorsTests(unittest.TestCase):
"""
descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR)
routers = descriptors.parseExtraInfoFiles(descFile)
- bridge = routers.values()[0]
+ bridge = list(routers.values())[0]
self.assertIsInstance(bridge, RelayExtraInfoDescriptor)
def test_parse_descriptors_parseExtraInfoFiles_one_file(self):
@@ -488,12 +488,12 @@ class ParseDescriptorsTests(unittest.TestCase):
"""
descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR)
routers = descriptors.parseExtraInfoFiles(descFile)
- bridge = routers.values()[0]
+ bridge = list(routers.values())[0]
# The number of transports we parsed should be equal to the number of
# 'transport' lines in the descriptor:
self.assertEqual(len(bridge.transport),
- BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport '))
+ BRIDGE_EXTRA_INFO_DESCRIPTOR.count(b'transport '))
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
@@ -522,7 +522,7 @@ class ParseDescriptorsTests(unittest.TestCase):
"We shouldn't have any duplicate descriptors.")
# We should only have the newest descriptor:
- bridge = routers.values()[0]
+ bridge = list(routers.values())[0]
self.assertEqual(
bridge.published,
datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"),
@@ -541,7 +541,7 @@ class ParseDescriptorsTests(unittest.TestCase):
self.assertEqual(len(routers), 1,
"We shouldn't have any duplicate descriptors.")
- bridge = routers.values()[0]
+ bridge = list(routers.values())[0]
self.assertEqual(
bridge.published,
datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"),
@@ -564,7 +564,7 @@ class ParseDescriptorsTests(unittest.TestCase):
"We shouldn't have any duplicate descriptors.")
# We should only have the newest descriptor:
- bridge = routers.values()[0]
+ bridge = list(routers.values())[0]
self.assertEqual(
bridge.published,
datetime.datetime.strptime("2014-12-04 03:10:25", "%Y-%m-%d %H:%M:%S"),
@@ -581,10 +581,10 @@ class ParseDescriptorsTests(unittest.TestCase):
descFiles = []
# The timestamp and fingerprint from BRIDGE_EXTRA_INFO_DESCRIPTOR:
- timestamp = "2014-11-04 06:23:22"
- Y, M, rest = timestamp.split("-")
- fpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0"
- newerFpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0"
+ timestamp = b"2014-11-04 06:23:22"
+ Y, M, rest = timestamp.split(b"-")
+ fpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0"
+ newerFpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0"
total = 0
needed = b * n
@@ -592,15 +592,15 @@ class ParseDescriptorsTests(unittest.TestCase):
if total >= needed:
break
# Re-digest the fingerprint to create a "new" bridge
- newerFpr = hashlib.sha1(newerFpr).hexdigest().upper()
+ newerFpr = hashlib.sha1(newerFpr).hexdigest().upper().encode('utf-8')
# Generate n extrainfos with different timestamps:
count = 0
- for year in range(1, ((n + 1)/ 12) + 2): # Start from the next year
+ for year in range(1, ((n + 1)// 12) + 2): # Start from the next year
if count >= n:
break
for month in range(1, 13):
if count < n:
- newerTimestamp = "-".join([str(int(Y) + year), "%02d" % month, rest])
+ newerTimestamp = b"-".join([str(int(Y) + year).encode('utf-8'), b"%02d" % month, rest])
newerDuplicate = BRIDGE_EXTRA_INFO_DESCRIPTOR[:].replace(
fpr, newerFpr).replace(
timestamp, newerTimestamp)
@@ -663,10 +663,10 @@ class ParseDescriptorsTests(unittest.TestCase):
"""
# Give it a bad geoip-db-digest:
unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace(
- "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
- "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace(
- "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F",
- "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR")
+ b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
+ b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace(
+ b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F",
+ b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR")
descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE)
descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR)
@@ -684,7 +684,7 @@ class ParseDescriptorsTests(unittest.TestCase):
"and one was unparseable, so that should only leave one "
"descriptor remaining."))
- bridge = routers.values()[0]
+ bridge = list(routers.values())[0]
self.assertEqual(
bridge.fingerprint,
"E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
@@ -703,14 +703,14 @@ class ParseDescriptorsTests(unittest.TestCase):
"""
# Mess up the bridge-ip-transports line:
unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace(
- "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
- "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace(
- "bridge-ip-transports <OR>=8",
- "bridge-ip-transports <OR>")
+ b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
+ b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace(
+ b"bridge-ip-transports <OR>=8",
+ b"bridge-ip-transports <OR>")
parseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace(
- "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
- "ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F")
+ b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
+ b"ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F")
descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR)
descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE)
@@ -750,8 +750,8 @@ class ParseDescriptorsTests(unittest.TestCase):
zero parsed descriptors.
"""
unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace(
- '-----END SIGNATURE-----',
- '-----END SIGNATURE FOR REALZ-----')
+ b'-----END SIGNATURE-----',
+ b'-----END SIGNATURE FOR REALZ-----')
# This must be a "real" file or _copyUnparseableDescriptorFile() will
# raise an AttributeError saying:
# '_io.BytesIO' object has no attribute 'rpartition'"
@@ -766,7 +766,7 @@ class ParseDescriptorsTests(unittest.TestCase):
missing the signature should return zero parsed descriptors.
"""
# Remove the signature
- BEGIN_SIG = '-----BEGIN SIGNATURE-----'
+ BEGIN_SIG = b'-----BEGIN SIGNATURE-----'
unparseable, _ = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG)
# This must be a "real" file or _copyUnparseableDescriptorFile() will
# raise an AttributeError saying:
@@ -782,7 +782,7 @@ class ParseDescriptorsTests(unittest.TestCase):
bad signature should raise an InvalidExtraInfoSignature exception.
"""
# Truncate the signature to 50 bytes
- BEGIN_SIG = '-----BEGIN SIGNATURE-----'
+ BEGIN_SIG = b'-----BEGIN SIGNATURE-----'
doc, sig = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG)
unparseable = BEGIN_SIG.join([doc, sig[:50]])
# This must be a "real" file or _copyUnparseableDescriptorFile() will
@@ -803,10 +803,10 @@ class ParseDescriptorsTests(unittest.TestCase):
"""
# Give it a bad geoip-db-digest:
unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace(
- "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
- "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace(
- "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F",
- "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR")
+ b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0",
+ b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace(
+ b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F",
+ b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR")
descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR)
descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE)
@@ -817,7 +817,7 @@ class ParseDescriptorsTests(unittest.TestCase):
def test_parse_descriptors_parseExtraInfoFiles_empty_file(self):
"""Test parsing an empty extrainfo descriptors file."""
- routers = descriptors.parseExtraInfoFiles(io.BytesIO(''))
+ routers = descriptors.parseExtraInfoFiles(io.BytesIO(b''))
self.assertIsInstance(routers, dict)
self.assertEqual(len(routers), 0)
@@ -846,7 +846,7 @@ class ParseDescriptorsTests(unittest.TestCase):
True when the new file is successfully created.
"""
filename = "bridge-descriptors"
- with open(filename, 'w') as fh:
+ with open(filename, 'wb') as fh:
fh.write(BRIDGE_SERVER_DESCRIPTOR)
fh.flush()
@@ -858,7 +858,7 @@ class ParseDescriptorsTests(unittest.TestCase):
copy of the bad file with a specific filename format.
"""
filename = "bridge-descriptors"
- with open(filename, 'w') as fh:
+ with open(filename, 'wb') as fh:
fh.write(BRIDGE_SERVER_DESCRIPTOR)
fh.flush()
diff --git a/bridgedb/test/test_persistent.py b/bridgedb/test/test_persistent.py
index 858ad1a..21aa67f 100644
--- a/bridgedb/test/test_persistent.py
+++ b/bridgedb/test/test_persistent.py
@@ -17,10 +17,10 @@ functioning as expected.
from __future__ import print_function
+import io
import os.path
from copy import deepcopy
-from io import StringIO
from bridgedb import persistent
from bridgedb.parse.options import MainOptions
@@ -33,9 +33,9 @@ from sure import the
from sure import expect
-TEST_CONFIG_FILE = StringIO(unicode("""\
+TEST_CONFIG_FILE = io.StringIO("""\
BRIDGE_FILES = ['bridge-descriptors', 'bridge-descriptors.new']
-LOGFILE = 'bridgedb.log'"""))
+LOGFILE = 'bridgedb.log'""")
class StateTest(unittest.TestCase):
@@ -47,10 +47,10 @@ class StateTest(unittest.TestCase):
configuration = {}
TEST_CONFIG_FILE.seek(0)
compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec')
- exec compiled in configuration
+ exec(compiled, configuration)
config = persistent.Conf(**configuration)
- fakeArgs = ['-c', os.path.join(os.getcwdu(), '..', 'bridgedb.conf')]
+ fakeArgs = ['-c', os.path.join(os.getcwd(), '..', 'bridgedb.conf')]
options = MainOptions()
options.parseOptions(fakeArgs)
@@ -142,8 +142,8 @@ class StateTest(unittest.TestCase):
setattr(thatConfig, 'BAR', 'all of the things')
setattr(thatConfig, 'LOGFILE', 42)
- this(thatConfig).should.have.property('FOO').being.a(basestring)
- this(thatConfig).should.have.property('BAR').being.a(basestring)
+ this(thatConfig).should.have.property('FOO').being.a(str)
+ this(thatConfig).should.have.property('BAR').being.a(str)
this(thatConfig).should.have.property('LOGFILE').being.an(int)
this(thatConfig).should.have.property('BRIDGE_FILES').being.a(list)
@@ -156,8 +156,8 @@ class StateTest(unittest.TestCase):
thatState.useChangedSettings(thatConfig)
the(thatState.FOO).should.equal('fuuuuu')
- the(thatState).should.have.property('FOO').being.a(basestring)
- the(thatState).should.have.property('BAR').being.a(basestring)
+ the(thatState).should.have.property('FOO').being.a(str)
+ the(thatState).should.have.property('BAR').being.a(str)
the(thatState).should.have.property('LOGFILE').being.an(int)
the(thatState.FOO).must.equal(thatConfig.FOO)
the(thatState.BAR).must.equal(thatConfig.BAR)
diff --git a/bridgedb/test/test_persistentSaveAndLoad.py b/bridgedb/test/test_persistentSaveAndLoad.py
index 6d6584d..bcdcdd7 100644
--- a/bridgedb/test/test_persistentSaveAndLoad.py
+++ b/bridgedb/test/test_persistentSaveAndLoad.py
@@ -18,19 +18,19 @@ are all functioning as expected.
This module should not import :mod:`sure`.
"""
+import io
import os
from copy import deepcopy
-from io import StringIO
from twisted.trial import unittest
from bridgedb import persistent
-TEST_CONFIG_FILE = StringIO(unicode("""\
+TEST_CONFIG_FILE = io.StringIO("""\
BRIDGE_FILES = ['bridge-descriptors', 'bridge-descriptors.new']
-LOGFILE = 'bridgedb.log'"""))
+LOGFILE = 'bridgedb.log'""")
class StateSaveAndLoadTests(unittest.TestCase):
@@ -42,7 +42,7 @@ class StateSaveAndLoadTests(unittest.TestCase):
configuration = {}
TEST_CONFIG_FILE.seek(0)
compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec')
- exec compiled in configuration
+ exec(compiled, configuration)
config = persistent.Conf(**configuration)
self.config = config
@@ -57,10 +57,9 @@ class StateSaveAndLoadTests(unittest.TestCase):
self.assertIsInstance(loadedState, persistent.State)
self.assertNotIdentical(self.state, loadedState)
self.assertNotEqual(self.state, loadedState)
- # For some reason, twisted.trial.unittest.TestCase in Python2.6
- # doesn't have an 'assertItemsEqual' attribute...
- self.assertEqual(self.state.__dict__.keys().sort(),
- loadedState.__dict__.keys().sort())
+
+ self.assertEqual(list(self.state.__dict__.keys()).sort(),
+ list(loadedState.__dict__.keys()).sort())
def savedStateAssertions(self, savedStatefile=None):
self.assertTrue(os.path.isfile(str(self.state.statefile)))
@@ -85,12 +84,12 @@ class StateSaveAndLoadTests(unittest.TestCase):
def test_get_statefile(self):
statefile = self.state._get_statefile()
- self.assertIsInstance(statefile, basestring)
+ self.assertIsInstance(statefile, str)
def test_set_statefile(self):
self.state._set_statefile('bar.state')
statefile = self.state._get_statefile()
- self.assertIsInstance(statefile, basestring)
+ self.assertIsInstance(statefile, str)
def test_set_statefile_new_dir(self):
config = self.config
@@ -149,7 +148,7 @@ class StateSaveAndLoadTests(unittest.TestCase):
self.state.load, 'quux.state')
def test_load_with_statefile_opened(self):
- fh = open('quux.state', 'w+')
+ fh = open('quux.state', 'wb+')
self.assertRaises(persistent.MissingState, self.state.load, fh)
fh.close()
diff --git a/bridgedb/test/test_proxy.py b/bridgedb/test/test_proxy.py
index 0a69eb0..69b2b99 100644
--- a/bridgedb/test/test_proxy.py
+++ b/bridgedb/test/test_proxy.py
@@ -254,7 +254,7 @@ class ProxySetUnittests(unittest.TestCase):
def test_ProxySet_proxies_getter(self):
"""ProxySet.proxies should list all proxies."""
- self.assertItemsEqual(self.proxyList.proxies, set(self.proxies))
+ self.assertCountEqual(self.proxyList.proxies, set(self.proxies))
def test_ProxySet_proxies_setter(self):
"""``ProxySet.proxies = ['foo']`` should raise an ``AttributeError``."""
@@ -273,7 +273,7 @@ class ProxySetUnittests(unittest.TestCase):
def test_ProxySet_exitRelays_getter(self):
"""ProxySet.exitRelays should list all exit relays."""
self.proxyList.addExitRelays(self.moarProxies)
- self.assertItemsEqual(self.proxyList.exitRelays, set(self.moarProxies))
+ self.assertCountEqual(self.proxyList.exitRelays, set(self.moarProxies))
def test_ProxySet_exitRelays_setter(self):
"""``ProxySet.exitRelays = ['foo']`` should raise an ``AttributeError``."""
@@ -350,7 +350,7 @@ class ProxySetUnittests(unittest.TestCase):
whatever tags we want.
"""
tags = ['foo', 'bar', 'baz']
- extraProxies = zip(self.moarProxies, tags)
+ extraProxies = list(zip(self.moarProxies, tags))
self.proxyList.addProxies(extraProxies)
self.assertEquals(len(self.proxyList), len(self.proxies) + len(extraProxies))
self.assertIn(extraProxies[0][0], self.proxyList)
@@ -393,13 +393,13 @@ class ProxySetUnittests(unittest.TestCase):
def test_ProxySet_addProxies_bad_type(self):
"""``ProxySet.addProxies()`` called with something which is neither an
- iterable, a basestring, or an int should raise a ValueError.
+ iterable, a str, or an int should raise a ValueError.
"""
self.assertRaises(ValueError, self.proxyList.addProxies, object)
def test_ProxySet_addProxies_list_of_bad_types(self):
"""``ProxySet.addProxies()`` called with something which is neither an
- iterable, a basestring, or an int should raise a ValueError.
+ iterable, a str, or an int should raise a ValueError.
"""
self.assertRaises(ValueError, self.proxyList.addProxies, [object, object, object])
@@ -447,9 +447,9 @@ class ProxySetUnittests(unittest.TestCase):
"""
proxySetA = self.proxyList
proxySetB = proxy.ProxySet(self.moarProxies)
- self.assertItemsEqual(proxySetA.difference(proxySetB),
+ self.assertCountEqual(proxySetA.difference(proxySetB),
set(self.proxies))
- self.assertItemsEqual(proxySetB.difference(proxySetA),
+ self.assertCountEqual(proxySetB.difference(proxySetA),
set(self.moarProxies))
def test_ProxySet_firstSeen_returns_timestamp(self):
@@ -493,7 +493,7 @@ class ProxySetUnittests(unittest.TestCase):
a.extend(self.moarProxies)
a = set(a)
b = self.proxyList.intersection(set(self.moarProxies))
- self.assertItemsEqual(a, b)
+ self.assertCountEqual(a, b)
def test_ProxySet_remove(self):
"""ProxySet.remove() should subtract proxies which were already added
@@ -534,7 +534,7 @@ class ProxySetUnittests(unittest.TestCase):
proxyListA = proxy.ProxySet(self.proxies)
proxyListB = proxy.ProxySet(self.proxies)
self.assertEqual(proxyListA, proxyListB)
- self.assertItemsEqual(proxyListA, proxyListB)
+ self.assertCountEqual(proxyListA, proxyListB)
self.assertEqual(hash(proxyListA), hash(proxyListB))
diff --git a/bridgedb/test/test_schedule.py b/bridgedb/test/test_schedule.py
index 1bcec07..c95d6f9 100644
--- a/bridgedb/test/test_schedule.py
+++ b/bridgedb/test/test_schedule.py
@@ -168,7 +168,7 @@ class ScheduledIntervalTests(unittest.TestCase):
ts = sched.getInterval(now)
self.assertIsInstance(ts, str)
secs = [int(x) for x in ts.replace('-', ' ').replace(':', ' ').split()]
- [secs.append(0) for _ in xrange(6-len(secs))]
+ [secs.append(0) for _ in range(6-len(secs))]
secs = schedule.calendar.timegm(secs)
self.assertApproximates(now, secs, variance)
diff --git a/bridgedb/test/test_smtp.py b/bridgedb/test/test_smtp.py
index de443b3..f7261da 100644
--- a/bridgedb/test/test_smtp.py
+++ b/bridgedb/test/test_smtp.py
@@ -5,7 +5,7 @@ from __future__ import print_function
import smtplib
import asyncore
import threading
-import Queue
+import queue
import random
import os
@@ -44,7 +44,7 @@ LOCAL_SMTP_SERVER_PORT = 2525 # Must be the same as bridgedb's EMAIL_SMTP_PORT
class EmailServer(SMTPServer):
- def process_message(self, peer, mailfrom, rcpttos, data):
+ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
''' Overridden from SMTP server, called whenever a message is received'''
self.message_queue.put(data)
@@ -58,7 +58,7 @@ class EmailServer(SMTPServer):
self.close()
def start(self):
- self.message_queue = Queue.Queue()
+ self.message_queue = queue.Queue()
self._stop = threading.Event()
self._thread = threading.Thread(target=self.thread_proc)
# Ensures that if any tests do fail, then threads will exit when the
@@ -93,7 +93,7 @@ class EmailServer(SMTPServer):
# failures:
#
# https://travis-ci.org/isislovecruft/bridgedb/jobs/58996136#L3281
- except Queue.Empty:
+ except queue.Empty:
pass
else:
assert message.find(text) != -1, ("Message did not contain text '%s'."
@@ -103,7 +103,7 @@ class EmailServer(SMTPServer):
def checkNoMessageReceived(self, timeoutInSecs=2.0):
try:
self.message_queue.get(block=True, timeout=timeoutInSecs)
- except Queue.Empty:
+ except queue.Empty:
return True
assert False, "Found a message in the queue, but expected none"
@@ -152,7 +152,7 @@ class SMTPTests(unittest.TestCase):
# then check that our local SMTP server received a response
# and that response contained some bridges
- self.server.getAndCheckMessageContains("Here are your bridges")
+ self.server.getAndCheckMessageContains(b"Here are your bridges")
def test_getBridges_rateLimitExceeded(self):
if os.environ.get("CI"):
@@ -168,14 +168,14 @@ class SMTPTests(unittest.TestCase):
# then check that our local SMTP server received a response
# and that response contained some bridges
- self.server.getAndCheckMessageContains("Here are your bridges")
+ self.server.getAndCheckMessageContains(b"Here are your bridges")
# send another request from the same email address
sendMail(FROM_ADDRESS)
# this time, the email response should not contain any bridges
self.server.getAndCheckMessageContains(
- "You have exceeded the rate limit. Please slow down!")
+ b"You have exceeded the rate limit. Please slow down!")
# then we send another request from the same email address
sendMail(FROM_ADDRESS)
@@ -203,4 +203,4 @@ class SMTPTests(unittest.TestCase):
% random.randint(MIN_FROM_ADDRESS, MAX_FROM_ADDRESS))
for i in range(NUM_MAILS):
- self.server.getAndCheckMessageContains("Here are your bridges")
+ self.server.getAndCheckMessageContains(b"Here are your bridges")
diff --git a/bridgedb/test/test_translations.py b/bridgedb/test/test_translations.py
index 7d14cc0..4b504f1 100644
--- a/bridgedb/test/test_translations.py
+++ b/bridgedb/test/test_translations.py
@@ -10,7 +10,7 @@
from twisted.trial import unittest
-from bridgedb import translations
+from bridgedb import _langs, translations
from bridgedb.test.https_helpers import DummyRequest
@@ -40,15 +40,16 @@ class TranslationsMiscTests(unittest.TestCase):
request = DummyRequest([b"bridges"])
request.headers.update(REALISH_HEADERS)
request.args.update({
- b'transport': [b'obfs3',],
- b'lang': [b'ar',],
+ 'transport': ['obfs3',],
+ 'lang': ['ar',],
})
parsed = translations.getLocaleFromHTTPRequest(request)
+
+ self.assertEqual(len(parsed), 3)
self.assertEqual(parsed[0], 'ar')
self.assertEqual(parsed[1], 'en')
self.assertEqual(parsed[2], 'en_US')
- self.assertEqual(len(parsed), 3)
def test_getLocaleFromHTTPRequest_withLangParam_AcceptLanguage(self):
"""This request uses a '?lang=ar' param, with an 'Accept-Language'
@@ -58,14 +59,15 @@ class TranslationsMiscTests(unittest.TestCase):
"""
request = DummyRequest([b"options"])
request.headers.update(ACCEPT_LANGUAGE_HEADER)
- request.args.update({b'lang': [b'fa']})
+ request.args.update({'lang': ['fa']})
parsed = translations.getLocaleFromHTTPRequest(request)
+
+ self.assertEqual(len(parsed), 3)
self.assertEqual(parsed[0], 'fa')
self.assertEqual(parsed[1], 'en')
self.assertEqual(parsed[2], 'en_US')
#self.assertEqual(parsed[3], 'en-gb')
- self.assertEqual(len(parsed), 3)
def test_getLocaleFromPlusAddr(self):
emailAddr = 'bridges@torproject.org'
@@ -80,4 +82,6 @@ class TranslationsMiscTests(unittest.TestCase):
def test_usingRTLLang(self):
self.assertFalse(translations.usingRTLLang(['foo_BAR']))
self.assertFalse(translations.usingRTLLang(['en']))
- self.assertTrue(translations.usingRTLLang(['fa']))
+
+ if 'fa' in _langs.get_langs():
+ self.assertTrue(translations.usingRTLLang(['fa']))
diff --git a/bridgedb/test/test_txrecaptcha.py b/bridgedb/test/test_txrecaptcha.py
index 2550c3d..db37506 100644
--- a/bridgedb/test/test_txrecaptcha.py
+++ b/bridgedb/test/test_txrecaptcha.py
@@ -142,7 +142,7 @@ class BodyProducerTests(unittest.TestCase):
def setUp(self):
"""Setup the tests."""
- self.content = 'Line 1\r\nLine 2\r\n'
+ self.content = b'Line 1\r\nLine 2\r\n'
self.producer = txrecaptcha._BodyProducer(self.content)
def test_interface(self):
@@ -220,7 +220,7 @@ class SubmitTests(unittest.TestCase):
"""
self.assertIsInstance(response, txrecaptcha.RecaptchaResponse)
self.assertIsInstance(response.is_valid, bool)
- self.assertIsInstance(response.error_code, basestring)
+ self.assertIsInstance(response.error_code, str)
d = txrecaptcha.submit(self.challenge, self.response, self.key,
self.ip)
@@ -265,12 +265,3 @@ class MiscTests(unittest.TestCase):
result = txrecaptcha._ebRequest(fail)
self.assertIsInstance(result, txrecaptcha.RecaptchaResponse)
self.assertRegexpMatches(result.error_code, msg)
-
- def test_encodeIfNecessary(self):
- """:func:`txrecapcha._encodeIfNecessary` should convert unicode objects
- into strings.
- """
- origString = unicode('abc')
- self.assertIsInstance(origString, unicode)
- newString = txrecaptcha._encodeIfNecessary(origString)
- self.assertIsInstance(newString, str)
diff --git a/bridgedb/test/test_util.py b/bridgedb/test/test_util.py
index e858861..55ce35f 100644
--- a/bridgedb/test/test_util.py
+++ b/bridgedb/test/test_util.py
@@ -169,7 +169,7 @@ class JustifiedLogFormatterTests(unittest.TestCase):
formatter = util.JustifiedLogFormatter()
formatted = formatter.format(self.record)
self.assertIsInstance(formatter, logging.Formatter)
- self.assertIsInstance(formatted, basestring)
+ self.assertIsInstance(formatted, str)
self.assertNotEqual(formatted, '')
self.assertTrue('INFO' in formatted)
self.assertTrue('This is a message' in formatted)
diff --git a/bridgedb/test/util.py b/bridgedb/test/util.py
index 9fc16b8..43219d5 100644
--- a/bridgedb/test/util.py
+++ b/bridgedb/test/util.py
@@ -135,7 +135,7 @@ def randomIPv6():
return ipaddr.IPv6Address(random.getrandbits(128))
def randomIP():
- if random.choice(xrange(2)):
+ if random.choice(range(2)):
return randomIPv4()
return randomIPv6()
@@ -146,7 +146,7 @@ def randomIPv6String():
return bracketIPv6(randomIPv6().compressed)
def randomIPString():
- if random.choice(xrange(2)):
+ if random.choice(range(2)):
return randomIPv4String()
return randomIPv6String()
@@ -194,7 +194,7 @@ def generateFakeBridges(n=500):
# Real tor currently only supports one extra ORAddress, and it can
# only be IPv6.
addrs = [(randomValidIPv6(), randomHighPort(), 6)]
- fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40))
+ fpr = "".join(random.choice('abcdef0123456789') for _ in range(40))
supported = ["obfs2", "obfs3", "fte"]
transports = []
@@ -282,7 +282,7 @@ class DummyBridge(object):
self.address = ipaddr.IPv4Address(ipv4)
self.orPort = randomPort()
self.fingerprint = "".join(random.choice('abcdef0123456789')
- for _ in xrange(40))
+ for _ in range(40))
self.orAddresses = [(randomIPv6(), randomPort(), 6)]
def getBridgeLine(self, bridgeRequest, includeFingerprint=True):
diff --git a/bridgedb/translations.py b/bridgedb/translations.py
index 6d7d332..b6a9ef3 100644
--- a/bridgedb/translations.py
+++ b/bridgedb/translations.py
@@ -58,7 +58,10 @@ def getFirstSupportedLang(langs):
if l in supported:
lang = l
break
- return lang
+
+ # crop locales (like 'en-US') to just the language
+
+ return lang.split('-')[0]
def getLocaleFromHTTPRequest(request):
"""Retrieve the languages from an HTTP ``Accept-Language:`` header.
@@ -77,7 +80,7 @@ def getLocaleFromHTTPRequest(request):
logging.debug("Client sent no 'Accept-Language' header. Using fallback.")
header = 'en,en-US'
- langs = headers.parseAcceptLanguage(header)
+ langs = list(headers.parseAcceptLanguage(header))
if not safelog.safe_logging: # pragma: no cover
logging.debug("Client Accept-Language (top 5): %s" % langs[:5])
@@ -87,6 +90,10 @@ def getLocaleFromHTTPRequest(request):
logging.debug("Client requested language: %r" % chosenLang)
langs.insert(0, chosenLang)
+ # normalize languages to be unicode
+
+ langs = list(map(lambda l: l if isinstance(l, str) else l.decode('utf-8'), langs))
+
installTranslations(langs)
return langs
@@ -124,9 +131,9 @@ def installTranslations(langs):
gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR,
languages=langs, fallback=True))
except IOError as error:
- logging.error(error.message)
+ logging.error(str(error))
- language.install(unicode=True)
+ language.install()
return language
def usingRTLLang(langs):
@@ -140,11 +147,11 @@ def usingRTLLang(langs):
:returns: ``True`` if the preferred language is right-to-left; ``False``
otherwise.
"""
+
lang = getFirstSupportedLang(langs)
- rtl = False
try:
- rtl = babel.core.Locale.parse(lang).text_direction == "rtl"
+ return babel.core.Locale.parse(lang).text_direction == "rtl"
except ValueError as err:
logging.warning("Couldn't parse locale %s: %s" % (lang, err))
- return rtl
+ return False
diff --git a/bridgedb/txrecaptcha.py b/bridgedb/txrecaptcha.py
index fc4ca2f..7292360 100644
--- a/bridgedb/txrecaptcha.py
+++ b/bridgedb/txrecaptcha.py
@@ -41,17 +41,17 @@ from twisted.web import client
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer
-from zope.interface import implements
+from zope.interface import implementer
from bridgedb.crypto import SSLVerifyingContextFactory
#: This was taken from :data:`recaptcha.client.captcha.API_SSL_SERVER`.
-API_SSL_SERVER = API_SERVER = "https://www.google.com/recaptcha/api"
-API_SSL_VERIFY_URL = "%s/verify" % API_SSL_SERVER
+API_SSL_SERVER = API_SERVER = b"https://www.google.com/recaptcha/api"
+API_SSL_VERIFY_URL = b"%s/verify" % API_SSL_SERVER
#: (:class:`OpenSSL.crypto.X509`) Only trust certificate for the reCAPTCHA
#: :data:`API_SSL_SERVER` which were signed by the Google Internet Authority CA.
-GOOGLE_INTERNET_AUTHORITY_CA_CERT = load_certificate(FILETYPE_PEM, bytes("""\
+GOOGLE_INTERNET_AUTHORITY_CA_CERT = load_certificate(FILETYPE_PEM, b"""\
-----BEGIN CERTIFICATE-----
MIICsDCCAhmgAwIBAgIDFXfhMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT
MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0
@@ -68,7 +68,7 @@ Y3Jscy9zZWN1cmVjYS5jcmwwDQYJKoZIhvcNAQEFBQADgYEAvprjecFG+iJsxzEF
ZUNgujFQodUovxOWZshcnDW7fZ7mTlk3zpeVJrGPZzhaDhvuJjIfKqHweFB7gwB+
ARlIjNvrPq86fpVg0NOTawALkSqOUMl3MynBQO+spR7EHcRbADQ/JemfTEh2Ycfl
vZqhEFBfurZkX0eTANq98ZvVfpg=
------END CERTIFICATE-----"""))
+-----END CERTIFICATE-----""")
# `t.w.client.HTTPConnectionPool` isn't available in Twisted-12.0.0
# (see ticket #11219: https://bugs.torproject.org/11219):
@@ -209,9 +209,9 @@ class RecaptchaResponseProtocol(protocol.Protocol):
self.finished.callback(result)
+@implementer(IBodyProducer)
class _BodyProducer(object):
"""I write a string into the HTML body of an open request."""
- implements(IBodyProducer)
def __init__(self, body):
self.body = body
@@ -253,12 +253,6 @@ def _ebRequest(fail):
error = fail.getErrorMessage() or "possible problem in _ebRequest()"
return RecaptchaResponse(is_valid=False, error_code=error)
-def _encodeIfNecessary(string):
- """Encode unicode objects in utf-8 if necessary."""
- if isinstance(string, unicode):
- return string.encode('utf-8')
- return string
-
def submit(recaptcha_challenge_field, recaptcha_response_field,
private_key, remoteip, agent=_agent):
"""Submits a reCaptcha request for verification. This function is a patched
@@ -291,14 +285,15 @@ def submit(recaptcha_challenge_field, recaptcha_response_field,
d.errback(failure.Failure(ValueError('incorrect-captcha-sol')))
return d
- params = urllib.urlencode({
- 'privatekey': _encodeIfNecessary(private_key),
- 'remoteip': _encodeIfNecessary(remoteip),
- 'challenge': _encodeIfNecessary(recaptcha_challenge_field),
- 'response': _encodeIfNecessary(recaptcha_response_field)})
+ params = urllib.parse.urlencode({
+ 'privatekey': private_key,
+ 'remoteip': remoteip,
+ 'challenge': recaptcha_challenge_field,
+ 'response': recaptcha_response_field,
+ }).encode('utf-8')
body = _BodyProducer(params)
headers = Headers({"Content-type": ["application/x-www-form-urlencoded"],
"User-agent": ["reCAPTCHA Python"]})
- d = agent.request('POST', API_SSL_VERIFY_URL, headers, body)
+ d = agent.request(b'POST', API_SSL_VERIFY_URL, headers, body)
d.addCallbacks(_cbRequest, _ebRequest)
return d
diff --git a/bridgedb/util.py b/bridgedb/util.py
index 63f508e..9572ad1 100644
--- a/bridgedb/util.py
+++ b/bridgedb/util.py
@@ -18,6 +18,7 @@ import logging
import logging.config
import logging.handlers
import os
+import re
import time
from twisted.python import components
@@ -260,17 +261,15 @@ def replaceControlChars(text, replacement=None, encoding="utf-8"):
:rtype: str
:returns: The sanitized **text**.
"""
- escaped = bytearray()
- for byte in bytearray(text, encoding):
- if byte in range(0, 32) + [92, 127]:
- if replacement:
- byte = replacement
- else:
- continue
- escaped += bytearray([byte])
+ if replacement is None:
+ replacement = ''
+
+ # the following replaces characters 0-31, 92, and 127
+
+ text = text.decode(encoding) if isinstance(text, bytes) else text
+ return re.sub(r'[\x00-\x1f\x5c\x7f]', '', text)
- return str(escaped)
def registerAdapter(adapter, adapted, interface):
"""Register a Zope interface adapter for global use.
@@ -308,7 +307,6 @@ class JustifiedLogFormatter(logging.Formatter):
:param bool logTrace: If ``True``, include information on the calling
function in formatted log messages.
"""
- super(JustifiedLogFormatter, self).__init__(datefmt=datefmt)
self.logThreads = logThreads
self.logTrace = logTrace
@@ -318,7 +316,7 @@ class JustifiedLogFormatter(logging.Formatter):
_fmt.append("%(callingFunc)s")
_fmt.append("%(message)s")
- self._fmt = " ".join(_fmt)
+ super(JustifiedLogFormatter, self).__init__(fmt = " ".join(_fmt), datefmt=datefmt)
def _formatCallingFuncName(self, record):
"""Format the combined module name and function name of the place where
@@ -360,7 +358,7 @@ class JustifiedLogFormatter(logging.Formatter):
return super(JustifiedLogFormatter, self).format(record)
-class mixin:
+class mixin(metaclass=abc.ABCMeta):
"""Subclasses of me can be used as a mixin class by registering another
class, ``ClassA``, which should be mixed with the ``mixin`` subclass, in
order to provide simple, less error-prone, multiple inheritance models::
@@ -407,4 +405,3 @@ class mixin:
.. info:: This class' name is lowercased because pylint is hardcoded to
expect mixin classes to end in ``'mixin'``.
"""
- __metaclass__ = abc.ABCMeta
diff --git a/requirements.txt b/requirements.txt
index 5769e11..eed4889 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
Babel==2.8.0
-BeautifulSoup==3.2.2
+beautifulsoup4==4.8.2
+html5lib==1.0b8
Mako==1.1.1
pycryptodome==3.9.6
Twisted==19.10.0
diff --git a/scripts/bridgedb b/scripts/bridgedb
index 876fde9..5c92179 100644
--- a/scripts/bridgedb
+++ b/scripts/bridgedb
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# This file is part of BridgeDB, a Tor bridge distribution system.
diff --git a/scripts/create_descriptors b/scripts/create_descriptors
index ab6ea2e..60022db 100755
--- a/scripts/create_descriptors
+++ b/scripts/create_descriptors
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
#
# This file is part of BridgeDB, a Tor bridge distribution system.
@@ -6,7 +6,7 @@ import os
import random
import sys
import time
-import ipaddr
+import ipaddress
import math
import argparse
import hashlib
@@ -117,10 +117,13 @@ def get_hex_string(size):
return hexstr
-def get_random_ipv6_addr():
+def get_random_addr(ip_version=4):
valid_addr = None
while not valid_addr:
- maybe = ipaddr.IPv6Address(random.getrandbits(128))
+ if ip_version == 4:
+ maybe = ipaddress.IPv4Address(random.getrandbits(32))
+ else:
+ maybe = ipaddress.IPv6Address(random.getrandbits(128))
valid = check_ip_validity(maybe)
if valid:
valid_addr = maybe
@@ -128,13 +131,6 @@ def get_random_ipv6_addr():
return str(valid_addr)
-def get_random_ipv4_addr():
- return "%i.%i.%i.%i" % (random.randint(0, 255),
- random.randint(0, 255),
- random.randint(0, 255),
- random.randint(0, 255))
-
-
def get_protocol(tor_version):
line = ""
if tor_version is not None:
@@ -198,8 +194,8 @@ def create_server_desc(signing_key):
timestamp = make_timestamp(variation=True, period=36)
server_desc = RelayDescriptor.create({
- "router": "%s %s %s 0 0" % (nickname, get_random_ipv4_addr(), port),
- "or-address": "[%s]:%s" % (get_random_ipv6_addr(), port-1),
+ "router": "%s %s %s 0 0" % (nickname, get_random_addr(ip_version=4), port),
+ "or-address": "[%s]:%s" % (get_random_addr(ip_version=6), port-1),
"platform": "Tor %s on Linux" % tor_version,
get_protocol(tor_version): "",
"published": timestamp,
diff --git a/scripts/get-tor-exits b/scripts/get-tor-exits
index 6ab2201..c250dbe 100755
--- a/scripts/get-tor-exits
+++ b/scripts/get-tor-exits
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# This file is part of BridgeDB, a Tor bridge distribution system.
@@ -17,8 +17,9 @@ from __future__ import print_function
import os.path
import socket
import sys
+import io
-from ipaddr import IPAddress
+from ipaddress import IPv4Address
from OpenSSL import SSL
@@ -57,10 +58,10 @@ def getSelfIPAddress():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('bridges.torproject.org', 443))
name = s.getsockname()[0]
- ip = IPAddress(name)
+ ip = IPv4Address(name)
if ip.is_link_local or ip.is_private or ip.is_reserved:
name = s.getpeername()[0]
- ip = IPAddress(name)
+ ip = IPv4Address(name)
except ValueError as error:
log.err("get-tor-exits: A socket gave us something that wasn't an IP: %s"
% error)
@@ -117,7 +118,7 @@ class FileWriter(protocol.Protocol):
"""Write a portion of the download with ``bytes`` size to disk."""
if self.remaining:
display = bytes[:self.remaining]
- self.fh.write(display)
+ self.fh.write(display.decode("utf-8"))
self.fh.flush()
self.remaining -= len(display)
@@ -128,32 +129,11 @@ class FileWriter(protocol.Protocol):
self.finished.callback(None)
-class WebClientContextFactory(ssl.ClientContextFactory):
- """An HTTPS client."""
-
- def getContext(self, hostname, port):
- """Get this connection's OpenSSL context.
-
- By default, :api:`twisted.internet.ssl.ClientContextFactory` uses
- ``OpenSSL.SSL.SSLv23_METHOD``, which allows SSLv2, SSLv3, and TLSv1,
- then they disable SSLv2 in the
- :api:`twisted.internet.ssl.ClientContextFactory.getContext` method.
-
- We disable SSLv3 also.
-
- :rtype: ``OpenSSL.SSL.Context``
- :returns: An OpenSSL context with options set.
- """
- ctx = self._contextFactory(self.method)
- ctx.set_options(SSL.OP_NO_SSLv2 ^ SSL.OP_NO_SSLv3)
- return ctx
-
-
def main(filename=None, address=None, port=None):
fh = filename
if filename:
- if (not isinstance(filename, file)) and (filename is not sys.stdout):
+ if (not isinstance(filename, io.TextIOBase)) and (filename is not sys.stdout):
fh = open(filename, 'w')
if not address:
@@ -170,9 +150,8 @@ def main(filename=None, address=None, port=None):
log.msg("get-tor-exits: Requesting %s..." % check)
- contextFactory = WebClientContextFactory()
- agent = client.Agent(reactor, contextFactory)
- d = agent.request("GET", check)
+ agent = client.Agent(reactor)
+ d = agent.request(b"GET", check.encode("utf-8"))
d.addCallback(writeToFile, fh)
d.addErrback(handle)
d.addCallbacks(log.msg, log.err)
@@ -182,7 +161,7 @@ def main(filename=None, address=None, port=None):
reactor.run()
if filename:
- if (not isinstance(filename, file)) and (filename is not sys.stdout):
+ if (not isinstance(filename, io.TextIOBase)) and (filename is not sys.stdout):
fh.flush()
fh.close()
diff --git a/setup.py b/setup.py
index 55dea95..18c391b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
#_____________________________________________________________________________
#
# This file is part of BridgeDB, a Tor bridge distribution system.