diff options
| author | hiro <hiro@torproject.org> | 2019-02-05 19:12:59 +0100 |
|---|---|---|
| committer | hiro <hiro@torproject.org> | 2019-02-05 19:12:59 +0100 |
| commit | 9f5394e7b32c502f1a0e4d294605996ace50ceaa (patch) | |
| tree | 16a85e2a006d767dd0a89f967a98f6b58ded6107 | |
| parent | ba19003400fe11a68444bbb585fcdb12a93c100f (diff) | |
Start with main task
| -rw-r--r-- | AUTHORS | 1 | ||||
| -rw-r--r-- | doc/INSTALL-SMTP (renamed from INSTALL-SMTP) | 0 | ||||
| -rw-r--r-- | gettor/Storage.py | 490 | ||||
| -rw-r--r-- | gettor/__init__.py | 17 | ||||
| -rw-r--r-- | gettor/main.py | 94 | ||||
| -rw-r--r-- | gettor/strings.py | 80 | ||||
| -rw-r--r-- | share/locale/available_locales.json | 3 | ||||
| -rw-r--r-- | share/locale/en.json | 10 | ||||
| -rw-r--r-- | share/version.txt | 1 |
9 files changed, 695 insertions, 1 deletions
@@ -1,4 +1,5 @@ Current maintainer/core developers: + hiro <hiro@torproject.org> Israel Leiva <ilv@torproject.org> 4096R/540BFC0E Past core developers: diff --git a/INSTALL-SMTP b/doc/INSTALL-SMTP index ae7099f..ae7099f 100644 --- a/INSTALL-SMTP +++ b/doc/INSTALL-SMTP diff --git a/gettor/Storage.py b/gettor/Storage.py new file mode 100644 index 0000000..4182c4b --- /dev/null +++ b/gettor/Storage.py @@ -0,0 +1,490 @@ +# BridgeDB by Nick Mathewson. +# Copyright (c) 2007-2009, The Tor Project, Inc. +# See LICENSE for licensing information + +import calendar +import logging +import binascii +import sqlite3 +import time +import hashlib +from contextlib import GeneratorContextManager +from functools import wraps +from ipaddr import IPAddress +import sys + +from bridgedb.Stability import BridgeHistory +import threading + +toHex = binascii.b2a_hex +fromHex = binascii.a2b_hex +HEX_ID_LEN = 40 + +def _escapeValue(v): + return "'%s'" % v.replace("'", "''") + +def timeToStr(t): + return time.strftime("%Y-%m-%d %H:%M", time.gmtime(t)) +def strToTime(t): + return calendar.timegm(time.strptime(t, "%Y-%m-%d %H:%M")) + +# The old DB system was just a key->value mapping DB, with special key +# prefixes to indicate which database they fell into. +# +# sp|<ID> -- given to bridgesplitter; maps bridgeID to ring name. +# em|<emailaddr> -- given to emailbaseddistributor; maps email address +# to concatenated ID. +# fs|<ID> -- Given to BridgeTracker, maps to time when a router was +# first seen (YYYY-MM-DD HH:MM) +# ls|<ID> -- given to bridgetracker, maps to time when a router was +# last seen (YYYY-MM-DD HH:MM) +# +# We no longer want to use em| at all, since we're not doing that kind +# of persistence any more. + +# Here is the SQL schema. +SCHEMA2_SCRIPT = """ + CREATE TABLE Config ( + key PRIMARY KEY NOT NULL, + value + ); + + CREATE TABLE Bridges ( + id INTEGER PRIMARY KEY NOT NULL, + hex_key, + address, + or_port, + distributor, + first_seen, + last_seen + ); + + CREATE UNIQUE INDEX BridgesKeyIndex ON Bridges ( hex_key ); + + CREATE TABLE EmailedBridges ( + email PRIMARY KEY NOT NULL, + when_mailed + ); + + CREATE INDEX EmailedBridgesWhenMailed on EmailedBridges ( email ); + + CREATE TABLE BlockedBridges ( + id INTEGER PRIMARY KEY NOT NULL, + hex_key, + blocking_country + ); + + CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); + + CREATE TABLE WarnedEmails ( + email PRIMARY KEY NOT NULL, + when_warned + ); + + CREATE INDEX WarnedEmailsWasWarned on WarnedEmails ( email ); + + INSERT INTO Config VALUES ( 'schema-version', 2 ); +""" + +SCHEMA_2TO3_SCRIPT = """ + CREATE TABLE BridgeHistory ( + fingerprint PRIMARY KEY NOT NULL, + address, + port INT, + weightedUptime LONG, + weightedTime LONG, + weightedRunLength LONG, + totalRunWeights DOUBLE, + lastSeenWithDifferentAddressAndPort LONG, + lastSeenWithThisAddressAndPort LONG, + lastDiscountedHistoryValues LONG, + lastUpdatedWeightedTime LONG + ); + + CREATE INDEX BridgeHistoryIndex on BridgeHistory ( fingerprint ); + + INSERT OR REPLACE INTO Config VALUES ( 'schema-version', 3 ); + """ +SCHEMA3_SCRIPT = SCHEMA2_SCRIPT + SCHEMA_2TO3_SCRIPT + + +class BridgeData(object): + """Value class carrying bridge information: + hex_key - The unique hex key of the given bridge + address - Bridge IP address + or_port - Bridge TCP port + distributor - The distributor (or pseudo-distributor) through which + this bridge is being announced + first_seen - When did we first see this bridge online? + last_seen - When was the last time we saw this bridge online? + """ + def __init__(self, hex_key, address, or_port, distributor="unallocated", + first_seen="", last_seen=""): + self.hex_key = hex_key + self.address = address + self.or_port = or_port + self.distributor = distributor + self.first_seen = first_seen + self.last_seen = last_seen + + +class Database(object): + def __init__(self, sqlite_fname): + self._conn = openDatabase(sqlite_fname) + self._cur = self._conn.cursor() + self.sqlite_fname = sqlite_fname + + def commit(self): + self._conn.commit() + + def rollback(self): + self._conn.rollback() + + def close(self): + #print "Closing DB" + self._cur.close() + self._conn.close() + + def getBridgeDistributor(self, bridge, validRings): + """If a ``bridge`` is already in the database, get its distributor. + + :rtype: None or str + :returns: The ``bridge`` distribution method, if one was + already assigned, otherwise, returns None. + """ + distribution_method = None + cur = self._cur + + cur.execute("SELECT id, distributor FROM Bridges WHERE hex_key = ?", + (bridge.fingerprint,)) + result = cur.fetchone() + + if result: + if result[1] in validRings: + distribution_method = result[1] + + return distribution_method + + def insertBridgeAndGetRing(self, bridge, setRing, seenAt, validRings, + defaultPool="unallocated"): + '''Updates info about bridge, setting ring to setRing if none was set. + Also sets distributor to `defaultPool' if the bridge was found in + the database, but its distributor isn't valid anymore. + + Returns the name of the distributor the bridge is assigned to. + ''' + cur = self._cur + + t = timeToStr(seenAt) + h = bridge.fingerprint + assert len(h) == HEX_ID_LEN + + cur.execute("SELECT id, distributor " + "FROM Bridges WHERE hex_key = ?", (h,)) + v = cur.fetchone() + if v is not None: + i, ring = v + # Check if this is currently a valid ring name. If not, move back + # into default pool. + if ring not in validRings: + ring = defaultPool + # Update last_seen, address, port and (possibly) distributor. + cur.execute("UPDATE Bridges SET address = ?, or_port = ?, " + "distributor = ?, last_seen = ? WHERE id = ?", + (str(bridge.address), bridge.orPort, ring, + timeToStr(seenAt), i)) + return ring + else: + # Check if this is currently a valid ring name. If not, move back + # into default pool. + if setRing not in validRings: + setRing = defaultPool + # Insert it. + cur.execute("INSERT INTO Bridges (hex_key, address, or_port, " + "distributor, first_seen, last_seen) " + "VALUES (?, ?, ?, ?, ?, ?)", + (h, str(bridge.address), bridge.orPort, setRing, t, t)) + return setRing + + def cleanEmailedBridges(self, expireBefore): + cur = self._cur + t = timeToStr(expireBefore) + cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,)) + + def getEmailTime(self, addr): + addr = hashlib.sha1(addr).hexdigest() + cur = self._cur + cur.execute("SELECT when_mailed FROM EmailedBridges WHERE email = ?", (addr,)) + v = cur.fetchone() + if v is None: + return None + return strToTime(v[0]) + + def setEmailTime(self, addr, whenMailed): + addr = hashlib.sha1(addr).hexdigest() + cur = self._cur + t = timeToStr(whenMailed) + cur.execute("INSERT OR REPLACE INTO EmailedBridges " + "(email,when_mailed) VALUES (?,?)", (addr, t)) + + def getAllBridges(self): + """Return a list of BridgeData value classes of all bridges in the + database + """ + retBridges = [] + cur = self._cur + cur.execute("SELECT hex_key, address, or_port, distributor, " + "first_seen, last_seen FROM Bridges") + for b in cur.fetchall(): + bridge = BridgeData(b[0], b[1], b[2], b[3], b[4], b[5]) + retBridges.append(bridge) + + return retBridges + + def getBridgesForDistributor(self, distributor): + """Return a list of BridgeData value classes of all bridges in the + database that are allocated to distributor 'distributor' + """ + retBridges = [] + cur = self._cur + cur.execute("SELECT hex_key, address, or_port, distributor, " + "first_seen, last_seen FROM Bridges WHERE " + "distributor = ?", (distributor, )) + for b in cur.fetchall(): + bridge = BridgeData(b[0], b[1], b[2], b[3], b[4], b[5]) + retBridges.append(bridge) + + return retBridges + + def updateDistributorForHexKey(self, distributor, hex_key): + cur = self._cur + cur.execute("UPDATE Bridges SET distributor = ? WHERE hex_key = ?", + (distributor, hex_key)) + + def getWarnedEmail(self, addr): + addr = hashlib.sha1(addr).hexdigest() + cur = self._cur + cur.execute("SELECT * FROM WarnedEmails WHERE email = ?", (addr,)) + v = cur.fetchone() + if v is None: + return False + return True + + def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()): + addr = hashlib.sha1(addr).hexdigest() + t = timeToStr(whenWarned) + cur = self._cur + if warned == True: + cur.execute("INSERT INTO WarnedEmails" + "(email,when_warned) VALUES (?,?)", (addr, t,)) + elif warned == False: + cur.execute("DELETE FROM WarnedEmails WHERE email = ?", (addr,)) + + def cleanWarnedEmails(self, expireBefore): + cur = self._cur + t = timeToStr(expireBefore) + + cur.execute("DELETE FROM WarnedEmails WHERE when_warned < ?", (t,)) + + def updateIntoBridgeHistory(self, bh): + cur = self._cur + cur.execute("INSERT OR REPLACE INTO BridgeHistory values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (bh.fingerprint, str(bh.ip), bh.port, + bh.weightedUptime, bh.weightedTime, bh.weightedRunLength, + bh.totalRunWeights, bh.lastSeenWithDifferentAddressAndPort, + bh.lastSeenWithThisAddressAndPort, bh.lastDiscountedHistoryValues, + bh.lastUpdatedWeightedTime)) + return bh + + def delBridgeHistory(self, fp): + cur = self._cur + cur.execute("DELETE FROM BridgeHistory WHERE fingerprint = ?", (fp,)) + + def getBridgeHistory(self, fp): + cur = self._cur + cur.execute("SELECT * FROM BridgeHistory WHERE fingerprint = ?", (fp,)) + h = cur.fetchone() + if h is None: + return + return BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10]) + + def getAllBridgeHistory(self): + cur = self._cur + v = cur.execute("SELECT * FROM BridgeHistory") + if v is None: return + for h in v: + yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10]) + + def getBridgesLastUpdatedBefore(self, statusPublicationMillis): + cur = self._cur + v = cur.execute("SELECT * FROM BridgeHistory WHERE lastUpdatedWeightedTime < ?", + (statusPublicationMillis,)) + if v is None: return + for h in v: + yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10]) + + +def openDatabase(sqlite_file): + conn = sqlite3.Connection(sqlite_file) + cur = conn.cursor() + try: + try: + cur.execute("SELECT value FROM Config WHERE key = 'schema-version'") + val, = cur.fetchone() + if val == 2: + logging.info("Adding new table BridgeHistory") + cur.executescript(SCHEMA_2TO3_SCRIPT) + elif val != 3: + logging.warn("Unknown schema version %s in database.", val) + except sqlite3.OperationalError: + logging.warn("No Config table found in DB; creating tables") + cur.executescript(SCHEMA3_SCRIPT) + conn.commit() + finally: + cur.close() + return conn + + +class DBGeneratorContextManager(GeneratorContextManager): + """Helper for @contextmanager decorator. + + Overload __exit__() so we can call the generator many times + """ + def __exit__(self, type, value, traceback): + """Handle exiting a with statement block + + Progress generator or throw exception + + Significantly based on contextlib.py + + :throws: `RuntimeError` if the generator doesn't stop after + exception is thrown + """ + if type is None: + try: + self.gen.next() + except StopIteration: + return + return + else: + if value is None: + # Need to force instantiation so we can reliably + # tell if we get the same exception back + value = type() + try: + self.gen.throw(type, value, traceback) + raise RuntimeError("generator didn't stop after throw()") + except StopIteration, exc: + # Suppress the exception *unless* it's the same exception that + # was passed to throw(). This prevents a StopIteration + # raised inside the "with" statement from being suppressed + return exc is not value + except: + # only re-raise if it's *not* the exception that was + # passed to throw(), because __exit__() must not raise + # an exception unless __exit__() itself failed. But throw() + # has to raise the exception to signal propagation, so this + # fixes the impedance mismatch between the throw() protocol + # and the __exit__() protocol. + # + if sys.exc_info()[1] is not value: + raise + +def contextmanager(func): + """Decorator to for :func:`Storage.getDB()` + + Define getDB() for use by with statement content manager + """ + @wraps(func) + def helper(*args, **kwds): + return DBGeneratorContextManager(func(*args, **kwds)) + return helper + +_DB_FNAME = None +_LOCK = None +_LOCKED = 0 +_OPENED_DB = None +_REFCOUNT = 0 + +def clearGlobalDB(): + """Start from scratch. + + This is currently only used in unit tests. + """ + global _DB_FNAME + global _LOCK + global _LOCKED + global _OPENED_DB + + _DB_FNAME = None + _LOCK = None + _LOCKED = 0 + _OPENED_DB = None + _REFCOUNT = 0 + +def initializeDBLock(): + """Create the lock + + This must be called before the first database query + """ + global _LOCK + + if not _LOCK: + _LOCK = threading.RLock() + assert _LOCK + +def setDBFilename(sqlite_fname): + global _DB_FNAME + _DB_FNAME = sqlite_fname + +@contextmanager +def getDB(block=True): + """Generator: Return a usable database handler + + Always return a :class:`bridgedb.Storage.Database` that is + usable within the current thread. If a connection already exists + and it was created by the current thread, then return the + associated :class:`bridgedb.Storage.Database` instance. Otherwise, + create a new instance, blocking until the existing connection + is closed, if applicable. + + Note: This is a blocking call (by default), be careful about + deadlocks! + + :rtype: :class:`bridgedb.Storage.Database` + :returns: An instance of :class:`bridgedb.Storage.Database` used to + query the database + """ + global _DB_FNAME + global _LOCK + global _LOCKED + global _OPENED_DB + global _REFCOUNT + + assert _LOCK + try: + own_lock = _LOCK.acquire(block) + if own_lock: + _LOCKED += 1 + + if not _OPENED_DB: + assert _REFCOUNT == 0 + _OPENED_DB = Database(_DB_FNAME) + + _REFCOUNT += 1 + yield _OPENED_DB + else: + yield False + finally: + assert own_lock + try: + _REFCOUNT -= 1 + if _REFCOUNT == 0: + _OPENED_DB.close() + _OPENED_DB = None + finally: + _LOCKED -= 1 + _LOCK.release() + +def dbIsLocked(): + return _LOCKED != 0 diff --git a/gettor/__init__.py b/gettor/__init__.py index c87425a..ef52564 100644 --- a/gettor/__init__.py +++ b/gettor/__init__.py @@ -1 +1,16 @@ -# yes it's empty, of such a fullness +# -*- coding: utf-8 -*- +""" +This file is part of GetTor, a service providing alternative methods to download +the Tor Browser. + +:authors: Hiro <hiro@torproject.org> + please also see AUTHORS file +:copyright: (c) 2008-2014, The Tor Project, Inc. + (c) 2014, all entities within the AUTHORS file +:license: see included LICENSE for information +""" + +from . import strings + +__version__ = get_version() +__locales__ = get_locales() diff --git a/gettor/main.py b/gettor/main.py new file mode 100644 index 0000000..d294c6c --- /dev/null +++ b/gettor/main.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +""" +This file is part of GetTor, a service providing alternative methods to download +the Tor Browser. + +:authors: Hiro <hiro@torproject.org> + please also see AUTHORS file +:copyright: (c) 2008-2014, The Tor Project, Inc. + (c) 2014, all entities within the AUTHORS file +:license: see included LICENSE for information +""" + +"""This module sets up BridgeDB and starts the servers running.""" + +import logging +import os +import signal +import sys +import time + +from twisted.internet import reactor +from twisted.internet import task + +from . import Storage + +def run(options, reactor=reactor): + """This is GetTor's main entry point and main runtime loop. + + Given the parsed commandline options, this function handles locating the + configuration file, loading and parsing it, and then either (re)parsing + plus (re)starting the servers, or dumping bridge assignments to files. + + :type options: :class:`gettor.parse.options.MainOptions` + :param options: A pre-parsed options class containing any arguments and + options given in the commandline we were called with. + :type state: :class:`bridgedb.persistent.State` + :ivar state: A persistent state object which holds config changes. + :param reactor: An implementer of + :api:`twisted.internet.interfaces.IReactorCore`. This parameter is + mainly for testing; the default + :api:`twisted.internet.epollreactor.EPollReactor` is fine for normal + application runs. + """ + # Change to the directory where we're supposed to run. This must be done + # before parsing the config file, otherwise there will need to be two + # copies of the config file, one in the directory BridgeDB is started in, + # and another in the directory it changes into. + os.chdir(options['rundir']) + if options['verbosity'] <= 10: # Corresponds to logging.DEBUG + print("Changed to runtime directory %r" % os.getcwd()) + + config = loadConfig(options['config']) + config.RUN_IN_DIR = options['rundir'] + + # Set up logging as early as possible. We cannot import from the gettor + # package any of our modules which import :mod:`logging` and start using + # it, at least, not until :func:`safelog.configureLogging` is + # called. Otherwise a default handler that logs to the console will be + # created by the imported module, and all further calls to + # :func:`logging.basicConfig` will be ignored. + util.configureLogging(config) + + if options.subCommand is not None: + runSubcommand(options, config) + + # Write the pidfile only after any options.subCommands are run (because + # these exit when they are finished). Otherwise, if there is a subcommand, + # the real PIDFILE would get overwritten with the PID of the temporary + # bridgedb process running the subcommand. + if config.PIDFILE: + logging.debug("Writing server PID to file: '%s'" % config.PIDFILE) + with open(config.PIDFILE, 'w') as pidfile: + pidfile.write("%s\n" % os.getpid()) + pidfile.flush() + + from bridgedb import persistent + + state = persistent.State(config=config) + + from bridgedb.distributors.email.server import addServer as addSMTPServer + from bridgedb.distributors.https.server import addWebServer + from bridgedb.distributors.moat.server import addMoatServer + + # Load the master key, or create a new one. + key = crypto.getKey(config.MASTER_KEY_FILE) + proxies = proxy.ProxySet() + emailDistributor = None + ipDistributor = None + moatDistributor = None + + # Save our state + state.proxies = proxies + state.key = key + state.save() diff --git a/gettor/strings.py b/gettor/strings.py new file mode 100644 index 0000000..45c33ce --- /dev/null +++ b/gettor/strings.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" +This file is part of GetTor, a service providing alternative methods to download +the Tor Browser. + +:authors: Hiro <hiro@torproject.org> + please also see AUTHORS file +:copyright: (c) 2008-2014, The Tor Project, Inc. + (c) 2014, all entities within the AUTHORS file +:license: see included LICENSE for information +""" + +import json +import locale +import os + +strings = {} +translations = {} + + +def get_resource_path(filename): + """ + Returns the absolute path of a resource + """ + prefix = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))), 'share') + if not os.path.exists(prefix): + prefix = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(prefix)))), 'share') + + return os.path.join(prefix, filename) + +def get_version(): + # The current version + version = "" + with open(get_resource_path('version.txt')) as f: + version = f.read().strip() + return version + +def get_locales(): + locale_dir = get_resource_path('locale') + filename = os.path.join(locale_dir, "available_locales.json") + locales = {} + with open(filename, encoding='utf-8') as f: + locales = json.load(f) + return locales + +def load_strings(current_locale): + """ + Loads translated strings and fallback to English + if the translation does not exist. + """ + global strings, translations + + # Load all translations + translations = {} + available_locales = get_locales() + + for locale in available_locales: + locale_dir = get_resource_path('locale') + filename = os.path.join(locale_dir, "{}.json".format(locale)) + with open(filename, encoding='utf-8') as f: + translations[locale] = json.load(f) + + # Build strings + default_locale = 'en' + + strings = {} + for s in translations[default_locale]: + if s in translations[current_locale] and translations[current_locale][s] != "": + strings[s] = translations[current_locale][s] + else: + strings[s] = translations[default_locale][s] + + +def translated(k): + """ + Returns a translated string. + """ + return strings[k] + +_ = translated diff --git a/share/locale/available_locales.json b/share/locale/available_locales.json new file mode 100644 index 0000000..a259b5c --- /dev/null +++ b/share/locale/available_locales.json @@ -0,0 +1,3 @@ +{ + "en": "English" +} diff --git a/share/locale/en.json b/share/locale/en.json index e69de29..97b841e 100644 --- a/share/locale/en.json +++ b/share/locale/en.json @@ -0,0 +1,10 @@ +{ + "smtp_links_subject": "[GetTor] Links for your request", + "smtp_mirrors_subject": "[GetTor] Mirrors", + "smtp_help_subject": "[GetTor] Help", + "smtp_unsupported_locale_subject": "[GetTor] Unsupported locale", + "smtp_unsupported_locale_msg": "The locale you requested '%s' is not supported.", + "smtp_vlinks_msg": "You requested Tor Browser for %s.\n \nYou will need only one of the links below to download the bundle. If a link does not work for you, try the next one.\n \n%s\n \n \n--\nGetTor", + "smtp_mirrors_msg": "Hi! this is the GetTor robot.\n \nThank you for your request. Attached to this email you will find\nan updated list of mirrors of Tor Project's website.", + "smtp_help_msg": "Hi! This is the GetTor robot. I am here to help you download the\nlatest version of Tor Browser.\n \nPlease reply to this message with one of the options below:\n \nwindows\nlinux\nosx\nmirrors\n \nI will then send you the download instructions.\n \nIf you are unsure, just send a blank reply to this message." +} diff --git a/share/version.txt b/share/version.txt new file mode 100644 index 0000000..aa8add4 --- /dev/null +++ b/share/version.txt @@ -0,0 +1 @@ +2.0.dev1 |
