firewall.py: don't die if a given sysctl doesn't exist.

Instead, get a list of known sysctls in the interesting prefix (net.inet.ip)
and check if there's an entry in the list for each sysctl we want to change.
If there isn't, then don't try to change it.

This fixes a problem with FreeBSD, which doesn't have
net.inet.ip.scopedroute but also doesn't need it.  Probably also fixes MacOS
10.5, which probably didn't have that either, but I don't know for sure.

Reported by Ed Maste.

README.md

@@ -1,14 +1,10 @@
 sshuttle: where transparent proxy meets VPN meets ssh
 =====================================================
 
-I just spent an afternoon working on a new kind of VPN.  You can get
-the first release, <a href="http://github.com/apenwarr/sshuttle">sshuttle
-0.10, on github</a>.
-
 As far as I know, sshuttle is the only program that solves the following
 common case:
 
- - Your client machine (or router) is Linux.
+ - Your client machine (or router) is Linux, FreeBSD, or MacOS.
 
  - You have access to a remote network via ssh.
 
@@ -53,10 +49,8 @@ This is how you use it:
 -----------------------
 
  - <tt>git clone git://github.com/apenwarr/sshuttle</tt>
-    on your client and server machines. The server can be
+    on your client machine. You'll need root or sudo
-    any ssh server with python available; the client must
+    access, and python needs to be installed.
-    be Linux with iptables, and you'll need root or sudo
-    access.
 
  - <tt>./sshuttle -r username@sshserver 0.0.0.0/0 -vv</tt>
 
@@ -161,3 +155,6 @@ later.  You're welcome.
 --
 Avery Pennarun <apenwarr@gmail.com>
 
+Mailing list:
+   Subscribe by sending a message to <sshuttle+subscribe@googlegroups.com>
+   List archives are at: http://groups.google.com/group/sshuttle

assembler.py

@@ -7,7 +7,7 @@ while 1:
     if name:
         nbytes = int(sys.stdin.readline())
         if verbosity >= 2:
-            sys.stderr.write('remote assembling %r (%d bytes)\n'
+            sys.stderr.write('server: assembling %r (%d bytes)\n'
                              % (name, nbytes))
         content = z.decompress(sys.stdin.read(nbytes))
         exec compile(content, name, "exec")

client.py

@@ -1,8 +1,10 @@
-import struct, socket, select, subprocess, errno
+import struct, socket, select, errno, re
+import compat.ssubprocess as ssubprocess
 import helpers, ssnet, ssh
 from ssnet import SockWrapper, Handler, Proxy, Mux, MuxWrapper
 from helpers import *
 
+
 def original_dst(sock):
     try:
         SO_ORIGINAL_DST = 80
@@ -20,13 +22,14 @@ def original_dst(sock):
 
 
 class FirewallClient:
-    def __init__(self, port, subnets):
+    def __init__(self, port, subnets_include, subnets_exclude):
         self.port = port
-        self.subnets = subnets
+        self.auto_nets = []
-        subnets_str = ['%s/%d' % (ip,width) for ip,width in subnets]
+        self.subnets_include = subnets_include
+        self.subnets_exclude = subnets_exclude
         argvbase = ([sys.argv[0]] +
                     ['-v'] * (helpers.verbose or 0) +
-                    ['--firewall', str(port)] + subnets_str)
+                    ['--firewall', str(port)])
         argv_tries = [
             ['sudo'] + argvbase,
             ['su', '-c', ' '.join(argvbase)],
@@ -44,7 +47,7 @@ class FirewallClient:
         e = None
         for argv in argv_tries:
             try:
-                self.p = subprocess.Popen(argv, stdout=s1, preexec_fn=setup)
+                self.p = ssubprocess.Popen(argv, stdout=s1, preexec_fn=setup)
                 e = None
                 break
             except OSError, e:
@@ -66,6 +69,11 @@ class FirewallClient:
             raise Fatal('%r returned %d' % (self.argv, rv))
 
     def start(self):
+        self.pfile.write('ROUTES\n')
+        for (ip,width) in self.subnets_include+self.auto_nets:
+            self.pfile.write('%d,0,%s\n' % (width, ip))
+        for (ip,width) in self.subnets_exclude:
+            self.pfile.write('%d,1,%s\n' % (width, ip))
         self.pfile.write('GO\n')
         self.pfile.flush()
         line = self.pfile.readline()
@@ -73,6 +81,12 @@ class FirewallClient:
         if line != 'STARTED\n':
             raise Fatal('%r expected STARTED, got %r' % (self.argv, line))
 
+    def sethostip(self, hostname, ip):
+        assert(not re.search(r'[^-\w]', hostname))
+        assert(not re.search(r'[^0-9.]', ip))
+        self.pfile.write('HOST %s,%s\n' % (hostname, ip))
+        self.pfile.flush()
+
     def done(self):
         self.pfile.close()
         rv = self.p.wait()
@@ -80,14 +94,21 @@ class FirewallClient:
             raise Fatal('cleanup: %r returned %d' % (self.argv, rv))
 
 
-def _main(listener, fw, use_server, remotename):
+def _main(listener, fw, use_server, remotename, python, seed_hosts, auto_nets):
     handlers = []
     if use_server:
         if helpers.verbose >= 1:
             helpers.logprefix = 'c : '
         else:
             helpers.logprefix = 'client: '
-        (serverproc, serversock) = ssh.connect(remotename)
+        debug1('connecting to server...\n')
+        try:
+            (serverproc, serversock) = ssh.connect(remotename, python)
+        except socket.error, e:
+            if e.errno == errno.EPIPE:
+                raise Fatal("failed to establish ssh session")
+            else:
+                raise
         mux = Mux(serversock, serversock)
         handlers.append(mux)
 
@@ -101,10 +122,32 @@ def _main(listener, fw, use_server, remotename):
         if initstring != expected:
             raise Fatal('expected server init string %r; got %r'
                             % (expected, initstring))
+        debug1('connected.\n')
+
+    def onroutes(routestr):
+        if auto_nets:
+            for line in routestr.strip().split('\n'):
+                (ip,width) = line.split(',', 1)
+                fw.auto_nets.append((ip,int(width)))
 
         # we definitely want to do this *after* starting ssh, or we might end
         # up intercepting the ssh connection!
+        #
+        # Moreover, now that we have the --auto-nets option, we have to wait
+        # for the server to send us that message anyway.  Even if we haven't
+        # set --auto-nets, we might as well wait for the message first, then
+        # ignore its contents.
+        mux.got_routes = None
         fw.start()
+    mux.got_routes = onroutes
+
+    def onhostlist(hostlist):
+        debug2('got host list: %r\n' % hostlist)
+        for line in hostlist.strip().split():
+            if line:
+                name,ip = line.split(',', 1)
+                fw.sethostip(name, ip)
+    mux.got_host_list = onhostlist
 
     def onaccept():
         sock,srcip = listener.accept()
@@ -124,32 +167,24 @@ def _main(listener, fw, use_server, remotename):
         handlers.append(Proxy(SockWrapper(sock, sock), outwrap))
     handlers.append(Handler([listener], onaccept))
 
+    if seed_hosts != None:
+        debug1('seed_hosts: %r\n' % seed_hosts)
+        mux.send(0, ssnet.CMD_HOST_REQ, '\n'.join(seed_hosts))
+    
     while 1:
         if use_server:
             rv = serverproc.poll()
             if rv:
                 raise Fatal('server died with error code %d' % rv)
         
-        r = set()
+        ssnet.runonce(handlers, mux)
-        w = set()
-        x = set()
-        handlers = filter(lambda s: s.ok, handlers)
-        for s in handlers:
-            s.pre_select(r,w,x)
-        debug2('Waiting: %d[%d,%d,%d]...\n' 
-            % (len(handlers), len(r), len(w), len(x)))
-        (r,w,x) = select.select(r,w,x)
-        #log('r=%r w=%r x=%r\n' % (r,w,x))
-        ready = set(r) | set(w) | set(x)
-        for s in handlers:
-            if s.socks & ready:
-                s.callback()
         if use_server:
             mux.callback()
             mux.check_fullness()
 
 
-def main(listenip, use_server, remotename, subnets):
+def main(listenip, use_server, remotename, python, seed_hosts, auto_nets,
+         subnets_include, subnets_exclude):
     debug1('Starting sshuttle proxy.\n')
     listener = socket.socket()
     listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -176,9 +211,10 @@ def main(listenip, use_server, remotename, subnets):
     listenip = listener.getsockname()
     debug1('Listening on %r.\n' % (listenip,))
 
-    fw = FirewallClient(listenip[1], subnets)
+    fw = FirewallClient(listenip[1], subnets_include, subnets_exclude)
     
     try:
-        return _main(listener, fw, use_server, remotename)
+        return _main(listener, fw, use_server, remotename,
+                     python, seed_hosts, auto_nets)
     finally:
         fw.done()

firewall.py

@@ -1,11 +1,12 @@
-import subprocess, re
+import re, errno
+import compat.ssubprocess as ssubprocess
 import helpers
 from helpers import *
 
 
 def ipt_chain_exists(name):
     argv = ['iptables', '-t', 'nat', '-nL']
-    p = subprocess.Popen(argv, stdout = subprocess.PIPE)
+    p = ssubprocess.Popen(argv, stdout = ssubprocess.PIPE)
     for line in p.stdout:
         if line.startswith('Chain %s ' % name):
             return True
@@ -17,7 +18,7 @@ def ipt_chain_exists(name):
 def ipt(*args):
     argv = ['iptables', '-t', 'nat'] + list(args)
     debug1('>> %s\n' % ' '.join(argv))
-    rv = subprocess.call(argv)
+    rv = ssubprocess.call(argv)
     if rv:
         raise Fatal('%r returned %d' % (argv, rv))
 
@@ -43,8 +44,17 @@ def do_iptables(port, subnets):
         ipt('-I', 'OUTPUT', '1', '-j', chain)
         ipt('-I', 'PREROUTING', '1', '-j', chain)
 
-        # create new subnet entries
+        # create new subnet entries.  Note that we're sorting in a very
-        for snet,swidth in subnets:
+        # particular order: we need to go from most-specific (largest swidth)
+        # to least-specific, and at any given level of specificity, we want
+        # excludes to come first.  That's why the columns are in such a non-
+        # intuitive order.
+        for swidth,sexclude,snet in sorted(subnets, reverse=True):
+            if sexclude:
+                ipt('-A', chain, '-j', 'RETURN',
+                    '--dest', '%s/%s' % (snet,swidth),
+                    '-p', 'tcp')
+            else:
                 ipt('-A', chain, '-j', 'REDIRECT',
                     '--dest', '%s/%s' % (snet,swidth),
                     '-p', 'tcp',
@@ -55,73 +65,98 @@ def do_iptables(port, subnets):
 
 def ipfw_rule_exists(n):
     argv = ['ipfw', 'list']
-    p = subprocess.Popen(argv, stdout = subprocess.PIPE)
+    p = ssubprocess.Popen(argv, stdout = ssubprocess.PIPE)
+    found = False
     for line in p.stdout:
         if line.startswith('%05d ' % n):
-            if line[5:].find('ipttl 42') < 0:
+            if not ('ipttl 42 setup keep-state' in line
+                    or ('skipto %d' % (n+1)) in line
+                    or 'check-state' in line):
+                log('non-sshuttle ipfw rule: %r\n' % line.strip())
                 raise Fatal('non-sshuttle ipfw rule #%d already exists!' % n)
-            return True
+            found = True
     rv = p.wait()
     if rv:
         raise Fatal('%r returned %d' % (argv, rv))
+    return found
 
 
-def sysctl_get(name):
+_oldctls = {}
-    argv = ['sysctl', '-n', name]
+def _fill_oldctls(prefix):
-    p = subprocess.Popen(argv, stdout = subprocess.PIPE)
+    argv = ['sysctl', prefix]
-    line = p.stdout.readline()
+    p = ssubprocess.Popen(argv, stdout = ssubprocess.PIPE)
+    for line in p.stdout:
+        assert(line[-1] == '\n')
+        (k,v) = line[:-1].split(': ', 1)
+        _oldctls[k] = v
     rv = p.wait()
     if rv:
         raise Fatal('%r returned %d' % (argv, rv))
     if not line:
         raise Fatal('%r returned no data' % (argv,))
-    assert(line[-1] == '\n')
-    return line[:-1]
 
 
 def _sysctl_set(name, val):
     argv = ['sysctl', '-w', '%s=%s' % (name, val)]
     debug1('>> %s\n' % ' '.join(argv))
-    rv = subprocess.call(argv, stdout = open('/dev/null', 'w'))
+    rv = ssubprocess.call(argv, stdout = open('/dev/null', 'w'))
 
 
-_oldctls = []
+_changedctls = []
 def sysctl_set(name, val):
-    oldval = sysctl_get(name)
+    PREFIX = 'net.inet.ip'
-    if str(val) != str(oldval):
+    assert(name.startswith(PREFIX + '.'))
-        _oldctls.append((name, oldval))
+    val = str(val)
+    if not _oldctls:
+        _fill_oldctls(PREFIX)
+    if not (name in _oldctls):
+        debug1('>> No such sysctl: %r\n' % name)
+        return
+    oldval = _oldctls[name]
+    if val != oldval:
+        _changedctls.append(name)
         return _sysctl_set(name, val)
     
 
 def ipfw(*args):
     argv = ['ipfw', '-q'] + list(args)
     debug1('>> %s\n' % ' '.join(argv))
-    rv = subprocess.call(argv)
+    rv = ssubprocess.call(argv)
     if rv:
         raise Fatal('%r returned %d' % (argv, rv))
 
 
 def do_ipfw(port, subnets):
     sport = str(port)
+    xsport = str(port+1)
 
     # cleanup any existing rules
     if ipfw_rule_exists(port):
-        ipfw('del', sport)
+        ipfw('delete', sport)
 
-    while _oldctls:
+    while _changedctls:
-        (name,oldval) = _oldctls.pop()
+        name = _changedctls.pop()
+        oldval = _oldctls[name]
         _sysctl_set(name, oldval)
 
     if subnets:
         sysctl_set('net.inet.ip.fw.enable', 1)
-        sysctl_set('net.inet.ip.forwarding', 1)
+        sysctl_set('net.inet.ip.scopedroute', 0)
+
+        ipfw('add', sport, 'check-state', 'ip',
+             'from', 'any', 'to', 'any')
         
         # create new subnet entries
-        for snet,swidth in subnets:
+        for swidth,sexclude,snet in sorted(subnets, reverse=True):
+            if sexclude:
+                ipfw('add', sport, 'skipto', xsport,
+                     'log', 'tcp',
+                     'from', 'any', 'to', '%s/%s' % (snet,swidth))
+            else:
                 ipfw('add', sport, 'fwd', '127.0.0.1,%d' % port,
                      'log', 'tcp',
                      'from', 'any', 'to', '%s/%s' % (snet,swidth),
-                 'not', 'ipttl', '42')
+                     'not', 'ipttl', '42', 'keep-state', 'setup')
 
 
 def program_exists(name):
@@ -131,6 +166,47 @@ def program_exists(name):
         if os.path.exists(fn):
             return not os.path.isdir(fn) and os.access(fn, os.X_OK)
 
+hostmap = {}
+def rewrite_etc_hosts(port):
+    HOSTSFILE='/etc/hosts'
+    BAKFILE='%s.sbak' % HOSTSFILE
+    APPEND='# sshuttle-firewall-%d AUTOCREATED' % port
+    old_content = ''
+    st = None
+    try:
+        old_content = open(HOSTSFILE).read()
+        st = os.stat(HOSTSFILE)
+    except IOError, e:
+        if e.errno == errno.ENOENT:
+            pass
+        else:
+            raise
+    if old_content.strip() and not os.path.exists(BAKFILE):
+        os.link(HOSTSFILE, BAKFILE)
+    tmpname = "%s.%d.tmp" % (HOSTSFILE, port)
+    f = open(tmpname, 'w')
+    for line in old_content.rstrip().split('\n'):
+        if line.find(APPEND) >= 0:
+            continue
+        f.write('%s\n' % line)
+    for (name,ip) in sorted(hostmap.items()):
+        f.write('%-30s %s\n' % ('%s %s' % (ip,name), APPEND))
+    f.close()
+
+    if st:
+        os.chown(tmpname, st.st_uid, st.st_gid)
+        os.chmod(tmpname, st.st_mode)
+    else:
+        os.chown(tmpname, 0, 0)
+        os.chmod(tmpname, 0644)
+    os.rename(tmpname, HOSTSFILE)
+
+
+def restore_etc_hosts(port):
+    global hostmap
+    hostmap = {}
+    rewrite_etc_hosts(port)
+
 
 # This is some voodoo for setting up the kernel's transparent
 # proxying stuff.  If subnets is empty, we just delete our sshuttle rules;
@@ -140,7 +216,7 @@ def program_exists(name):
 # exit.  In case that fails, it's not the end of the world; future runs will
 # supercede it in the transproxy list, at least, so the leftover rules
 # are hopefully harmless.
-def main(port, subnets):
+def main(port):
     assert(port > 0)
     assert(port <= 65535)
 
@@ -173,8 +249,22 @@ def main(port, subnets):
     line = sys.stdin.readline(128)
     if not line:
         return  # parent died; nothing to do
-    if line != 'GO\n':
+
-        raise Fatal('firewall: expected GO but got %r' % line)
+    subnets = []
+    if line != 'ROUTES\n':
+        raise Fatal('firewall: expected ROUTES but got %r' % line)
+    while 1:
+        line = sys.stdin.readline(128)
+        if not line:
+            raise Fatal('firewall: expected route but got %r' % line)
+        elif line == 'GO\n':
+            break
+        try:
+            (width,exclude,ip) = line.strip().split(',', 2)
+        except:
+            raise Fatal('firewall: expected route or GO but got %r' % line)
+        subnets.append((int(width), bool(int(exclude)), ip))
+        
     try:
         if line:
             debug1('firewall manager: starting transproxy.\n')
@@ -183,20 +273,28 @@ def main(port, subnets):
         
         try:
             sys.stdout.flush()
-            
-            # Now we wait until EOF or any other kind of exception.  We need
-            # to stay running so that we don't need a *second* password
-            # authentication at shutdown time - that cleanup is important!
-            while sys.stdin.readline(128):
-                pass
         except IOError:
             # the parent process died for some reason; he's surely been loud
             # enough, so no reason to report another error
             return
 
+        # Now we wait until EOF or any other kind of exception.  We need
+        # to stay running so that we don't need a *second* password
+        # authentication at shutdown time - that cleanup is important!
+        while 1:
+            line = sys.stdin.readline(128)
+            if line.startswith('HOST '):
+                (name,ip) = line[5:].strip().split(',', 1)
+                hostmap[name] = ip
+                rewrite_etc_hosts(port)
+            elif line:
+                raise Fatal('expected EOF, got %r' % line)
+            else:
+                break
     finally:
         try:
             debug1('firewall manager: undoing changes.\n')
         except:
             pass
         do_it(port, [])
+        restore_etc_hosts(port)

helpers.py

@@ -4,9 +4,14 @@ logprefix = ''
 verbose = 0
 
 def log(s):
+    try:
         sys.stdout.flush()
         sys.stderr.write(logprefix + s)
         sys.stderr.flush()
+    except IOError:
+        # this could happen if stderr gets forcibly disconnected, eg. because
+        # our tty closes.  That sucks, but it's no reason to abort the program.
+        pass
 
 def debug1(s):
     if verbose >= 1:
@@ -16,6 +21,17 @@ def debug2(s):
     if verbose >= 2:
         log(s)
 
+def debug3(s):
+    if verbose >= 3:
+        log(s)
+
 
 class Fatal(Exception):
     pass
+
+
+def list_contains_any(l, sub):
+    for i in sub:
+        if i in l:
+            return True
+    return False

hostwatch.py

@@ -0,0 +1,277 @@
+import time, socket, re, select, errno
+if not globals().get('skip_imports'):
+    import compat.ssubprocess as ssubprocess
+    import helpers
+    from helpers import *
+
+POLL_TIME = 60*15
+NETSTAT_POLL_TIME = 30
+CACHEFILE=os.path.expanduser('~/.sshuttle.hosts')
+
+
+_nmb_ok = True
+_smb_ok = True
+hostnames = {}
+queue = {}
+null = open('/dev/null', 'rb+')
+
+
+def _is_ip(s):
+    return re.match(r'\d+\.\d+\.\d+\.\d+$', s)
+
+
+def write_host_cache():
+    tmpname = '%s.%d.tmp' % (CACHEFILE, os.getpid())
+    try:
+        f = open(tmpname, 'wb')
+        for name,ip in sorted(hostnames.items()):
+            f.write('%s,%s\n' % (name, ip))
+        f.close()
+        os.rename(tmpname, CACHEFILE)
+    finally:
+        try:
+            os.unlink(tmpname)
+        except:
+            pass
+
+
+def read_host_cache():
+    try:
+        f = open(CACHEFILE)
+    except IOError, e:
+        if e.errno == errno.ENOENT:
+            return
+        else:
+            raise
+    for line in f:
+        words = line.strip().split(',')
+        if len(words) == 2:
+            (name,ip) = words
+            name = re.sub(r'[^-\w]', '-', name).strip()
+            ip = re.sub(r'[^0-9.]', '', ip).strip()
+            if name and ip:
+                found_host(name, ip)
+                
+
+def found_host(hostname, ip):
+    hostname = re.sub(r'\..*', '', hostname)
+    hostname = re.sub(r'[^-\w]', '_', hostname)
+    if (ip.startswith('127.') or ip.startswith('255.') 
+        or hostname == 'localhost'):
+        return
+    oldip = hostnames.get(hostname)
+    if oldip != ip:
+        hostnames[hostname] = ip
+        debug1('Found: %s: %s\n' % (hostname, ip))
+        sys.stdout.write('%s,%s\n' % (hostname, ip))
+        write_host_cache()
+
+
+def _check_etc_hosts():
+    debug2(' > hosts\n')
+    for line in open('/etc/hosts'):
+        line = re.sub(r'#.*', '', line)
+        words = line.strip().split()
+        if not words:
+            continue
+        ip = words[0]
+        names = words[1:]
+        if _is_ip(ip):
+            debug3('<    %s %r\n' % (ip, names))
+            for n in names:
+                check_host(n)
+                found_host(n, ip)
+
+
+def _check_revdns(ip):
+    debug2(' > rev: %s\n' % ip)
+    try:
+        r = socket.gethostbyaddr(ip)
+        debug3('<    %s\n' % r[0])
+        check_host(r[0])
+        found_host(r[0], ip)
+    except socket.herror, e:
+        pass
+
+
+def _check_dns(hostname):
+    debug2(' > dns: %s\n' % hostname)
+    try:
+        ip = socket.gethostbyname(hostname)
+        debug3('<    %s\n' % ip)
+        check_host(ip)
+        found_host(hostname, ip)
+    except socket.gaierror, e:
+        pass
+
+
+def _check_netstat():
+    debug2(' > netstat\n')
+    argv = ['netstat', '-n']
+    try:
+        p = ssubprocess.Popen(argv, stdout=ssubprocess.PIPE, stderr=null)
+        content = p.stdout.read()
+        p.wait()
+    except OSError, e:
+        log('%r failed: %r\n' % (argv, e))
+        return
+
+    for ip in re.findall(r'\d+\.\d+\.\d+\.\d+', content):
+        debug3('<    %s\n' % ip)
+        check_host(ip)
+        
+
+def _check_smb(hostname):
+    return
+    global _smb_ok
+    if not _smb_ok:
+        return
+    argv = ['smbclient', '-U', '%', '-L', hostname]
+    debug2(' > smb: %s\n' % hostname)
+    try:
+        p = ssubprocess.Popen(argv, stdout=ssubprocess.PIPE, stderr=null)
+        lines = p.stdout.readlines()
+        p.wait()
+    except OSError, e:
+        log('%r failed: %r\n' % (argv, e))
+        _smb_ok = False
+        return
+
+    lines.reverse()
+
+    # junk at top
+    while lines:
+        line = lines.pop().strip()
+        if re.match(r'Server\s+', line):
+            break
+
+    # server list section:
+    #    Server   Comment
+    #    ------   -------
+    while lines:
+        line = lines.pop().strip()
+        if not line or re.match(r'-+\s+-+', line):
+            continue
+        if re.match(r'Workgroup\s+Master', line):
+            break
+        words = line.split()
+        hostname = words[0].lower()
+        debug3('<    %s\n' % hostname)
+        check_host(hostname)
+
+    # workgroup list section:
+    #   Workgroup  Master
+    #   ---------  ------
+    while lines:
+        line = lines.pop().strip()
+        if re.match(r'-+\s+', line):
+            continue
+        if not line:
+            break
+        words = line.split()
+        (workgroup, hostname) = (words[0].lower(), words[1].lower())
+        debug3('<    group(%s) -> %s\n' % (workgroup, hostname))
+        check_host(hostname)
+        check_workgroup(workgroup)
+
+    if lines:
+        assert(0)
+
+
+def _check_nmb(hostname, is_workgroup, is_master):
+    return
+    global _nmb_ok
+    if not _nmb_ok:
+        return
+    argv = ['nmblookup'] + ['-M']*is_master + ['--', hostname]
+    debug2(' > n%d%d: %s\n' % (is_workgroup, is_master, hostname))
+    try:
+        p = ssubprocess.Popen(argv, stdout=ssubprocess.PIPE, stderr=null)
+        lines = p.stdout.readlines()
+        rv = p.wait()
+    except OSError, e:
+        log('%r failed: %r\n' % (argv, e))
+        _nmb_ok = False
+        return
+    if rv:
+        log('%r returned %d\n' % (argv, rv))
+        return
+    for line in lines:
+        m = re.match(r'(\d+\.\d+\.\d+\.\d+) (\w+)<\w\w>\n', line)
+        if m:
+            g = m.groups()
+            (ip, name) = (g[0], g[1].lower())
+            debug3('<    %s -> %s\n' % (name, ip))
+            if is_workgroup:
+                _enqueue(_check_smb, ip)
+            else:
+                found_host(name, ip)
+                check_host(name)
+
+
+def check_host(hostname):
+    if _is_ip(hostname):
+        _enqueue(_check_revdns, hostname)
+    else:
+        _enqueue(_check_dns, hostname)
+    _enqueue(_check_smb, hostname)
+    _enqueue(_check_nmb, hostname, False, False)
+
+
+def check_workgroup(hostname):
+    _enqueue(_check_nmb, hostname, True, False)
+    _enqueue(_check_nmb, hostname, True, True)
+
+
+def _enqueue(op, *args):
+    t = (op,args)
+    if queue.get(t) == None:
+        queue[t] = 0
+
+
+def _stdin_still_ok(timeout):
+    r,w,x = select.select([sys.stdin.fileno()], [], [], timeout)
+    if r:
+        b = os.read(sys.stdin.fileno(), 4096)
+        if not b:
+            return False
+    return True
+
+
+def hw_main(seed_hosts):
+    if helpers.verbose >= 2:
+        helpers.logprefix = 'HH: '
+    else:
+        helpers.logprefix = 'hostwatch: '
+
+    read_host_cache()
+        
+    _enqueue(_check_etc_hosts)
+    _enqueue(_check_netstat)
+    check_host('localhost')
+    check_host(socket.gethostname())
+    check_workgroup('workgroup')
+    check_workgroup('-')
+    for h in seed_hosts:
+        check_host(h)
+
+    while 1:
+        now = time.time()
+        for t,last_polled in queue.items():
+            (op,args) = t
+            if not _stdin_still_ok(0):
+                break
+            maxtime = POLL_TIME
+            if op == _check_netstat:
+                maxtime = NETSTAT_POLL_TIME
+            if now - last_polled > maxtime:
+                queue[t] = time.time()
+                op(*args)
+            try:
+                sys.stdout.flush()
+            except IOError:
+                break
+                
+        # FIXME: use a smarter timeout based on oldest last_polled
+        if not _stdin_still_ok(1):
+            break

main.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 import sys, os, re
-import helpers, options, client, server, firewall
+import helpers, options, client, server, firewall, hostwatch
 from helpers import *
 
 
@@ -49,12 +49,18 @@ sshuttle [-l [ip:]port] [-r [username@]sshserver[:port]] <subnets...>
 sshuttle --firewall <port> <subnets...>
 sshuttle --server
 --
-l,listen=  transproxy to this ip address and port number [default=0]
+l,listen=  transproxy to this ip address and port number [0.0.0.0:0]
+H,auto-hosts scan for remote hostnames and update local /etc/hosts
+N,auto-nets  automatically determine subnets to route
+python= specify the name/path of the python interpreter on the remote server [python]
 r,remote=  ssh hostname (and optional username) of remote sshuttle server
+x,exclude= exclude this subnet (can be used more than once)
 v,verbose  increase debug message verbosity
+seed-hosts= with -H, use these hostnames for initial scan (comma-separated)
 noserver   don't use a separate server process (mostly for debugging)
-server     [internal use only]
+server     (internal use only)
-firewall   [internal use only]
+firewall   (internal use only)
+hostwatch  (internal use only)
 """
 o = options.Options('sshuttle', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -63,22 +69,42 @@ helpers.verbose = opt.verbose
 
 try:
     if opt.server:
+        if len(extra) != 0:
+            o.fatal('no arguments expected')
         sys.exit(server.main())
     elif opt.firewall:
-        if len(extra) < 1:
+        if len(extra) != 1:
-            o.fatal('at least one argument expected')
+            o.fatal('exactly one argument expected')
-        sys.exit(firewall.main(int(extra[0]),
+        sys.exit(firewall.main(int(extra[0])))
-                               parse_subnets(extra[1:])))
+    elif opt.hostwatch:
+        sys.exit(hostwatch.hw_main(extra))
     else:
-        if len(extra) < 1:
+        if len(extra) < 1 and not opt.auto_nets:
-            o.fatal('at least one subnet expected')
+            o.fatal('at least one subnet (or -N) expected')
+        includes = extra
+        excludes = ['127.0.0.0/8']
+        for k,v in flags:
+            if k in ('-x','--exclude'):
+                excludes.append(v)
         remotename = opt.remote
         if remotename == '' or remotename == '-':
             remotename = None
+        if opt.seed_hosts and not opt.auto_hosts:
+            o.fatal('--seed-hosts only works if you also use -H')
+        if opt.seed_hosts:
+            sh = re.split(r'[\s,]+', (opt.seed_hosts or "").strip())
+        elif opt.auto_hosts:
+            sh = []
+        else:
+            sh = None
         sys.exit(client.main(parse_ipport(opt.listen or '0.0.0.0:0'),
                              not opt.noserver,
                              remotename,
-                             parse_subnets(extra)))
+                             (opt.python or "python"),
+                             sh,
+                             opt.auto_nets,
+                             parse_subnets(includes),
+                             parse_subnets(excludes)))
 except Fatal, e:
     log('fatal: %s\n' % e)
     sys.exit(99)

options.py

@@ -1,28 +1,92 @@
-import sys, textwrap, getopt, re
+"""Command-line options parser.
+With the help of an options spec string, easily parse command-line options.
+"""
+import sys, os, textwrap, getopt, re, struct
 
 class OptDict:
     def __init__(self):
         self._opts = {}
 
     def __setitem__(self, k, v):
+        if k.startswith('no-') or k.startswith('no_'):
+            k = k[3:]
+            v = not v
         self._opts[k] = v
 
     def __getitem__(self, k):
+        if k.startswith('no-') or k.startswith('no_'):
+            return not self._opts[k[3:]]
         return self._opts[k]
 
     def __getattr__(self, k):
         return self[k]
 
 
+def _default_onabort(msg):
+    sys.exit(97)
+
+
+def _intify(v):
+    try:
+        vv = int(v or '')
+        if str(vv) == v:
+            return vv
+    except ValueError:
+        pass
+    return v
+
+
+def _atoi(v):
+    try:
+        return int(v or 0)
+    except ValueError:
+        return 0
+
+
+def _remove_negative_kv(k, v):
+    if k.startswith('no-') or k.startswith('no_'):
+        return k[3:], not v
+    return k,v
+
+def _remove_negative_k(k):
+    return _remove_negative_kv(k, None)[0]
+
+
+def _tty_width():
+    s = struct.pack("HHHH", 0, 0, 0, 0)
+    try:
+        import fcntl, termios
+        s = fcntl.ioctl(sys.stderr.fileno(), termios.TIOCGWINSZ, s)
+    except (IOError, ImportError):
+        return _atoi(os.environ.get('WIDTH')) or 70
+    (ysize,xsize,ypix,xpix) = struct.unpack('HHHH', s)
+    return xsize
+
+
 class Options:
-    def __init__(self, exe, optspec, optfunc=getopt.gnu_getopt):
+    """Option parser.
+    When constructed, two strings are mandatory. The first one is the command
+    name showed before error messages. The second one is a string called an
+    optspec that specifies the synopsis and option flags and their description.
+    For more information about optspecs, consult the bup-options(1) man page.
+
+    Two optional arguments specify an alternative parsing function and an
+    alternative behaviour on abort (after having output the usage string).
+
+    By default, the parser function is getopt.gnu_getopt, and the abort
+    behaviour is to exit the program.
+    """
+    def __init__(self, exe, optspec, optfunc=getopt.gnu_getopt,
+                 onabort=_default_onabort):
         self.exe = exe
         self.optspec = optspec
+        self._onabort = onabort
         self.optfunc = optfunc
         self._aliases = {}
         self._shortopts = 'h?'
         self._longopts = ['help']
         self._hasparms = {}
+        self._defaults = {}
         self._usagestr = self._gen_usage()
 
     def _gen_usage(self):
@@ -36,10 +100,13 @@ class Options:
             out.append('%s: %s\n' % (first_syn and 'usage' or '   or', l))
             first_syn = False
         out.append('\n')
+        last_was_option = False
         while lines:
             l = lines.pop()
             if l.startswith(' '):
-                out.append('\n%s\n' % l.lstrip())
+                out.append('%s%s\n' % (last_was_option and '\n' or '',
+                                       l.lstrip()))
+                last_was_option = False
             elif l:
                 (flags, extra) = l.split(' ', 1)
                 extra = extra.strip()
@@ -48,18 +115,24 @@ class Options:
                     has_parm = 1
                 else:
                     has_parm = 0
+                g = re.search(r'\[([^\]]*)\]$', extra)
+                if g:
+                    defval = g.group(1)
+                else:
+                    defval = None
                 flagl = flags.split(',')
                 flagl_nice = []
                 for f in flagl:
-                    f_nice = re.sub(r'\W', '_', f)
+                    f,dvi = _remove_negative_kv(f, _intify(defval))
-                    self._aliases[f] = flagl[0]
+                    self._aliases[f] = _remove_negative_k(flagl[0])
-                    self._aliases[f_nice] = flagl[0]
                     self._hasparms[f] = has_parm
+                    self._defaults[f] = dvi
                     if len(f) == 1:
                         self._shortopts += f + (has_parm and ':' or '')
                         flagl_nice.append('-' + f)
                     else:
-                        assert(not f.startswith('no-')) # supported implicitly
+                        f_nice = re.sub(r'\W', '_', f)
+                        self._aliases[f_nice] = _remove_negative_k(flagl[0])
                         self._longopts.append(f + (has_parm and '=' or ''))
                         self._longopts.append('no-' + f)
                         flagl_nice.append('--' + f)
@@ -67,52 +140,62 @@ class Options:
                 if has_parm:
                     flags_nice += ' ...'
                 prefix = '    %-20s  ' % flags_nice
-                argtext = '\n'.join(textwrap.wrap(extra, width=70,
+                argtext = '\n'.join(textwrap.wrap(extra, width=_tty_width(),
                                                 initial_indent=prefix,
                                                 subsequent_indent=' '*28))
                 out.append(argtext + '\n')
+                last_was_option = True
             else:
                 out.append('\n')
+                last_was_option = False
         return ''.join(out).rstrip() + '\n'
 
-    def usage(self):
+    def usage(self, msg=""):
+        """Print usage string to stderr and abort."""
         sys.stderr.write(self._usagestr)
-        sys.exit(97)
+        e = self._onabort and self._onabort(msg) or None
+        if e:
+            raise e
 
     def fatal(self, s):
-        sys.stderr.write('error: %s\n' % s)
+        """Print an error message to stderr and abort with usage string."""
-        return self.usage()
+        msg = 'error: %s\n' % s
+        sys.stderr.write(msg)
+        return self.usage(msg)
 
     def parse(self, args):
+        """Parse a list of arguments and return (options, flags, extra).
+
+        In the returned tuple, "options" is an OptDict with known options,
+        "flags" is a list of option flags that were used on the command-line,
+        and "extra" is a list of positional arguments.
+        """
         try:
             (flags,extra) = self.optfunc(args, self._shortopts, self._longopts)
         except getopt.GetoptError, e:
             self.fatal(e)
 
         opt = OptDict()
-        for f in self._aliases.values():
+
-            opt[f] = None
+        for k,v in self._defaults.iteritems():
+            k = self._aliases[k]
+            opt[k] = v
+
         for (k,v) in flags:
-            while k.startswith('-'):
+            k = k.lstrip('-')
-                k = k[1:]
+            if k in ('h', '?', 'help'):
-            if k in ['h', '?', 'help']:
                 self.usage()
             if k.startswith('no-'):
                 k = self._aliases[k[3:]]
-                opt[k] = None
+                v = 0
             else:
                 k = self._aliases[k]
                 if not self._hasparms[k]:
                     assert(v == '')
-                    opt[k] = (opt._opts.get(k) or 0) + 1
+                    v = (opt._opts.get(k) or 0) + 1
                 else:
-                    try:
+                    v = _intify(v)
-                        vv = int(v)
-                        if str(vv) == v:
-                            v = vv
-                    except ValueError:
-                        pass
             opt[k] = v
-        for (f1,f2) in self._aliases.items():
+        for (f1,f2) in self._aliases.iteritems():
-            opt[f1] = opt[f2]
+            opt[f1] = opt._opts.get(f2)
         return (opt,flags,extra)

server.py

@@ -1,16 +1,122 @@
-import struct, socket, select
+import re, struct, socket, select, traceback
 if not globals().get('skip_imports'):
-    import ssnet, helpers
+    import ssnet, helpers, hostwatch
+    import compat.ssubprocess as ssubprocess
     from ssnet import SockWrapper, Handler, Proxy, Mux, MuxWrapper
     from helpers import *
 
 
+def _ipmatch(ipstr):
+    if ipstr == 'default':
+        ipstr = '0.0.0.0/0'
+    m = re.match(r'^(\d+(\.\d+(\.\d+(\.\d+)?)?)?)(?:/(\d+))?$', ipstr)
+    if m:
+        g = m.groups()
+        ips = g[0]
+        width = int(g[4] or 32)
+        if g[1] == None:
+            ips += '.0.0.0'
+            width = min(width, 8)
+        elif g[2] == None:
+            ips += '.0.0'
+            width = min(width, 16)
+        elif g[3] == None:
+            ips += '.0'
+            width = min(width, 24)
+        return (struct.unpack('!I', socket.inet_aton(ips))[0], width)
+
+
+def _ipstr(ip, width):
+    if width >= 32:
+        return ip
+    else:
+        return "%s/%d" % (ip, width)
+
+
+def _maskbits(netmask):
+    if not netmask:
+        return 32
+    for i in range(32):
+        if netmask[0] & _shl(1, i):
+            return 32-i
+    return 0
+    
+    
+def _shl(n, bits):
+    return n * int(2**bits)
+
+
+def _list_routes():
+    argv = ['netstat', '-rn']
+    p = ssubprocess.Popen(argv, stdout=ssubprocess.PIPE)
+    routes = []
+    for line in p.stdout:
+        cols = re.split(r'\s+', line)
+        ipw = _ipmatch(cols[0])
+        if not ipw:
+            continue  # some lines won't be parseable; never mind
+        maskw = _ipmatch(cols[2])  # linux only
+        mask = _maskbits(maskw)   # returns 32 if maskw is null
+        width = min(ipw[1], mask)
+        ip = ipw[0] & _shl(_shl(1, width) - 1, 32-width)
+        routes.append((socket.inet_ntoa(struct.pack('!I', ip)), width))
+    rv = p.wait()
+    if rv != 0:
+        log('WARNING: %r returned %d\n' % (argv, rv))
+        log('WARNING: That prevents --auto-nets from working.\n')
+    return routes
+
+
+def list_routes():
+    for (ip,width) in _list_routes():
+        if not ip.startswith('0.') and not ip.startswith('127.'):
+            yield (ip,width)
+
+
+def _exc_dump():
+    exc_info = sys.exc_info()
+    return ''.join(traceback.format_exception(*exc_info))
+
+
+def start_hostwatch(seed_hosts):
+    s1,s2 = socket.socketpair()
+    pid = os.fork()
+    if not pid:
+        # child
+        rv = 99
+        try:
+            try:
+                s2.close()
+                os.dup2(s1.fileno(), 1)
+                os.dup2(s1.fileno(), 0)
+                s1.close()
+                rv = hostwatch.hw_main(seed_hosts) or 0
+            except Exception, e:
+                log('%s\n' % _exc_dump())
+                rv = 98
+        finally:
+            os._exit(rv)
+    s1.close()
+    return pid,s2
+
+
+class Hostwatch:
+    def __init__(self):
+        self.pid = 0
+        self.sock = None
+
+
 def main():
     if helpers.verbose >= 1:
         helpers.logprefix = ' s: '
     else:
         helpers.logprefix = 'server: '
 
+    routes = list(list_routes())
+    debug1('available routes:\n')
+    for r in routes:
+        debug1('  %s/%d\n' % r)
+        
     # synchronization header
     sys.stdout.write('SSHUTTLE0001')
     sys.stdout.flush()
@@ -21,31 +127,49 @@ def main():
               socket.fromfd(sys.stdout.fileno(),
                             socket.AF_INET, socket.SOCK_STREAM))
     handlers.append(mux)
+    routepkt = ''
+    for r in routes:
+        routepkt += '%s,%d\n' % r
+    mux.send(0, ssnet.CMD_ROUTES, routepkt)
+
+    hw = Hostwatch()
+    hw.leftover = ''
+        
+    def hostwatch_ready():
+        assert(hw.pid)
+        content = hw.sock.recv(4096)
+        if content:
+            lines = (hw.leftover + content).split('\n')
+            if lines[-1]:
+                # no terminating newline: entry isn't complete yet!
+                hw.leftover = lines.pop()
+                lines.append('')
+            else:
+                hw.leftover = ''
+            mux.send(0, ssnet.CMD_HOST_LIST, '\n'.join(lines))
+        else:
+            raise Fatal('hostwatch process died')
+
+    def got_host_req(data):
+        if not hw.pid:
+            (hw.pid,hw.sock) = start_hostwatch(data.strip().split())
+            handlers.append(Handler(socks = [hw.sock],
+                                    callback = hostwatch_ready))
+    mux.got_host_req = got_host_req
 
     def new_channel(channel, data):
         (dstip,dstport) = data.split(',', 1)
         dstport = int(dstport)
         outwrap = ssnet.connect_dst(dstip,dstport)
         handlers.append(Proxy(MuxWrapper(mux, channel), outwrap))
-    
     mux.new_channel = new_channel
 
     while mux.ok:
-        r = set()
+        if hw.pid:
-        w = set()
+            (rpid, rv) = os.waitpid(hw.pid, os.WNOHANG)
-        x = set()
+            if rpid:
-        handlers = filter(lambda s: s.ok, handlers)
+                raise Fatal('hostwatch exited unexpectedly: code 0x%04x\n' % rv)
-        for s in handlers:
+        
-            s.pre_select(r,w,x)
+        ssnet.runonce(handlers, mux)
-        debug2('Waiting: %d[%d,%d,%d] (fullness=%d/%d)...\n' 
-               % (len(handlers), len(r), len(w), len(x),
-                  mux.fullness, mux.too_full))
-        (r,w,x) = select.select(r,w,x)
-        #log('r=%r w=%r x=%r\n' % (r,w,x))
-        ready = set(r) | set(w) | set(x)
-        for s in handlers:
-            #debug2('check: %r: %r\n' % (s, s.socks & ready))
-            if s.socks & ready:
-                s.callback()
         mux.check_fullness()
         mux.callback()

ssh.py

@@ -1,21 +1,27 @@
-import sys, os, re, subprocess, socket, zlib
+import sys, os, re, socket, zlib
+import compat.ssubprocess as ssubprocess
 import helpers
 from helpers import *
 
 
 def readfile(name):
     basedir = os.path.dirname(os.path.abspath(sys.argv[0]))
-    fullname = os.path.join(basedir, name)
+    path = [basedir] + sys.path
+    for d in path:
+        fullname = os.path.join(d, name)
+        if os.path.exists(fullname):
             return open(fullname, 'rb').read()
+    raise Exception("can't find file %r in any of %r" % (name, path))
 
 
 def empackage(z, filename):
+    (path,basename) = os.path.split(filename)
     content = z.compress(readfile(filename))
     content += z.flush(zlib.Z_SYNC_FLUSH)
-    return '%s\n%d\n%s' % (filename,len(content), content)
+    return '%s\n%d\n%s' % (basename,len(content), content)
 
 
-def connect(rhostport):
+def connect(rhostport, python):
     main_exe = sys.argv[0]
     l = (rhostport or '').split(':', 1)
     rhost = l[0]
@@ -29,7 +35,9 @@ def connect(rhostport):
     z = zlib.compressobj(1)
     content = readfile('assembler.py')
     content2 = (empackage(z, 'helpers.py') +
+                empackage(z, 'compat/ssubprocess.py') +
                 empackage(z, 'ssnet.py') +
+                empackage(z, 'hostwatch.py') +
                 empackage(z, 'server.py') +
                 "\n")
     
@@ -43,18 +51,17 @@ def connect(rhostport):
 
         
     if not rhost:
-        argv = ['python', '-c', pyscript]
+        argv = [python, '-c', pyscript]
     else:
-        argv = ['ssh'] + portl + [rhost, '--', "python -c '%s'" % pyscript]
+        argv = ['ssh'] + portl + [rhost, '--', "'%s' -c '%s'" % (python, pyscript)]
     (s1,s2) = socket.socketpair()
     def setup():
         # runs in the child process
         s2.close()
-        os.setsid()
     s1a,s1b = os.dup(s1.fileno()), os.dup(s1.fileno())
     s1.close()
     debug2('executing: %r\n' % argv)
-    p = subprocess.Popen(argv, stdin=s1a, stdout=s1b, preexec_fn=setup,
+    p = ssubprocess.Popen(argv, stdin=s1a, stdout=s1b, preexec_fn=setup,
                          close_fds=True)
     os.close(s1a)
     os.close(s1b)

ssnet.py

@@ -2,6 +2,12 @@ import struct, socket, errno, select
 if not globals().get('skip_imports'):
     from helpers import *
     
+# these don't exist in the socket module in python 2.3!
+SHUT_RD = 0
+SHUT_WR = 1
+SHUT_RDWR = 2
+
+
 HDR_LEN = 8
 
 
@@ -12,6 +18,9 @@ CMD_CONNECT = 0x4203
 CMD_CLOSE = 0x4204
 CMD_EOF = 0x4205
 CMD_DATA = 0x4206
+CMD_ROUTES = 0x4207
+CMD_HOST_REQ = 0x4208
+CMD_HOST_LIST = 0x4209
 
 cmd_to_name = {
     CMD_EXIT: 'EXIT',
@@ -21,10 +30,29 @@ cmd_to_name = {
     CMD_CLOSE: 'CLOSE',
     CMD_EOF: 'EOF',
     CMD_DATA: 'DATA',
+    CMD_ROUTES: 'ROUTES',
+    CMD_HOST_REQ: 'HOST_REQ',
+    CMD_HOST_LIST: 'HOST_LIST',
 }
     
 
 
+def _add(l, elem):
+    if not elem in l:
+        l.append(elem)
+
+
+def _fds(l):
+    out = []
+    for i in l:
+        try:
+            out.append(i.fileno())
+        except AttributeError:
+            out.append(i)
+    out.sort()
+    return out
+
+
 def _nb_clean(func, *args):
     try:
         return func(*args)
@@ -32,6 +60,7 @@ def _nb_clean(func, *args):
         if e.errno not in (errno.EWOULDBLOCK, errno.EAGAIN):
             raise
         else:
+            debug3('%s: err was: %s\n' % (func.__name__, e))
             return None
 
 
@@ -63,27 +92,38 @@ class SockWrapper:
             debug1('%r: error was: %r\n' % (self, self.exc))
 
     def __repr__(self):
-        return 'SW:%s' % (self.peername,)
+        if self.rsock == self.wsock:
+            fds = '#%d' % self.rsock.fileno()
+        else:
+            fds = '#%d,%d' % (self.rsock.fileno(), self.wsock.fileno())
+        return 'SW%s:%s' % (fds, self.peername)
 
     def seterr(self, e):
         if not self.exc:
             self.exc = e
 
     def try_connect(self):
+        if self.connect_to and self.shut_write:
+            self.noread()
+            self.connect_to = None
         if not self.connect_to:
             return  # already connected
         self.rsock.setblocking(False)
+        debug3('%r: trying connect to %r\n' % (self, self.connect_to))
         try:
             self.rsock.connect(self.connect_to)
             # connected successfully (Linux)
             self.connect_to = None
         except socket.error, e:
+            debug3('%r: connect result: %r\n' % (self, e))
             if e.args[0] in [errno.EINPROGRESS, errno.EALREADY]:
                 pass  # not connected yet
             elif e.args[0] == errno.EISCONN:
                 # connected successfully (BSD)
                 self.connect_to = None
-            elif e.args[0] in [errno.ECONNREFUSED, errno.ETIMEDOUT]:
+            elif e.args[0] in [errno.ECONNREFUSED, errno.ETIMEDOUT,
+                               errno.EHOSTUNREACH, errno.ENETUNREACH,
+                               errno.EACCES, errno.EPERM]:
                 # a "normal" kind of error
                 self.connect_to = None
                 self.seterr(e)
@@ -94,14 +134,14 @@ class SockWrapper:
         if not self.shut_read:
             debug2('%r: done reading\n' % self)
             self.shut_read = True
-            #self.rsock.shutdown(socket.SHUT_RD)  # doesn't do anything anyway
+            #self.rsock.shutdown(SHUT_RD)  # doesn't do anything anyway
         
     def nowrite(self):
         if not self.shut_write:
             debug2('%r: done writing\n' % self)
             self.shut_write = True
             try:
-                self.wsock.shutdown(socket.SHUT_WR)
+                self.wsock.shutdown(SHUT_WR)
             except socket.error, e:
                 self.seterr(e)
 
@@ -151,7 +191,7 @@ class SockWrapper:
             wrote = outwrap.write(self.buf[0])
             self.buf[0] = self.buf[0][wrote:]
         while self.buf and not self.buf[0]:
-            self.buf[0:1] = []
+            self.buf.pop(0)
         if not self.buf and self.shut_read:
             outwrap.nowrite()
 
@@ -159,12 +199,13 @@ class SockWrapper:
 class Handler:
     def __init__(self, socks = None, callback = None):
         self.ok = True
-        self.socks = set(socks or [])
+        self.socks = socks or []
         if callback:
             self.callback = callback
 
     def pre_select(self, r, w, x):
-        r |= self.socks
+        for i in self.socks:
+            _add(r, i)
 
     def callback(self):
         log('--no callback defined-- %r\n' % self)
@@ -173,7 +214,7 @@ class Handler:
             v = s.recv(4096)
             if not v:
                 log('--closed-- %r\n' % self)
-                self.socks = set()
+                self.socks = []
                 self.ok = False
 
 
@@ -186,20 +227,20 @@ class Proxy(Handler):
 
     def pre_select(self, r, w, x):
         if self.wrap1.connect_to:
-            w.add(self.wrap1.rsock)
+            _add(w, self.wrap1.rsock)
         elif self.wrap1.buf:
             if not self.wrap2.too_full():
-                w.add(self.wrap2.wsock)
+                _add(w, self.wrap2.wsock)
         elif not self.wrap1.shut_read:
-            r.add(self.wrap1.rsock)
+            _add(r, self.wrap1.rsock)
 
         if self.wrap2.connect_to:
-            w.add(self.wrap2.rsock)
+            _add(w, self.wrap2.rsock)
         elif self.wrap2.buf:
             if not self.wrap1.too_full():
-                w.add(self.wrap1.wsock)
+                _add(w, self.wrap1.wsock)
         elif not self.wrap2.shut_read:
-            r.add(self.wrap2.rsock)
+            _add(r, self.wrap2.rsock)
 
     def callback(self):
         self.wrap1.try_connect()
@@ -208,6 +249,12 @@ class Proxy(Handler):
         self.wrap2.fill()
         self.wrap1.copy_to(self.wrap2)
         self.wrap2.copy_to(self.wrap1)
+        if self.wrap1.buf and self.wrap2.shut_write:
+            self.wrap1.buf = []
+            self.wrap1.noread()
+        if self.wrap2.buf and self.wrap1.shut_write:
+            self.wrap2.buf = []
+            self.wrap2.noread()
         if (self.wrap1.shut_read and self.wrap2.shut_read and
             not self.wrap1.buf and not self.wrap2.buf):
             self.ok = False
@@ -218,7 +265,8 @@ class Mux(Handler):
         Handler.__init__(self, [rsock, wsock])
         self.rsock = rsock
         self.wsock = wsock
-        self.new_channel = None
+        self.new_channel = self.got_routes = None
+        self.got_host_req = self.got_host_list = None
         self.channels = {}
         self.chani = 0
         self.want = 0
@@ -238,7 +286,10 @@ class Mux(Handler):
                 return self.chani
 
     def amount_queued(self):
-        return sum(len(b) for b in self.outbuf)
+        total = 0
+        for b in self.outbuf:
+            total += len(b)
+        return total
             
     def check_fullness(self):
         if self.fullness > 32768:
@@ -257,12 +308,13 @@ class Mux(Handler):
         p = struct.pack('!ccHHH', 'S', 'S', channel, cmd, len(data)) + data
         self.outbuf.append(p)
         debug2(' > channel=%d cmd=%s len=%d (fullness=%d)\n'
-               % (channel, cmd_to_name[cmd], len(data), self.fullness))
+               % (channel, cmd_to_name.get(cmd,hex(cmd)),
+                  len(data), self.fullness))
         self.fullness += len(data)
 
     def got_packet(self, channel, cmd, data):
         debug2('<  channel=%d cmd=%s len=%d\n' 
-               % (channel, cmd_to_name[cmd], len(data)))
+               % (channel, cmd_to_name.get(cmd,hex(cmd)), len(data)))
         if cmd == CMD_PING:
             self.send(0, CMD_PONG, data)
         elif cmd == CMD_PONG:
@@ -275,6 +327,21 @@ class Mux(Handler):
             assert(not self.channels.get(channel))
             if self.new_channel:
                 self.new_channel(channel, data)
+        elif cmd == CMD_ROUTES:
+            if self.got_routes:
+                self.got_routes(data)
+            else:
+                raise Exception('got CMD_ROUTES without got_routes?')
+        elif cmd == CMD_HOST_REQ:
+            if self.got_host_req:
+                self.got_host_req(data)
+            else:
+                raise Exception('got CMD_HOST_REQ without got_host_req?')
+        elif cmd == CMD_HOST_LIST:
+            if self.got_host_list:
+                self.got_host_list(data)
+            else:
+                raise Exception('got CMD_HOST_LIST without got_host_list?')
         else:
             callback = self.channels[channel]
             callback(cmd, data)
@@ -283,7 +350,7 @@ class Mux(Handler):
         self.wsock.setblocking(False)
         if self.outbuf and self.outbuf[0]:
             wrote = _nb_clean(os.write, self.wsock.fileno(), self.outbuf[0])
-            debug2('mux wrote: %d/%d\n' % (wrote, len(self.outbuf[0])))
+            debug2('mux wrote: %r/%d\n' % (wrote, len(self.outbuf[0])))
             if wrote:
                 self.outbuf[0] = self.outbuf[0][wrote:]
         while self.outbuf and not self.outbuf[0]:
@@ -321,9 +388,9 @@ class Mux(Handler):
                 break
 
     def pre_select(self, r, w, x):
-        r.add(self.rsock)
+        _add(r, self.rsock)
         if self.outbuf:
-            w.add(self.wsock)
+            _add(w, self.wsock)
 
     def callback(self):
         (r,w,x) = select.select([self.rsock], [self.wsock], [], 0)
@@ -395,3 +462,28 @@ def connect_dst(ip, port):
     return SockWrapper(outsock, outsock,
                        connect_to = (ip,port),
                        peername = '%s:%d' % (ip,port))
+
+
+def runonce(handlers, mux):
+    r = []
+    w = []
+    x = []
+    handlers = filter(lambda s: s.ok, handlers)
+    for s in handlers:
+        s.pre_select(r,w,x)
+    debug2('Waiting: %d r=%r w=%r x=%r (fullness=%d/%d)\n' 
+            % (len(handlers), _fds(r), _fds(w), _fds(x),
+               mux.fullness, mux.too_full))
+    (r,w,x) = select.select(r,w,x)
+    debug2('  Ready: %d r=%r w=%r x=%r\n' 
+        % (len(handlers), _fds(r), _fds(w), _fds(x)))
+    ready = r+w+x
+    did = {}
+    for h in handlers:
+        for s in h.socks:
+            if s in ready:
+                h.callback()
+                did[s] = 1
+    for s in ready:
+        if not s in did:
+            raise Fatal('socket %r was not used by any handler' % s)