Merge branch 'fullness'

Tests with speedtest.net to a linode.com server:

                       Downstream     Upstream

No sshuttle            1.25 Mbit/s    0.55 Mbit/s
Default                0.75 Mbit/s    0.51 Mbit/s
--no-latency-control   1.25 Mbit/s    0.55 Mbit/s

* fullness:
  man page for the --no-latency-control option.
  options: remove unused 'exe' parameter
  options.py: generate usage string correctly for no-* options.
  Implement the optional fullness checking a bit more like I like it.
  new option to disable fullness checking

client.py

@@ -189,7 +189,8 @@ class FirewallClient:
             raise Fatal('cleanup: %r returned %d' % (self.argv, rv))
 
 
-def _main(listener, fw, ssh_cmd, remotename, python, seed_hosts, auto_nets,
+def _main(listener, fw, ssh_cmd, remotename, python, latency_control,
+          seed_hosts, auto_nets,
           syslog, daemon):
     handlers = []
     if helpers.verbose >= 1:
@@ -200,7 +201,8 @@ def _main(listener, fw, ssh_cmd, remotename, python, seed_hosts, auto_nets,
 
     try:
         (serverproc, serversock) = ssh.connect(ssh_cmd, remotename, python,
-                        stderr=ssyslog._p and ssyslog._p.stdin)
+                        stderr=ssyslog._p and ssyslog._p.stdin,
+                        options=dict(latency_control=latency_control))
     except socket.error, e:
         if e.args[0] == errno.EPIPE:
             raise Fatal("failed to establish ssh session (1)")
@@ -300,11 +302,13 @@ def _main(listener, fw, ssh_cmd, remotename, python, seed_hosts, auto_nets,
             raise Fatal('server died with error code %d' % rv)
         
         ssnet.runonce(handlers, mux)
+        if latency_control:
+            mux.check_fullness()
         mux.callback()
-        mux.check_fullness()
 
 
-def main(listenip, ssh_cmd, remotename, python, seed_hosts, auto_nets,
+def main(listenip, ssh_cmd, remotename, python, latency_control,
+         seed_hosts, auto_nets,
          subnets_include, subnets_exclude, syslog, daemon, pidfile):
     if syslog:
         ssyslog.start_syslog()
@@ -344,7 +348,8 @@ def main(listenip, ssh_cmd, remotename, python, seed_hosts, auto_nets,
     
     try:
         return _main(listener, fw, ssh_cmd, remotename,
-                     python, seed_hosts, auto_nets, syslog, daemon)
+                     python, latency_control,
+                     seed_hosts, auto_nets, syslog, daemon)
     finally:
         try:
             if daemon:

main.py

@@ -60,6 +60,7 @@ x,exclude= exclude this subnet (can be used more than once)
 v,verbose  increase debug message verbosity
 e,ssh-cmd= the command to use to connect to the remote [ssh]
 seed-hosts= with -H, use these hostnames for initial scan (comma-separated)
+no-latency-control  sacrifice latency to improve bandwidth benchmarks
 D,daemon   run in the background as a daemon
 syslog     send log messages to syslog (default if you use --daemon)
 pidfile=   pidfile name (only if using --daemon) [./sshuttle.pid]
@@ -67,7 +68,7 @@ server     (internal use only)
 firewall   (internal use only)
 hostwatch  (internal use only)
 """
-o = options.Options('sshuttle', optspec)
+o = options.Options(optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 
 if opt.daemon:
@@ -78,6 +79,7 @@ try:
     if opt.server:
         if len(extra) != 0:
             o.fatal('no arguments expected')
+        server.latency_control = opt.latency_control
         sys.exit(server.main())
     elif opt.firewall:
         if len(extra) != 1:
@@ -108,6 +110,7 @@ try:
                              opt.ssh_cmd,
                              remotename,
                              opt.python,
+                             opt.latency_control,
                              sh,
                              opt.auto_nets,
                              parse_subnets(includes),

options.py

@@ -76,9 +76,8 @@ class Options:
     By default, the parser function is getopt.gnu_getopt, and the abort
     behaviour is to exit the program.
     """
-    def __init__(self, exe, optspec, optfunc=getopt.gnu_getopt,
+    def __init__(self, optspec, optfunc=getopt.gnu_getopt,
                  onabort=_default_onabort):
-        self.exe = exe
         self.optspec = optspec
         self._onabort = onabort
         self.optfunc = optfunc
@@ -122,8 +121,8 @@ class Options:
                     defval = None
                 flagl = flags.split(',')
                 flagl_nice = []
-                for f in flagl:
+                for _f in flagl:
-                    f,dvi = _remove_negative_kv(f, _intify(defval))
+                    f,dvi = _remove_negative_kv(_f, _intify(defval))
                     self._aliases[f] = _remove_negative_k(flagl[0])
                     self._hasparms[f] = has_parm
                     self._defaults[f] = dvi
@@ -135,7 +134,7 @@ class Options:
                         self._aliases[f_nice] = _remove_negative_k(flagl[0])
                         self._longopts.append(f + (has_parm and '=' or ''))
                         self._longopts.append('no-' + f)
-                        flagl_nice.append('--' + f)
+                        flagl_nice.append('--' + _f)
                 flags_nice = ', '.join(flagl_nice)
                 if has_parm:
                     flags_nice += ' ...'

server.py

@@ -111,6 +111,7 @@ def main():
         helpers.logprefix = ' s: '
     else:
         helpers.logprefix = 'server: '
+    debug1('latency control setting = %r\n' % latency_control)
 
     routes = list(list_routes())
     debug1('available routes:\n')
@@ -172,5 +173,6 @@ def main():
                 raise Fatal('hostwatch exited unexpectedly: code 0x%04x\n' % rv)
         
         ssnet.runonce(handlers, mux)
-        mux.check_fullness()
+        if latency_control:
+            mux.check_fullness()
         mux.callback()

ssh.py

@@ -14,14 +14,16 @@ def readfile(name):
     raise Exception("can't find file %r in any of %r" % (name, path))
 
 
-def empackage(z, filename):
+def empackage(z, filename, data=None):
     (path,basename) = os.path.split(filename)
-    content = z.compress(readfile(filename))
+    if not data:
+        data = readfile(filename)
+    content = z.compress(data)
     content += z.flush(zlib.Z_SYNC_FLUSH)
-    return '%s\n%d\n%s' % (basename,len(content), content)
+    return '%s\n%d\n%s' % (basename, len(content), content)
 
 
-def connect(ssh_cmd, rhostport, python, stderr):
+def connect(ssh_cmd, rhostport, python, stderr, options):
     main_exe = sys.argv[0]
     portl = []
 
@@ -52,7 +54,9 @@ def connect(ssh_cmd, rhostport, python, stderr):
 
     z = zlib.compressobj(1)
     content = readfile('assembler.py')
-    content2 = (empackage(z, 'helpers.py') +
+    optdata = ''.join("%s=%r\n" % (k,v) for (k,v) in options.items())
+    content2 = (empackage(z, 'cmdline_options.py', optdata) +
+                empackage(z, 'helpers.py') +
                 empackage(z, 'compat/ssubprocess.py') +
                 empackage(z, 'ssnet.py') +
                 empackage(z, 'hostwatch.py') +

sshuttle.md

@@ -1,6 +1,6 @@
-% sshuttle(8) Sshuttle 0.44
+% sshuttle(8) Sshuttle 0.46
 % Avery Pennarun <apenwarr@gmail.com>
-% 2010-12-31
+% 2011-01-25
 
 # NAME
 
@@ -109,6 +109,22 @@ entire subnet to the VPN.
     if you use this option to give it a few names to start
     from.
     
+--no-latency-control
+:   sacrifice latency to improve bandwidth benchmarks. ssh
+    uses really big socket buffers, which can overload the
+    connection if you start doing large file transfers,
+    thus making all your other sessions inside the same
+    tunnel go slowly. Normally, sshuttle tries to avoid
+    this problem using a "fullness check" that allows only
+    a certain amount of outstanding data to be buffered at
+    a time.  But on high-bandwidth links, this can leave a
+    lot of your bandwidth underutilized.  It also makes
+    sshuttle seem slow in bandwidth benchmarks (benchmarks
+    rarely test ping latency, which is what sshuttle is
+    trying to control).  This option disables the latency
+    control feature, maximizing bandwidth usage.  Use at
+    your own risk.
+    
 -D, --daemon
 :   automatically fork into the background after connecting
     to the remote server.  Implies `--syslog`.