Fix/pep8 (#277)

* re-organized imports according to pep8
* fixed all remaining pep8 issues
* moved common config into setup.cfg, additionally test `tests`
* removed --select=X -- the errors selected where by default not in
  flake8's --ignore list so effectively had no effect
* update .travis.yml to reflect changes in tox.ini
* make travis just use tox in order to avoid code duplaction
* replace py.test with pytest
* fixed .travis.yml
* try different pypy toxenv
* hopefully fixed testenv for pypy
* added pypy basepython, removed unused python2.6
* install dev package before testing (fixes missing coverage)
* fixed empty exception pass blocks with noqa
* Added dummy log message on empty try-except-pass blocks to make dodacy happy :(
* Replaced Exception with BaseException

.travis.yml

@@ -1,19 +1,20 @@
 language: python
-python:
-- 2.7
-- 3.4
-- 3.5
-- 3.6
-- pypy
+
+matrix:
+  include:
+    - python: 2.7
+      env: TOXENV=py27
+    - python: 3.4
+      env: TOXENV=py34
+    - python: 3.5
+      env: TOXENV=py35
+    - python: 3.6
+      env: TOXENV=py36
+    - python: pypy
+      env: TOXENV=pypy
 
 install:
-    - travis_retry pip install -q -r requirements-tests.txt
-
-before_script:
-    # stop the build if there are Python syntax errors or undefined names.
-    - flake8 sshuttle --count --select=E901,E999,F821,F822,F823 --show-source --statistics
-    # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide.
-    - flake8 sshuttle --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+  - pip install tox
 
 script:
-  - PYTHONPATH=. py.test
+  - tox

setup.cfg

@@ -8,5 +8,10 @@ universal = 1
 sign=true
 identity=0x1784577F811F6EAC
 
+[flake8]
+count=true
+show-source=true
+statistics=true
+
 [tool:pytest]
 addopts = --cov=sshuttle --cov-branch --cov-report=term-missing

sshuttle/assembler.py

@@ -2,6 +2,10 @@ import sys
 import zlib
 import imp
 
+import sshuttle.helpers
+import sshuttle.cmdline_options as options
+from sshuttle.server import main
+
 verbosity = verbosity  # noqa: F821 must be a previously defined global
 z = zlib.decompressobj()
 while 1:
@@ -30,10 +34,7 @@ while 1:
 sys.stderr.flush()
 sys.stdout.flush()
 
-import sshuttle.helpers
 sshuttle.helpers.verbose = verbosity
 
-import sshuttle.cmdline_options as options
-from sshuttle.server import main
 main(options.latency_control, options.auto_hosts, options.to_nameserver,
      options.auto_nets)

sshuttle/client.py

@@ -3,13 +3,14 @@ import re
 import signal
 import time
 import subprocess as ssubprocess
-import sshuttle.helpers as helpers
 import os
+import sys
+import platform
+
+import sshuttle.helpers as helpers
 import sshuttle.ssnet as ssnet
 import sshuttle.ssh as ssh
 import sshuttle.ssyslog as ssyslog
-import sys
-import platform
 from sshuttle.ssnet import SockWrapper, Handler, Proxy, Mux, MuxWrapper
 from sshuttle.helpers import log, debug1, debug2, debug3, Fatal, islocal, \
     resolvconf_nameservers
@@ -268,11 +269,13 @@ class FirewallClient:
         self.pfile.write(b'ROUTES\n')
         for (family, ip, width, fport, lport) \
                 in self.subnets_include + self.auto_nets:
-            self.pfile.write(b'%d,%d,0,%s,%d,%d\n'
-                    % (family, width, ip.encode("ASCII"), fport, lport))
+            self.pfile.write(b'%d,%d,0,%s,%d,%d\n' % (family, width,
+                                                      ip.encode("ASCII"),
+                                                      fport, lport))
         for (family, ip, width, fport, lport) in self.subnets_exclude:
-            self.pfile.write(b'%d,%d,1,%s,%d,%d\n'
-                    % (family, width, ip.encode("ASCII"), fport, lport))
+            self.pfile.write(b'%d,%d,1,%s,%d,%d\n' % (family, width,
+                                                      ip.encode("ASCII"),
+                                                      fport, lport))
 
         self.pfile.write(b'NSLIST\n')
         for (family, ip) in self.nslist:
@@ -495,7 +498,8 @@ def _main(tcp_listener, udp_listener, fw, ssh_cmd, remotename,
     def onroutes(routestr):
         if auto_nets:
             for line in routestr.strip().split(b'\n'):
-                if not line: continue
+                if not line:
+                    continue
                 (family, ip, width) = line.split(b',', 2)
                 family = int(family)
                 width = int(width)
@@ -707,7 +711,8 @@ def main(listenip_v6, listenip_v4,
         ports = range(12300, 9000, -1)
         for port in ports:
             debug2(' %d' % port)
-            if port in used_ports: continue
+            if port in used_ports:
+                continue
 
             dns_listener = MultiListener(socket.SOCK_DGRAM)
 

sshuttle/firewall.py

@@ -1,12 +1,13 @@
 import errno
 import socket
 import signal
-import sshuttle.ssyslog as ssyslog
-import sshuttle.sdnotify as sdnotify
 import sys
 import os
 import platform
 import traceback
+
+import sshuttle.ssyslog as ssyslog
+import sshuttle.sdnotify as sdnotify
 from sshuttle.helpers import debug1, debug2, Fatal
 from sshuttle.methods import get_auto_method, get_method
 
@@ -132,7 +133,7 @@ def main(method_name, syslog):
         try:
             (family, width, exclude, ip, fport, lport) = \
                     line.strip().split(',', 5)
-        except:
+        except BaseException:
             raise Fatal('firewall: expected route or NSLIST but got %r' % line)
         subnets.append((
             int(family),
@@ -154,7 +155,7 @@ def main(method_name, syslog):
             break
         try:
             (family, ip) = line.strip().split(',', 1)
-        except:
+        except BaseException:
             raise Fatal('firewall: expected nslist or PORTS but got %r' % line)
         nslist.append((int(family), ip))
         debug2('firewall manager: Got partial nslist: %r\n' % nslist)
@@ -248,43 +249,43 @@ def main(method_name, syslog):
         try:
             sdnotify.send(sdnotify.stop())
             debug1('firewall manager: undoing changes.\n')
-        except:
-            pass
+        except BaseException:
+            debug2('An error occurred, ignoring it.')
 
         try:
             if subnets_v6 or nslist_v6:
                 debug2('firewall manager: undoing IPv6 changes.\n')
                 method.restore_firewall(port_v6, socket.AF_INET6, udp, user)
-        except:
+        except BaseException:
             try:
                 debug1("firewall manager: "
                        "Error trying to undo IPv6 firewall.\n")
                 for line in traceback.format_exc().splitlines():
                     debug1("---> %s\n" % line)
-            except:
-                pass
+            except BaseException:
+                debug2('An error occurred, ignoring it.')
 
         try:
             if subnets_v4 or nslist_v4:
                 debug2('firewall manager: undoing IPv4 changes.\n')
                 method.restore_firewall(port_v4, socket.AF_INET, udp, user)
-        except:
+        except BaseException:
             try:
                 debug1("firewall manager: "
                        "Error trying to undo IPv4 firewall.\n")
                 for line in traceback.format_exc().splitlines():
                     debug1("firewall manager: ---> %s\n" % line)
-            except:
-                pass
+            except BaseException:
+                debug2('An error occurred, ignoring it.')
 
         try:
             debug2('firewall manager: undoing /etc/hosts changes.\n')
             restore_etc_hosts(port_v6 or port_v4)
-        except:
+        except BaseException:
             try:
                 debug1("firewall manager: "
                        "Error trying to undo /etc/hosts changes.\n")
                 for line in traceback.format_exc().splitlines():
                     debug1("firewall manager: ---> %s\n" % line)
-            except:
-                pass
+            except BaseException:
+                debug2('An error occurred, ignoring it.')

sshuttle/helpers.py

@@ -56,22 +56,22 @@ class Fatal(Exception):
 
 
 def resolvconf_nameservers():
-    l = []
+    lines = []
     for line in open('/etc/resolv.conf'):
         words = line.lower().split()
         if len(words) >= 2 and words[0] == 'nameserver':
-            l.append(family_ip_tuple(words[1]))
-    return l
+            lines.append(family_ip_tuple(words[1]))
+    return lines
 
 
 def resolvconf_random_nameserver():
-    l = resolvconf_nameservers()
-    if l:
-        if len(l) > 1:
+    lines = resolvconf_nameservers()
+    if lines:
+        if len(lines) > 1:
             # don't import this unless we really need it
             import random
-            random.shuffle(l)
-        return l[0]
+            random.shuffle(lines)
+        return lines[0]
     else:
         return (socket.AF_INET, '127.0.0.1')
 

sshuttle/hostwatch.py

@@ -44,7 +44,7 @@ def write_host_cache():
     finally:
         try:
             os.unlink(tmpname)
-        except:
+        except BaseException:
             pass
 
 

sshuttle/linux.py

@@ -2,6 +2,7 @@ import re
 import os
 import socket
 import subprocess as ssubprocess
+
 from sshuttle.helpers import log, debug1, Fatal, family_to_string
 
 

sshuttle/methods/ipfw.py

@@ -29,8 +29,8 @@ IPV6_RECVDSTADDR = 74
 if recvmsg == "python":
     def recv_udp(listener, bufsize):
         debug3('Accept UDP python using recvmsg.\n')
-        data, ancdata, _, srcip = \
-                listener.recvmsg(4096, socket.CMSG_SPACE(4))
+        data, ancdata, _, srcip = listener.recvmsg(4096,
+                                                   socket.CMSG_SPACE(4))
         dstip = None
         for cmsg_level, cmsg_type, cmsg_data in ancdata:
             if cmsg_level == socket.SOL_IP and cmsg_type == IP_RECVDSTADDR:
@@ -42,8 +42,8 @@ if recvmsg == "python":
 elif recvmsg == "socket_ext":
     def recv_udp(listener, bufsize):
         debug3('Accept UDP using socket_ext recvmsg.\n')
-        srcip, data, adata, _ = \
-                listener.recvmsg((bufsize,), socket.CMSG_SPACE(4))
+        srcip, data, adata, _ = listener.recvmsg((bufsize,),
+                                                 socket.CMSG_SPACE(4))
         dstip = None
         for a in adata:
             if a.cmsg_level == socket.SOL_IP and a.cmsg_type == IP_RECVDSTADDR:
@@ -134,6 +134,7 @@ def sysctl_set(name, val, permanent=False):
             _changedctls.append(name)
         return True
 
+
 def ipfw(*args):
     argv = ['ipfw', '-q'] + list(args)
     debug1('>> %s\n' % ' '.join(argv))
@@ -147,6 +148,7 @@ def ipfw_noexit(*args):
     debug1('>> %s\n' % ' '.join(argv))
     ssubprocess.call(argv)
 
+
 class Method(BaseMethod):
 
     def get_supported_features(self):
@@ -238,8 +240,9 @@ class Method(BaseMethod):
 
         if subnets:
             # create new subnet entries
-            for _, swidth, sexclude, snet \
-                in sorted(subnets, key=lambda s: s[1], reverse=True):
+            for _, swidth, sexclude, snet in sorted(subnets,
+                                                    key=lambda s: s[1],
+                                                    reverse=True):
                 if sexclude:
                     ipfw('table', '125', 'add', '%s/%s' % (snet, swidth))
             else:

sshuttle/methods/pf.py

@@ -124,12 +124,14 @@ class Generic(object):
         memmove(addressof(pr) + self.RULE_ACTION_OFFSET,
                 struct.pack('I', kind), 4)  # rule.action = kind
 
-        memmove(addressof(pr) + self.ACTION_OFFSET, struct.pack(
-            'I', self.PF_CHANGE_GET_TICKET), 4) # action = PF_CHANGE_GET_TICKET
+        memmove(addressof(pr) + self.ACTION_OFFSET,
+                struct.pack('I', self.PF_CHANGE_GET_TICKET),
+                4)  # action = PF_CHANGE_GET_TICKET
         ioctl(pf_get_dev(), pf.DIOCCHANGERULE, pr)
 
-        memmove(addressof(pr) + self.ACTION_OFFSET, struct.pack(
-            'I', self.PF_CHANGE_ADD_TAIL), 4) # action = PF_CHANGE_ADD_TAIL
+        memmove(addressof(pr) + self.ACTION_OFFSET,
+                struct.pack('I', self.PF_CHANGE_ADD_TAIL),
+                4)  # action = PF_CHANGE_ADD_TAIL
         ioctl(pf_get_dev(), pf.DIOCCHANGERULE, pr)
 
     @staticmethod
@@ -151,7 +153,6 @@ class Generic(object):
         return b'skip' in pfctl('-s Interfaces -i lo -v')[0]
 
 
-
 class FreeBsd(Generic):
     RULE_ACTION_OFFSET = 2968
 

sshuttle/methods/tproxy.py

@@ -169,7 +169,6 @@ class Method(BaseMethod):
             return proto + ('--dport', '%d:%d' % (fport, lport)) \
                     if fport else proto
 
-
         mark_chain = 'sshuttle-m-%s' % port
         tproxy_chain = 'sshuttle-t-%s' % port
         divert_chain = 'sshuttle-d-%s' % port

sshuttle/options.py

@@ -1,6 +1,7 @@
 import re
 import socket
 from argparse import ArgumentParser, Action, ArgumentTypeError as Fatal
+
 from sshuttle import __version__
 
 

sshuttle/sdnotify.py

@@ -1,7 +1,9 @@
 import socket
 import os
+
 from sshuttle.helpers import debug1
 
+
 def _notify(message):
     addr = os.environ.get("NOTIFY_SOCKET", None)
 
@@ -27,14 +29,18 @@ def _notify(message):
         debug1("Error notifying systemd: %s\n" % e)
         return False
 
+
 def send(*messages):
     return _notify(b'\n'.join(messages))
 
+
 def ready():
     return b"READY=1"
 
+
 def stop():
     return b"STOPPING=1"
 
+
 def status(message):
     return b"STATUS=%s" % message.encode('utf8')

sshuttle/server.py

@@ -289,7 +289,6 @@ def main(latency_control, auto_hosts, to_nameserver, auto_nets):
         helpers.logprefix = 'server: '
     debug1('latency control setting = %r\n' % latency_control)
 
-
     # synchronization header
     sys.stdout.write('\0\0SSHUTTLE0001')
     sys.stdout.flush()

sshuttle/ssh.py

@@ -6,6 +6,7 @@ import zlib
 import imp
 import subprocess as ssubprocess
 import shlex
+
 import sshuttle.helpers as helpers
 from sshuttle.helpers import debug2
 
@@ -76,10 +77,10 @@ def connect(ssh_cmd, rhostport, python, stderr, options):
         else:
             rhost = rhostport
     else:  # IPv4
-        l = (rhostport or '').rsplit(':', 1)
-        rhost = l[0]
-        if len(l) > 1:
-            portl = ['-p', str(int(l[1]))]
+        host_port = (rhostport or '').rsplit(':', 1)
+        rhost = host_port[0]
+        if len(host_port) > 1:
+            portl = ['-p', str(int(host_port[1]))]
 
     if rhost == '-':
         rhost = None

sshuttle/ssnet.py

@@ -4,6 +4,7 @@ import socket
 import errno
 import select
 import os
+
 from sshuttle.helpers import b, binary_type, log, debug1, debug2, debug3, Fatal
 
 MAX_CHANNEL = 65535

tests/client/test_firewall.py

@@ -1,7 +1,7 @@
-from mock import Mock, patch, call
 import io
 from socket import AF_INET, AF_INET6
 
+from mock import Mock, patch, call
 import sshuttle.firewall
 
 
@@ -85,8 +85,9 @@ def test_subnet_weight():
         (AF_INET, 0, 1, '0.0.0.0', 0, 0)
     ]
 
-    assert subnets_sorted == \
-            sorted(subnets, key=sshuttle.firewall.subnet_weight, reverse=True)
+    assert subnets_sorted == sorted(subnets,
+                                    key=sshuttle.firewall.subnet_weight,
+                                    reverse=True)
 
 
 @patch('sshuttle.firewall.rewrite_etc_hosts')

tests/client/test_helpers.py

@@ -1,10 +1,10 @@
-from mock import patch, call
 import sys
 import io
 import socket
 from socket import AF_INET, AF_INET6
 import errno
 
+from mock import patch, call
 import sshuttle.helpers
 
 

tests/client/test_methods_nat.py

@@ -1,9 +1,9 @@
-import pytest
-from mock import Mock, patch, call
 import socket
 from socket import AF_INET, AF_INET6
 import struct
 
+import pytest
+from mock import Mock, patch, call
 from sshuttle.helpers import Fatal
 from sshuttle.methods import get_method
 

tests/client/test_methods_pf.py

@@ -1,8 +1,8 @@
-import pytest
-from mock import Mock, patch, call, ANY
 import socket
 from socket import AF_INET, AF_INET6
 
+import pytest
+from mock import Mock, patch, call, ANY
 from sshuttle.methods import get_method
 from sshuttle.helpers import Fatal
 from sshuttle.methods.pf import FreeBsd, Darwin, OpenBsd

tests/client/test_options.py

@@ -1,8 +1,10 @@
 import socket
-import pytest
-import sshuttle.options
 from argparse import ArgumentTypeError as Fatal
 
+import pytest
+
+import sshuttle.options
+
 _ip4_reprs = {
         '0.0.0.0': '0.0.0.0',
         '255.255.255.255': '255.255.255.255',
@@ -25,6 +27,7 @@ _ip6_reprs = {
 
 _ip6_swidths = (48, 64, 96, 115, 128)
 
+
 def test_parse_subnetport_ip4():
     for ip_repr, ip in _ip4_reprs.items():
         assert sshuttle.options.parse_subnetport(ip_repr) \

tests/client/test_sdnotify.py

@@ -1,6 +1,7 @@
-from mock import Mock, patch, call
 import socket
 
+from mock import Mock, patch, call
+
 import sshuttle.sdnotify
 
 

tests/server/test_server.py

@@ -1,8 +1,10 @@
 import io
 import socket
-import sshuttle.server
+
 from mock import patch, Mock
 
+import sshuttle.server
+
 
 def test__ipmatch():
     assert sshuttle.server._ipmatch("1.2.3.4") is not None

tox.ini

@@ -5,17 +5,21 @@ envlist =
     py34,
     py35,
     py36,
+    pypy,
 
 [testenv]
 basepython =
-    py26: python2.6
     py27: python2.7
     py34: python3.4
     py35: python3.5
     py36: python3.6
+    pypy: pypy
 commands =
-    flake8 sshuttle --count --select=E901,E999,F821,F822,F823 --show-source --statistics
-    flake8 sshuttle --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    py.test
+    pip install -e .
+    # actual flake8 test
+    flake8 sshuttle tests
+    # flake8 complexity warnings
+    flake8 sshuttle tests --exit-zero --max-complexity=10
+    pytest
 deps =
     -rrequirements-tests.txt