[Python 2 to 3] Main six fixes (secdev#630)

* Apply six fixes
* Include six directly in scapy
* Ignore six library in coverage
* Moved winpcapy to modules
* Iterator and list fixes

.coveragerc

@@ -6,3 +6,6 @@ omit =
     test/*
     bin/*
     scapy/tools/*
+    # Libraries
+    scapy/modules/six.py
+	scapy/modules/winpcapy.py

CONTRIBUTING.md

@@ -72,7 +72,7 @@ guidelines for new code.
   - [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html)
     is a nice read!
   - Avoid creating unnecessary `list` objects, particularly if they
-    can be huge (e.g., when possible, use `xrange()` instead of
+    can be huge (e.g., when possible, use `scapy.modules.six.range()` instead of
     `range()`, `for line in fdesc` instead of `for line in
     fdesc.readlines()`; more generally prefer generators over lists).
 
@@ -116,6 +116,11 @@ The project aims to provide code that works both on Python 2 and Python 3. There
 - byte-string must be defined as `b"\x00\x01\x02"`
 - exceptions must comply with the new Python 3 format: `except SomeError as e:`
 - lambdas must be written using a single argument when using tuples: use `lambda x_y: x_y[0] + f(x_y[1])` instead of `lambda (x, y): x + f(y)`.
+- use int instead of long
+- use list comprehension instead of map() and filter()
+- use scapy.modules.six.range instead of xrange and range
+- use scapy.modules.six.itervalues(dict) instead of dict.values() or dict.itervalues()
+- use scapy.modules.six.string_types instead of basestring
 
 ### Code review
 

scapy/ansmachine.py

@@ -11,9 +11,11 @@
 ## Answering machines ##
 ########################
 
+from __future__ import absolute_import
 from scapy.sendrecv import send,sendp,sniff
 from scapy.config import conf
 from scapy.error import log_interactive
+import scapy.modules.six as six
 
 class ReferenceAM(type):
     def __new__(cls, name, bases, dct):
@@ -23,8 +25,7 @@ def __new__(cls, name, bases, dct):
         return o
 
 
-class AnsweringMachine(object):
-    __metaclass__ = ReferenceAM
+class AnsweringMachine(six.with_metaclass(ReferenceAM, object)):
     function_name = ""
     filter = None
     sniff_options = { "store":0 }

scapy/arch/__init__.py

@@ -7,6 +7,7 @@
 Operating system specific functionality.
 """
 
+from __future__ import absolute_import
 import socket
 
 from scapy.consts import LINUX, OPENBSD, FREEBSD, NETBSD, DARWIN, \
@@ -18,7 +19,7 @@
 from scapy.data import *
 
 def str2mac(s):
-    return ("%02x:"*6)[:-1] % tuple(map(ord, s)) 
+    return ("%02x:"*6)[:-1] % tuple(ord(x) for x in s) 
 
 if not WINDOWS:
     if not scapy.config.conf.use_pcap and not scapy.config.conf.use_dnet:

scapy/arch/bpf/core.py

@@ -4,6 +4,7 @@
 Scapy *BSD native support - core
 """
 
+from __future__ import absolute_import
 from scapy.config import conf
 from scapy.error import Scapy_Exception, warning
 from scapy.data import ARPHDR_LOOPBACK, ARPHDR_ETHER
@@ -20,6 +21,7 @@
 from ctypes import cdll, cast, pointer, POINTER, Structure
 from ctypes import c_uint, c_uint32, c_int, c_ulong, c_char_p, c_ushort, c_ubyte
 from ctypes.util import find_library
+from scapy.modules.six.moves import range
 
 
 # ctypes definitions
@@ -132,7 +134,7 @@ def attach_filter(fd, iface, bpf_filter_string):
 
     # Fill the BPF instruction structures with the byte code
     lines = lines[1:]
-    for i in xrange(len(lines)):
+    for i in range(len(lines)):
         values = [int(v) for v in lines[i].split()]
         bip[i].code = c_ushort(values[0])
         bip[i].jt = c_ubyte(values[1])

scapy/arch/linux.py

@@ -7,6 +7,7 @@
 Linux specific functions.
 """
 
+from __future__ import absolute_import
 import sys,os,struct,socket,time
 from select import select
 from fcntl import ioctl
@@ -22,6 +23,7 @@
 import scapy.arch
 from scapy.error import warning, Scapy_Exception, log_interactive, log_loading
 from scapy.arch.common import get_if
+from scapy.modules.six.moves import range
 
 
 
@@ -182,7 +184,7 @@ def get_alias_address(iface_name, ip_mask):
     # Extract interfaces names
     out = struct.unpack("iL", ifreq)[0]
     names = names.tostring()
-    names = [names[i:i+offset].split('\0', 1)[0] for i in xrange(0, out, name_len)]
+    names = [names[i:i+offset].split('\0', 1)[0] for i in range(0, out, name_len)]
 
     # Look for the IP address
     for ifname in names:

scapy/arch/pcapdnet.py

@@ -23,7 +23,7 @@
 if conf.use_winpcapy:
   #mostly code from https://github.com/phaethon/scapy translated to python2.X
   try:
-      from scapy.arch.winpcapy import *
+      from scapy.modules.winpcapy import *
       def winpcapy_get_if_list():
           err = create_string_buffer(PCAP_ERRBUF_SIZE)
           devs = POINTER(pcap_if_t)()

scapy/arch/windows/__init__.py

@@ -6,6 +6,7 @@
 """
 Customizations needed to support Microsoft Windows.
 """
+from __future__ import absolute_import
 import os, re, sys, socket, time, itertools, platform
 import subprocess as sp
 from glob import glob
@@ -17,6 +18,9 @@
 from scapy.base_classes import Gen, Net, SetGen
 from scapy.data import MTU, ETHER_BROADCAST, ETH_P_ARP
 
+import scapy.modules.six as six
+from scapy.modules.six.moves import range, zip
+
 conf.use_pcap = False
 conf.use_dnet = False
 conf.use_winpcapy = True
@@ -90,7 +94,7 @@ def _vbs_exec_code(code, split_tag="@"):
     ps = sp.Popen([conf.prog.cscript, tmpfile.name],
                   stdout=sp.PIPE, stderr=open(os.devnull),
                   universal_newlines=True)
-    for _ in xrange(3):
+    for _ in range(3):
         # skip 3 first lines
         ps.stdout.readline()
     for line in ps.stdout:
@@ -459,14 +463,14 @@ def dev_from_name(self, name):
         """Return the first pcap device name for a given Windows
         device name.
         """
-        for iface in self.itervalues():
+        for iface in six.itervalues(self):
             if iface.name == name:
                 return iface
         raise ValueError("Unknown network interface %r" % name)
 
     def dev_from_pcapname(self, pcap_name):
         """Return Windows device name for given pcap device name."""
-        for iface in self.itervalues():
+        for iface in six.itervalues(self):
             if iface.pcap_name == pcap_name:
                 return iface
         raise ValueError("Unknown pypcap network interface %r" % pcap_name)
@@ -552,7 +556,7 @@ def _read_routes_xp():
     routes = []
     partial_routes = []
     # map local IP addresses to interfaces
-    local_addresses = {iface.ip: iface for iface in IFACES.itervalues()}
+    local_addresses = {iface.ip: iface for iface in six.itervalues(IFACES)}
     iface_indexes = {}
     for line in exec_query(['Get-WmiObject', 'Win32_IP4RouteTable'],
                            ['Name', 'Mask', 'NextHop', 'InterfaceIndex']):
@@ -799,7 +803,7 @@ def readLineScapy(prompt):
                 if line.strip() == "":
                     end = True
                 result = result + "\n" + line
-            return unicode(result)
+            return six.text_type(result)
         try:
             import readline
             console = readline.GetOutputFile()
@@ -843,7 +847,7 @@ def route_add_loopback(routes=None, ipv6=False, iflist=None):
     data['mac'] = '00:00:00:00:00:00'
     adapter = NetworkInterface(data)
     if iflist:
-        iflist.append(unicode("\\Device\\NPF_" + adapter.guid))
+        iflist.append(six.text_type("\\Device\\NPF_" + adapter.guid))
         return
     # Remove all LOOPBACK_NAME routes
     for route in list(conf.route.routes):

scapy/arch/windows/compatibility.py

@@ -7,6 +7,7 @@
 Instanciate part of the customizations needed to support Microsoft Windows.
 """
 
+from __future__ import absolute_import
 import itertools
 import os
 import re
@@ -23,6 +24,7 @@
 from scapy.arch.pcapdnet import PcapTimeoutElapsed
 from scapy.error import log_runtime
 from scapy.data import MTU, ETH_P_ARP,ETH_P_ALL
+import scapy.modules.six as six
 
 WINDOWS = True
 
@@ -141,7 +143,7 @@ def sndrcv(pks, pkt, timeout = 2, inter = 0, verbose=None, chainCC=0, retry=0, m
         finally:
             pass
 
-        remain = list(itertools.chain(*hsent.itervalues()))
+        remain = list(itertools.chain(*six.itervalues(hsent)))
         if multi:
             remain = [p for p in remain if not hasattr(p, '_answered')]
 

scapy/as_resolvers.py

@@ -8,6 +8,7 @@
 """
 
 
+from __future__ import absolute_import
 import socket, errno
 from scapy.config import conf
 

scapy/asn1/asn1.py

@@ -8,18 +8,21 @@
 ASN.1 (Abstract Syntax Notation One)
 """
 
+from __future__ import absolute_import
 import random
 from datetime import datetime
 from scapy.config import conf
 from scapy.error import Scapy_Exception, warning
 from scapy.volatile import RandField, RandIP
 from scapy.utils import Enum_metaclass, EnumElement, binrepr
+import scapy.modules.six as six
+from scapy.modules.six.moves import range
 
 class RandASN1Object(RandField):
     def __init__(self, objlist=None):
         self.objlist = [
             x._asn1_obj
-            for x in ASN1_Class_UNIVERSAL.__rdict__.itervalues()
+            for x in six.itervalues(ASN1_Class_UNIVERSAL.__rdict__)
             if hasattr(x, "_asn1_obj")
         ] if objlist is None else objlist
         self.chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
@@ -32,11 +35,11 @@ def _fix(self, n=0):
             return o(z)
         elif issubclass(o, ASN1_STRING):
             z = int(random.expovariate(0.05)+1)
-            return o("".join(random.choice(self.chars) for _ in xrange(z)))
+            return o("".join(random.choice(self.chars) for _ in range(z)))
         elif issubclass(o, ASN1_SEQUENCE) and (n < 10):
             z = int(random.expovariate(0.08)+1)
             return o([self.__class__(objlist=self.objlist)._fix(n + 1)
-                      for _ in xrange(z)])
+                      for _ in range(z)])
         return ASN1_INTEGER(int(random.gauss(0,1000)))
 
 
@@ -72,8 +75,7 @@ def get_stem(cls):
 class ASN1_Codecs_metaclass(Enum_metaclass):
     element_class = ASN1Codec
 
-class ASN1_Codecs:
-    __metaclass__ = ASN1_Codecs_metaclass
+class ASN1_Codecs(six.with_metaclass(ASN1_Codecs_metaclass)):
     BER = 1
     DER = 2
     PER = 3
@@ -112,12 +114,12 @@ class ASN1_Class_metaclass(Enum_metaclass):
     element_class = ASN1Tag
     def __new__(cls, name, bases, dct): # XXX factorise a bit with Enum_metaclass.__new__()
         for b in bases:
-            for k,v in b.__dict__.iteritems():
+            for k,v in six.iteritems(b.__dict__):
                 if k not in dct and isinstance(v,ASN1Tag):
                     dct[k] = v.clone()
 
         rdict = {}
-        for k,v in dct.iteritems():
+        for k,v in six.iteritems(dct):
             if isinstance(v, int):
                 v = ASN1Tag(k,v) 
                 dct[k] = v
@@ -133,8 +135,8 @@ def __new__(cls, name, bases, dct): # XXX factorise a bit with Enum_metaclass.__
         return cls
 
 
-class ASN1_Class:
-    __metaclass__ = ASN1_Class_metaclass
+class ASN1_Class(six.with_metaclass(ASN1_Class_metaclass)):
+    pass
 
 class ASN1_Class_UNIVERSAL(ASN1_Class):
     name = "UNIVERSAL"
@@ -185,8 +187,7 @@ def __new__(cls, name, bases, dct):
             warning("Error registering %r for %r" % (c.tag, c.codec))
         return c
 
-class ASN1_Object:
-    __metaclass__ = ASN1_Object_metaclass
+class ASN1_Object(six.with_metaclass(ASN1_Object_metaclass)):
     tag = ASN1_Class_UNIVERSAL.ANY
     def __init__(self, val):
         self.val = val

scapy/asn1/ber.py

@@ -9,9 +9,11 @@
 Basic Encoding Rules (BER) for ASN.1
 """
 
+from __future__ import absolute_import
 from scapy.error import warning
 from scapy.utils import binrepr,inet_aton,inet_ntoa
 from scapy.asn1.asn1 import ASN1_Decoding_Error,ASN1_Encoding_Error,ASN1_BadTag_Decoding_Error,ASN1_Codecs,ASN1_Class_UNIVERSAL,ASN1_Error,ASN1_DECODING_ERROR,ASN1_BADTAG
+import scapy.modules.six as six
 
 ##################
 ## BER encoding ##
@@ -184,8 +186,7 @@ def __new__(cls, name, bases, dct):
         return c
 
 
-class BERcodec_Object:
-    __metaclass__ = BERcodec_metaclass
+class BERcodec_Object(six.with_metaclass(BERcodec_metaclass)):
     codec = ASN1_Codecs.BER
     tag = ASN1_Class_UNIVERSAL.ANY
 

scapy/asn1/mib.py

@@ -8,11 +8,13 @@
 Management Information Base (MIB) parsing
 """
 
+from __future__ import absolute_import
 import re
 from glob import glob
 from scapy.dadict import DADict,fixname
 from scapy.config import conf
 from scapy.utils import do_graph
+import scapy.modules.six as six
 
 #################
 ## MIB parsing ##
@@ -32,7 +34,7 @@ def _findroot(self, x):
             x += "."
         max=0
         root="."
-        for k in self.iterkeys():
+        for k in six.iterkeys(self):
             if x.startswith(self[k]+"."):
                 if max < len(self[k]):
                     max = len(self[k])
@@ -53,8 +55,8 @@ def _oid(self, x):
     def _make_graph(self, other_keys=None, **kargs):
         if other_keys is None:
             other_keys = []
-        nodes = [(k, self[k]) for k in self.iterkeys()]
-        oids = [self[k] for k in self.iterkeys()]
+        nodes = [(k, self[k]) for k in six.iterkeys(self)]
+        oids = [self[k] for k in six.iterkeys(self)]
         for k in other_keys:
             if k not in oids:
                 nodes.append(self.oidname(k),k)
@@ -116,7 +118,7 @@ def mib_register(ident, value, the_mib, unresolved):
 def load_mib(filenames):
     the_mib = {'iso': ['1']}
     unresolved = {}
-    for k in conf.mib.iterkeys():
+    for k in six.iterkeys(conf.mib):
         mib_register(k, conf.mib[k].split("."), the_mib, unresolved)
 
     if isinstance(filenames, str):
@@ -138,9 +140,9 @@ def load_mib(filenames):
                 mib_register(ident, oid, the_mib, unresolved)
 
     newmib = MIBDict(_name="MIB")
-    for k,o in the_mib.iteritems():
+    for k,o in six.iteritems(the_mib):
         newmib[k]=".".join(o)
-    for k,o in unresolved.iteritems():
+    for k,o in six.iteritems(unresolved):
         newmib[k]=".".join(o)
 
     conf.mib=newmib