initial fixture based testing setup

* new dependencies files
* makes testing possible
* defines fixture storage paths
* creates initial fixture for testing
* defines test requirements
* fixes connection fault so that it raises a traceback for diagnostics

partial vmware#42
blocks vmware#55

.travis.yml

@@ -6,10 +6,10 @@ python:
 
 before_install:
     - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi
+    - pip install -r requirements.txt
+    - pip install -r test-requirements.txt
 
 install:
-    - python setup.py bdist_egg
-    - pip install -e file://$TRAVIS_BUILD_DIR
+    - python setup.py -q install
 
-script:
-    nosetests
+script: python setup.py test

pyVim/connect.py

@@ -23,7 +23,7 @@
 
 Detailed description (for [e]pydoc goes here).
 """
-
+from six import reraise
 import sys
 import threading
 import thread
@@ -313,7 +313,13 @@ def __Login(host, port, user, pwd, service, adapter, version, path,
    except vmodl.MethodFault:
       raise
    except Exception, e:
-      raise vim.fault.HostConnectFault(msg=str(e))
+      # NOTE (hartsock): preserve the traceback for diagnostics
+      # pulling and preserving the traceback makes diagnosing connection
+      # failures easier since the fault will also include where inside the
+      # library the fault occurred. Without the traceback we have no idea
+      # why the connection failed beyond the message string.
+      (type, value, traceback) = sys.exc_info()
+      reraise(vim.fault.HostConnectFault(msg=str(e)), None, traceback)
 
    # Get a ticket if we're connecting to localhost and password is not specified
    if host == 'localhost' and not pwd:

pyVmomi/SoapAdapter.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import httplib
+from six.moves import http_client
 import sys
 import os
 import time
@@ -854,15 +854,17 @@ def SerializeRequest(self, mo, info, args):
 ## Subclass of HTTPConnection that connects over a Unix domain socket
 ## instead of a TCP port.  The path of the socket is passed in place of
 ## the hostname.  Fairly gross but does the job.
-class UnixSocketConnection(httplib.HTTPConnection):
+# NOTE (hartsock): rewrite this class as a wrapper, see HTTPSConnectionWrapper
+# below for a guide.
+class UnixSocketConnection(http_client.HTTPConnection):
    # The HTTPConnection ctor expects a single argument, which it interprets
    # as the host to connect to; for UnixSocketConnection, we instead interpret
    # the parameter as the filesystem path of the Unix domain socket.
    def __init__(self, path):
       # Pass '' as the host to HTTPConnection; it doesn't really matter
       # what we pass (since we've overridden the connect method) as long
       # as it's a valid string.
-      httplib.HTTPConnection.__init__(self, '')
+      http_client.HTTPConnection.__init__(self, '')
       self.path = path
 
    def connect(self):
@@ -884,7 +886,7 @@ def _VerifyThumbprint(thumbprint, connection):
       '''If there is a thumbprint, connect to the server and verify that the
       SSL certificate matches the given thumbprint.  An exception is thrown
       if there is a mismatch.'''
-      if thumbprint and isinstance(connection, httplib.HTTPSConnection):
+      if thumbprint and isinstance(connection, http_client.HTTPSConnection):
          if not connection.sock:
             connection.connect()
          derCert = connection.sock.getpeercert(True)
@@ -903,21 +905,28 @@ def _VerifyThumbprint(thumbprint, connection):
    SSL_THUMBPRINTS_SUPPORTED = False
 
    def _VerifyThumbprint(thumbprint, connection):
-      if thumbprint and isinstance(connection, httplib.HTTPSConnection):
+      if thumbprint and isinstance(connection, http_client.HTTPSConnection):
          raise Exception(
             "Thumbprint verification not supported on python < 2.6")
 
    def _SocketWrapper(rawSocket, keyfile, certfile, *args, **kwargs):
       wrappedSocket = socket.ssl(rawSocket, keyfile, certfile)
-      return httplib.FakeSocket(rawSocket, wrappedSocket)
+      return http_client.FakeSocket(rawSocket, wrappedSocket)
 
-## Internal version of https connection
+## https connection wrapper
 #
+# NOTE (hartsock): do not override core library types or implementations
+# directly because this makes brittle code that is too easy to break and
+# closely tied to implementation details we do not control. Instead, wrap
+# the core object to introduce additional behaviors.
+#
+# Purpose:
 # Support ssl.wrap_socket params which are missing from httplib
 # HTTPSConnection (e.g. ca_certs)
 # Note: Only works iff the ssl params are passing in as kwargs
-class _HTTPSConnection(httplib.HTTPSConnection):
+class HTTPSConnectionWrapper(object):
    def __init__(self, *args, **kwargs):
+      wrapped = http_client.HTTPSConnection(*args, **kwargs)
       # Extract ssl.wrap_socket param unknown to httplib.HTTPConnection,
       # and push back the params in connect()
       self._sslArgs = {}
@@ -927,34 +936,31 @@ def __init__(self, *args, **kwargs):
                   "ciphers"]:
          if key in tmpKwargs:
             self._sslArgs[key] = tmpKwargs.pop(key)
-      httplib.HTTPSConnection.__init__(self, *args, **tmpKwargs)
+      self._wrapped = wrapped
 
    ## Override connect to allow us to pass in additional ssl paramters to
    #  ssl.wrap_socket (e.g. cert_reqs, ca_certs for ca cert verification)
-   def connect(self):
-      if len(self._sslArgs) == 0:
+   def connect(self, wrapped):
+      if len(self._sslArgs) == 0 or hasattr(self, '_baseclass'):
          # No override
-         httplib.HTTPSConnection.connect(self)
-         return
+         return wrapped.connect
 
       # Big hack. We have to copy and paste the httplib connect fn for
       # each python version in order to handle extra ssl paramters. Yuk!
       if hasattr(self, "source_address"):
          # Python 2.7
          sock = socket.create_connection((self.host, self.port),
                                          self.timeout, self.source_address)
-         if self._tunnel_host:
-            self.sock = sock
-            self._tunnel()
-         self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, **self._sslArgs)
+         if wrapped._tunnel_host:
+            wrapped.sock = sock
+            wrapped._tunnel()
+         wrapped.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, **self._sslArgs)
       elif hasattr(self, "timeout"):
          # Python 2.6
          sock = socket.create_connection((self.host, self.port), self.timeout)
-         self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, **self._sslArgs)
-      else:
-         # Unknown python version. Do nothing
-         httplib.HTTPSConnection.connect(self)
-         return
+         wrapped.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, **self._sslArgs)
+
+      return wrapped.connect
 
       # TODO: Additional verification of peer cert if needed
       #cert_reqs = self._sslArgs.get("cert_reqs", ssl.CERT_NONE)
@@ -965,6 +971,11 @@ def connect(self):
       #      dercert = self.sock.getpeercert(False)
       #      # pemcert = ssl.DER_cert_to_PEM_cert(dercert)
 
+   def __getattr__(self, item):
+       if item == 'connect':
+           return self.connect(self._wrapped)
+       return getattr(self._wrapped, item)
+
 ## Stand-in for the HTTPSConnection class that will connect to a proxy and
 ## issue a CONNECT command to start an SSL tunnel.
 class SSLTunnelConnection(object):
@@ -985,13 +996,13 @@ def __call__(self, path, key_file=None, cert_file=None, **kwargs):
       for arg in kwargs.keys():
          if arg not in ("port", "strict", "timeout", "source_address"):
             del kwargs[arg]
-      tunnel = httplib.HTTPConnection(path, **kwargs)
+      tunnel = http_client.HTTPConnection(path, **kwargs)
       tunnel.request('CONNECT', self.proxyPath)
       resp = tunnel.getresponse()
       tunnelSocket = resp.fp
       if resp.status != 200:
-         raise httplib.HTTPException("%d %s" % (resp.status, resp.reason))
-      retval = httplib.HTTPSConnection(path)
+         raise http_client.HTTPException("%d %s" % (resp.status, resp.reason))
+      retval = http_client.HTTPSConnection(path)
       retval.sock = _SocketWrapper(tunnelSocket,
                                    keyfile=key_file, certfile=cert_file)
       return retval
@@ -1121,11 +1132,11 @@ def __init__(self, host='localhost', port=443, ns=None, path='/sdk',
          # keyword argument as passed in.
          if urlpath not in ('', '/'):
             path = urlpath
-         self.scheme = scheme == "http" and httplib.HTTPConnection \
-                    or scheme == "https" and _HTTPSConnection
+         self.scheme = scheme == "http" and http_client.HTTPConnection \
+                    or scheme == "https" and HTTPSConnectionWrapper
       else:
-         port, self.scheme = port < 0 and (-port, httplib.HTTPConnection) \
-                                       or (port, _HTTPSConnection)
+         port, self.scheme = port < 0 and (-port, http_client.HTTPConnection) \
+                                       or (port, HTTPSConnectionWrapper)
          if host.find(':') != -1:  # is IPv6?
             host = '[' + host + ']'
          self.host = '%s:%d' % (host, port)
@@ -1141,7 +1152,7 @@ def __init__(self, host='localhost', port=443, ns=None, path='/sdk',
       if sslProxyPath:
          self.scheme = SSLTunnelConnection(sslProxyPath)
       elif httpProxyHost:
-         if self.scheme == _HTTPSConnection:
+         if self.scheme == HTTPSConnectionWrapper:
             self.scheme = SSLTunnelConnection(self.host)
          else:
             if url:
@@ -1206,7 +1217,7 @@ def InvokeMethod(self, mo, info, args, outerStub=None):
       try:
          conn.request('POST', self.path, req, headers)
          resp = conn.getresponse()
-      except (socket.error, httplib.HTTPException):
+      except (socket.error, http_client.HTTPException):
          # The server is probably sick, drop all of the cached connections.
          self.DropConnections()
          raise
@@ -1240,7 +1251,7 @@ def InvokeMethod(self, mo, info, args, outerStub=None):
             raise obj # pylint: disable-msg=E0702
       else:
          conn.close()
-         raise httplib.HTTPException("%d %s" % (resp.status, resp.reason))
+         raise http_client.HTTPException("%d %s" % (resp.status, resp.reason))
 
    ## Clean up connection pool to throw away idle timed-out connections
    #  SoapStubAdapter lock must be acquired before this method is called.
@@ -1461,7 +1472,7 @@ def InvokeMethod(self, mo, info, args):
                self._CallLoginMethod()
             # Invoke the method
             status, obj = self.soapStub.InvokeMethod(mo, info, args, self)
-         except (socket.error, httplib.HTTPException, ExpatError):
+         except (socket.error, http_client.HTTPException, ExpatError):
             if self.retryDelay and retriesLeft:
                time.sleep(self.retryDelay)
             retriesLeft -= 1
@@ -1497,7 +1508,7 @@ def InvokeAccessor(self, mo, info):
                self._CallLoginMethod()
             # Invoke the method
             obj = StubAdapterBase.InvokeAccessor(self, mo, info)
-         except (socket.error, httplib.HTTPException, ExpatError):
+         except (socket.error, http_client.HTTPException, ExpatError):
             if self.retryDelay and retriesLeft:
                time.sleep(self.retryDelay)
             retriesLeft -= 1

requirements.txt

@@ -0,0 +1,2 @@
+requests>=2.3.0
+six>=1.7.3

setup.py

@@ -16,9 +16,16 @@
 from setuptools import setup
 import os
 
+
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
+with open('requirements.txt') as f:
+    required = f.read().splitlines()
+
+with open('test-requirements.txt') as f:
+    required_for_tests = f.read().splitlines()
+
 setup(
    name='pyvmomi',
    version='5.5.0_2014.dev',
@@ -27,6 +34,7 @@ def read(fname):
    author_email='jhu@vmware.com',
    url='https://github.com/vmware/pyvmomi',
    packages=['pyVmomi', 'pyVim'],
+   install_requires=required,
    license='Apache',
    long_description=read('README.md'),
    classifiers=[
@@ -38,5 +46,7 @@ def read(fname):
       "Topic :: Software Development :: Libraries :: Python Modules",
       "Topic :: System :: Distributed Computing"
    ],
+   test_suite='tests',
+   tests_require= required_for_tests,
    zip_safe=True
 )

test-requirements.txt

@@ -0,0 +1,4 @@
+mock
+PyYAML>=3.11
+testtools>=0.9.34
+vcrpy>=1.0.2

tests/__init__.py

@@ -0,0 +1,23 @@
+# VMware vSphere Python SDK
+# Copyright (c) 2008-2014 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+
+def tests_resource_path(local_path=''):
+    this_file = os.path.dirname(os.path.abspath(__file__))
+    return os.path.join(this_file, local_path)
+
+# Fully qualified path to the fixtures directory underneath this module
+fixtures_path = tests_resource_path('fixtures')