Linting and license update (#5)

* lint and license

* linting fixes

* remove .vs

.gitignore

@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# Visual Studio 
+.vs/

cli/eventhandlers.py

@@ -1,3 +1,8 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+"""
+
 import logging
 import sys
 from common.api import NutterStatusEvents
@@ -22,7 +27,7 @@ def _get_and_handle(self, event_queue):
             event_instance = event_queue.get()
             if self._debug:
                 logging.debug(
-                        'Message from queue: {}'.format(event_instance))
+                    'Message from queue: {}'.format(event_instance))
                 return
             output = self._get_output(event_instance)
             self._print_output(output)
@@ -69,15 +74,18 @@ def _handle_testlistingresults(self, event):
         return '{} tests found'.format(event.data)
 
     def _handle_testsexecuted(self, event):
-        return '{} Success:{} {}'.format(event.data.notebook_path, event.data.success, event.data.notebook_run_page_url)
+        return '{} Success:{} {}'.format(event.data.notebook_path,
+                                         event.data.success,
+                                         event.data.notebook_run_page_url)
 
     def _handle_testsexecutionrequest(self, event):
         return 'Execution request: {}'.format(event.data)
 
     def _handle_testscheduling(self, event):
         num_of_tests = self._num_of_test_to_execute()
         self._scheduled_tests += 1
-        return '{} of {} tests scheduled for execution'.format(self._scheduled_tests, num_of_tests)
+        return '{} of {} tests scheduled for execution'.format(self._scheduled_tests,
+                                                               num_of_tests)
 
     def _handle_testsexecutionresult(self, event):
         num_of_tests = self._num_of_test_to_execute()

cli/nuttercli.py

@@ -1,20 +1,20 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+"""
+
 import fire
 import logging
 import os
-import sys
 import datetime
 
-from queue import Queue
 import common.api as api
 from common.apiclient import InvalidConfigurationException
-from common.apiclientresults import ExecuteNotebookResult
-from common.statuseventhandler import EventHandler, StatusEvent
-from common.api import NutterStatusEvents
 
 import common.resultsview as view
 from .eventhandlers import ConsoleEventHandler
 from .resultsvalidator import ExecutionResultsValidator
-from .reportsman import ReportWriterManager, ReportWriters, ReportWritersTypes
+from .reportsman import ReportWriters
 from . import reportsman as reports
 
 __version__ = '0.1.31'
@@ -42,16 +42,26 @@ class NutterCLI(object):
     def __init__(self, debug=False, log_to_file=False, version=False):
         self._logger = logging.getLogger('NutterCLI')
         self._handle_show_version(version)
-        #CLI only logger so the output is not dictated by the logging configuration of all the other components
+
+        # CLI only logger so the output is not dictated
+        # by the logging configuration of all the other components
         self._set_debugging(debug, log_to_file)
         self._print_cli_header()
         self._set_nutter(debug)
         super().__init__()
 
-    def run(self, test_pattern, cluster_id, timeout=120, junit_report=False, tags_report=False, max_parallel_tests=1, recursive=False):
+    def run(self, test_pattern, cluster_id,
+            timeout=120, junit_report=False,
+            tags_report=False, max_parallel_tests=1,
+            recursive=False):
         try:
-            logging.debug("Running tests. test_pattern: {} cluster_id: {} timeout: {} junit_report: {} max_parallel_tests: {} tags_report: {}  recursive:{} ".format(
-                test_pattern, cluster_id, timeout, junit_report, max_parallel_tests, tags_report, recursive))
+            logging.debug(""" Running tests. test_pattern: {} cluster_id: {} timeout: {}
+                               junit_report: {} max_parallel_tests: {}
+                               tags_report: {}  recursive:{} """
+                          .format(test_pattern, cluster_id, timeout,
+                                  junit_report, max_parallel_tests,
+                                  tags_report, recursive))
+
             logging.debug("Executing test(s): {}".format(test_pattern))
 
             if self._is_a_test_pattern(test_pattern):
@@ -135,7 +145,8 @@ def _is_a_test_pattern(self, pattern):
                 return False
             return True
         logging.Fatal(
-            'Invalid argument. The value must be the full path to the test or a pattern')
+            """ Invalid argument.
+                 The value must be the full path to the test or a pattern """)
 
     def _print_cli_header(self):
         print(get_cli_header())
@@ -159,7 +170,9 @@ def _get_version_label(self):
         return 'Nutter Version {}'.format(version)
 
     def _print_config_error_and_exit(self):
-        print('Invalid configuration.\nDATABRICKS_HOST and DATABRICKS_TOKEN environment variables are not set')
+        print(""" Invalid configuration.\n
+                  DATABRICKS_HOST and DATABRICKS_TOKEN
+                   environment variables are not set """)
         exit(1)
 
     def _set_debugging(self, debug, log_to_file):
@@ -169,7 +182,9 @@ def _set_debugging(self, debug, log_to_file):
                 log_name = 'nutter-exec-{0:%Y.%m.%d.%H%M%S%f}.log'.format(
                     datetime.datetime.utcnow())
             logging.basicConfig(
-                filename=log_name, format="%(asctime)s:%(levelname)s:%(message)s", level=logging.DEBUG)
+                filename=log_name,
+                format="%(asctime)s:%(levelname)s:%(message)s",
+                level=logging.DEBUG)
 
 
 def main():

cli/reportsman.py

@@ -1,5 +1,9 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+"""
+
 import common.api as nutter_api
-from common.testresult import TestResults
 from enum import Enum
 from enum import IntEnum
 

cli/resultsvalidator.py

@@ -1,6 +1,9 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+"""
 
 from common.apiclientresults import ExecuteNotebookResult
-import common.api as api
 from common.testresult import TestResults
 import logging
 
@@ -17,8 +20,8 @@ def _validate_result(self, result):
         if not isinstance(result, ExecuteNotebookResult):
             raise ValueError("Expected ExecuteNotebookResult")
         if result.is_error:
-            msg = 'The job is not in a successfull terminal state. Life cycle state:{}'.format(
-                result.task_result_state)
+            msg = """ The job is not in a successfull terminal state.
+                       Life cycle state:{} """.format(result.task_result_state)
             raise JobExecutionFailureException(message=msg)
         if result.notebook_result.is_error:
             msg = 'The notebook failed. result state:{}'.format(
@@ -33,8 +36,8 @@ def _validate_test_results(self, exit_output):
             test_results = TestResults().deserialize(exit_output)
         except Exception as ex:
             logging.debug(ex)
-            msg = 'The Notebook exit output value is invalid or missing. Additional info: {}'.format(
-                str(ex))
+            msg = """ The Notebook exit output value is invalid or missing.
+                       Additional info: {} """.format(str(ex))
             raise InvalidNotebookOutputException(msg)
 
         for test_result in test_results.results:

common/api.py

@@ -1,17 +1,19 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+"""
+
 from abc import abstractmethod, ABCMeta
 from .testresult import TestResults
 from . import scheduler
 from . import apiclient
-from junit_xml import TestSuite, TestCase
 from .resultreports import JunitXMLReportWriter, TestResultsReportWriter
 from .statuseventhandler import StatusEventsHandler
 
 import enum
 import logging
 
-import os
 import re
-import json
 import importlib
 
 
@@ -79,7 +81,7 @@ def list_tests(self, path, recursive=False):
             tests.append(test)
 
         self._add_status_event(
-                NutterStatusEvents.TestsListingResults, len(tests))
+            NutterStatusEvents.TestsListingResults, len(tests))
 
         return tests
 
@@ -94,7 +96,9 @@ def run_test(self, testpath, cluster_id, timeout=120):
 
         return result
 
-    def run_tests(self, pattern, cluster_id, timeout=120, max_parallel_tests=1, recursive=False):
+    def run_tests(self, pattern, cluster_id,
+                  timeout=120, max_parallel_tests=1, recursive=False):
+
         self._add_status_event(NutterStatusEvents.TestExecutionRequest, pattern)
         root, pattern_to_match = self._get_root_and_pattern(pattern)
 
@@ -118,7 +122,7 @@ def events_processor_wait(self):
         self._events_processor.wait()
 
     def _list_tests(self, path, recursive):
-        self._add_status_event(NutterStatusEvents.TestsListing,path)
+        self._add_status_event(NutterStatusEvents.TestsListing, path)
         workspace_objects = self.dbclient.list_objects(path)
 
         for notebook in workspace_objects.test_notebooks:
@@ -158,7 +162,8 @@ def _get_root_and_pattern(self, pattern):
 
         return root, valid_pattern
 
-    def _schedule_and_run(self, test_notebooks, cluster_id, max_parallel_tests, timeout):
+    def _schedule_and_run(self, test_notebooks, cluster_id,
+                          max_parallel_tests, timeout):
         func_scheduler = scheduler.get_scheduler(max_parallel_tests)
         for test_notebook in test_notebooks:
             self._add_status_event(
@@ -170,12 +175,13 @@ def _schedule_and_run(self, test_notebooks, cluster_id, max_parallel_tests, time
         return self._run_and_await(func_scheduler)
 
     def _execute_notebook(self, test_notebook_path, cluster_id, timeout):
-        result = self.dbclient.execute_notebook(test_notebook_path, cluster_id, None, timeout)
-        self._add_status_event(
-                NutterStatusEvents.TestExecuted, ExecutionResultEventData.from_execution_results(result))
-        logging.debug(
-                'Executed: {}'.format(test_notebook_path))
+        result = self.dbclient.execute_notebook(test_notebook_path,
+                                                cluster_id, None, timeout)
+        self._add_status_event(NutterStatusEvents.TestExecuted,
+                               ExecutionResultEventData.from_execution_results(result))
+        logging.debug('Executed: {}'.format(test_notebook_path))
         return result
+
     def _run_and_await(self, func_scheduler):
         logging.debug('Scheduler run and wait.')
         func_results = func_scheduler.run_and_wait()
@@ -209,7 +215,8 @@ def __init__(self, name, path):
         self.test_name = name.split("_")[1]
 
     def __eq__(self, obj):
-        return isinstance(obj, TestNotebook) and obj.name == self.name and obj.path == self.path
+        is_equal = obj.name == self.name and obj.path == self.path
+        return isinstance(obj, TestNotebook) and is_equal
 
     @classmethod
     def from_path(cls, path):
@@ -239,13 +246,15 @@ def __init__(self, pattern):
         try:
             # * is an invalid regex in python
             # however, we want to treat it as no filter
-            if pattern == '*' or pattern == None or pattern == '':
+            if pattern == '*' or pattern is None or pattern == '':
                 self._pattern = None
                 return
             re.compile(pattern)
         except re.error as ex:
+            logging.debug('Pattern could not be compiled. {}'.format(ex))
             raise ValueError(
-                "The pattern provided is invalid. The pattern must start with an alphanumeric character")
+                """ The pattern provided is invalid.
+                     The pattern must start with an alphanumeric character """)
         self._pattern = pattern
 
     def filter_by_pattern(self, test_notebooks):
@@ -271,7 +280,7 @@ def from_execution_results(cls, exec_results):
         notebook_run_page_url = exec_results.notebook_run_page_url
         notebook_path = exec_results.notebook_path
         try:
-          success = not exec_results.is_any_error
+            success = not exec_results.is_any_error
         except Exception as ex:
             logging.debug("Error while creating the ExecutionResultEventData {}", ex)
             success = False

common/apiclient.py

@@ -1,3 +1,8 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+"""
+
 import uuid
 import time
 from databricks_api import DatabricksAPI
@@ -24,12 +29,14 @@ def __init__(self):
 
         if config is None:
             raise InvalidConfigurationException
-        #TODO: remove the dependency with this API, an instead use httpclient/requests
+
+        # TODO: remove the dependency with this API, an instead use httpclient/requests
         db = DatabricksAPI(host=config.host,
                            token=config.token)
         self.inner_dbclient = db
-        #the retrier uses the recommended defaults
-        #https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/jobs
+
+        # The retrier uses the recommended defaults
+        # https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/jobs
         self._retrier = HTTPRetrier()
 
     def list_notebooks(self, path):
@@ -47,7 +54,8 @@ def list_objects(self, path):
 
         return workspace_path_obj
 
-    def execute_notebook(self, notebook_path, cluster_id, notebook_params=None, timeout=120):
+    def execute_notebook(self, notebook_path, cluster_id,
+                         notebook_params=None, timeout=120):
         if not notebook_path:
             raise ValueError("empty path")
         if not cluster_id:
@@ -87,8 +95,9 @@ def __pull_for_output(self, run_id, timeout):
             lcs = utils.recursive_find(
                 output, ['metadata', 'state', 'life_cycle_state'])
 
-            #As per https://docs.azuredatabricks.net/api/latest/jobs.html#jobsrunlifecyclestate
-            # all these are terminal states
+            # As per:
+            # https://docs.azuredatabricks.net/api/latest/jobs.html#jobsrunlifecyclestate
+            # All these are terminal states
             if lcs == 'TERMINATED' or lcs == 'SKIPPED' or lcs == 'INTERNAL_ERROR':
                 return lcs, output
             time.sleep(1)
@@ -98,7 +107,9 @@ def _raise_timeout(self, output):
         run_page_url = utils.recursive_find(
             output, ['metadata', 'run_page_url'])
         raise TimeOutException(
-            "Timeout while waiting for the result of a test.\nCheck the status of the execution\nRun page URL: {}".format(run_page_url))
+            """ Timeout while waiting for the result of a test.\n
+                Check the status of the execution\n
+                Run page URL: {} """.format(run_page_url))
 
     def __get_notebook_task(self, path, params):
         ntask = {}