Adding scripts that grab bazel licenses

Summary: These scripts are incorporated into genrules in follow up diff Test Plan: see follow up diff Reviewers: zasgar, michelle, #engineering Reviewed By: zasgar, #engineering Subscribers: yzhao Differential Revision: https://phab.corp.pixielabs.ai/D2222 GitOrigin-RevId: c880c09
ekmixon · Oct 9, 2019 · 522cdb4 · 522cdb4
1 parent 8538731
commit 522cdb4
Show file tree

Hide file tree

Showing 4 changed files with 209 additions and 2 deletions.
diff --git a/.flake8rc b/.flake8rc
@@ -1,8 +1,9 @@
 [flake8]
 max-line-length = 100
 
-# Function names have to be lower case. This is for GRPC service.
-ignore = N802
+# N802: Function names have to be lower case. This is for GRPC service.
+# E999: Mistaken error see https://github.com/PyCQA/pycodestyle/issues/584
+ignore = N802,E999
 
 [mypy]
 python_version=3.6

diff --git a/tools/licenses/assemble_manual_licenses.py b/tools/licenses/assemble_manual_licenses.py
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+# Script aimed to combining multiple license files into a single one.
+import argparse
+from collections import OrderedDict
+import json
+import urllib2
+
+import re
+
+
+def get_file_at_url(url):
+    try:
+        r = urllib2.urlopen(url)
+    except urllib2.HTTPError, e:
+        print(url)
+        raise e
+    return r.read()
+
+
+def read_file(fname):
+    with open(fname) as f:
+        return f.read()
+
+
+def github_username_and_project(license_url):
+    res = urllib2.urlparse.urlparse(license_url)
+    matches = res.path[1:].split('/')
+    assert len(matches) >= 2, "should be 2 or greater matches not {}".format(
+        len(matches))
+    user = matches[0]
+    project = matches[1]
+    return user, project
+
+
+def raw_github_url_to_github_repo(raw_github_url):
+    ''' Converts raw github url to the containing github repo '''
+    username, project = github_username_and_project(raw_github_url)
+    return 'https://github.com/{username}/{project}'.format(username=username, project=project)
+
+
+def package_license(package_name, license_url, override_license_contents):
+    if package_name not in override_license_contents:
+        contents = get_file_at_url(license_url)
+    else:
+        contents = override_license_contents[package_name]
+
+    username, project = github_username_and_project(license_url)
+    return {
+        "name": project,
+        "content": contents,
+        "url": raw_github_url_to_github_repo(license_url)
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Combines the license files into a map to be used.')
+    parser.add_argument('out_fname', type=str,
+                        help='The markdown file to write.')
+    parser.add_argument('json_file', metavar='N', type=str,
+                        help='the filename of the json mappings to combine')
+    args = parser.parse_args()
+
+    with open(args.json_file) as f:
+        manual_license_map = json.load(f, object_pairs_hook=OrderedDict)
+
+    license_src_key = "license_sources"
+    license_content_key = "license_contents"
+    print(manual_license_map.keys())
+
+    license_sources = manual_license_map[license_src_key]
+    override_license_contents = manual_license_map[license_content_key]
+    license_contents = OrderedDict()
+    for package_name, license_url in license_sources.items():
+        license_contents[package_name] = package_license(
+            package_name, license_url, override_license_contents)
+
+    with open(args.out_fname, 'w') as f:
+        json.dump(license_contents, f, indent=4)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/licenses/combine_bazel_licenses.py b/tools/licenses/combine_bazel_licenses.py
@@ -0,0 +1,85 @@
+#!/usr/bin/python
+# Script aimed to combining multiple license files into a single one.
+import argparse
+from collections import OrderedDict
+import json
+import urllib2
+
+
+def read_file(fname):
+    with open(fname) as f:
+        return f.read()
+
+
+def get_package_name(license_fname):
+    directory_name = license_fname.split('/')[1]
+    split_underscores = directory_name.split('_')
+    # If no underscores, this is the full name.
+    if '_' not in directory_name:
+        return directory_name
+
+    splits = directory_name.split('_')
+    if splits[0] == "com":
+        splits = splits[1:]
+    if splits[0] == "github":
+        splits = splits[1:]
+
+    assert len(splits) >= 2, directory_name
+    return "{}/{}".format(splits[0], '-'.join(splits[1:]))
+
+
+def does404(url):
+    try:
+        r = urllib2.urlopen(url)
+    except urllib2.HTTPError, e:
+        return e.code == 404
+    return False
+
+
+def get_package_url(license_fname):
+    directory_name = license_fname.split('/')[1]
+    split_underscores = directory_name.split('_')
+
+    # If no underscores, then we cannot resolve the url.
+    if '_' not in directory_name:
+        return None
+
+    github_url = "https://github.com/{}".format(
+        get_package_name(license_fname))
+    if not does404(github_url):
+        return github_url
+
+    return None
+
+
+def package_license(license_fname):
+    license_dict = {
+        "name": get_package_name(license_fname),
+        "content": read_file(license_fname)
+    }
+    url = get_package_url(license_fname)
+    if url:
+        license_dict['url'] = url
+
+    return license_dict
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Combines the license files into a map to be used.')
+    parser.add_argument('out_fname', type=str,
+                        help='Out filename.')
+    parser.add_argument('filenames', metavar='N', type=str, nargs='+',
+                        help='the filenames of the license files to combine.')
+    args = parser.parse_args()
+
+    license_mapping = OrderedDict()
+    for license_fname in args.filenames:
+        license_mapping[license_fname] = package_license(license_fname)
+
+    with open(args.out_fname, 'w') as f:
+        json.dump(license_mapping, f, indent=4)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/licenses/manual_license_map.json b/tools/licenses/manual_license_map.json
@@ -0,0 +1,38 @@
+{
+    "license_sources": {
+        "@com_github_google_glog//": "https://raw.githubusercontent.com/google/glog/master/COPYING",
+        "@com_github_gperftools_gperftools//": "https://raw.githubusercontent.com/gperftools/gperftools/master/COPYING",
+        "@com_github_nats_io_natsc//": "https://raw.githubusercontent.com/nats-io/nats.c/master/LICENSE",
+        "@com_google_absl//": "https://raw.githubusercontent.com/abseil/abseil-cpp/master/LICENSE",
+        "@com_google_benchmark//": "https://raw.githubusercontent.com/google/benchmark/master/LICENSE",
+        "@com_google_flatbuffers//": "https://raw.githubusercontent.com/google/flatbuffers/master/LICENSE.txt",
+        "@com_google_googltest//": "https://raw.githubusercontent.com/google/googletest/master/LICENSE",
+        "//third_party:bcc": "https://raw.githubusercontent.com/iovisor/bcc/master/LICENSE.txt",
+        "//third_party:bcc_source": "https://raw.githubusercontent.com/iovisor/bcc/master/LICENSE.txt",
+        "@com_github_nghttp2_nghttp2//": "https://raw.githubusercontent.com/nghttp2/nghttp2/master/COPYING",
+        "@io_bazel_rules_docker//": "https://raw.githubusercontent.com/bazelbuild/rules_docker/master/LICENSE",
+        "@remote_java_tools_darwin//": "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE",
+        "@remote_java_tools_linux//": "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE",
+        "@remote_java_tools_windows//": "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE",
+        "@pl//third_party:clang_tidy_stub": "https://raw.githubusercontent.com/llvm/llvm-project/master/clang-tools-extra/LICENSE.TXT",
+        "@com_llvm_lib//": "https://raw.githubusercontent.com/llvm/llvm-project/master/clang-tools-extra/LICENSE.TXT",
+        "@platforms//": "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE",
+        "@zlib//": "https://raw.githubusercontent.com/madler/zlib/master/zlib.h",
+        "@com_github_madler_zlib//": "https://raw.githubusercontent.com/madler/zlib/master/zlib.h",
+        "@com_github_h2o_picohttpparser//": "https://raw.githubusercontent.com/h2o/picohttpparser/master/picohttpparser.h",
+        "@com_github_google_go_containerregistry//": "https://raw.githubusercontent.com/google/go-containerregistry/master/LICENSE",
+        "@com_github_nanopb_nanopb//": "https://raw.githubusercontent.com/nanopb/nanopb/master/LICENSE.txt",
+        "@com_google_googletest//": "https://raw.githubusercontent.com/google/googletest/master/LICENSE",
+        "gopkg.in/yaml.v2": "https://raw.githubusercontent.com/go-yaml/yaml/v2.2.4/LICENSE",
+        "google.golang.org/grpc": "https://raw.githubusercontent.com/grpc/grpc-go/master/LICENSE",
+        "k8s.io/apimachinery": "https://raw.githubusercontent.com/kubernetes/apimachinery/master/LICENSE",
+        "google.golang.org/api": "https://raw.githubusercontent.com/googleapis/google-api-go-client/master/LICENSE",
+        "k8s.io/api": "https://raw.githubusercontent.com/kubernetes/api/master/LICENSE",
+        "k8s.io/client-go": "https://raw.githubusercontent.com/kubernetes/client-go/master/LICENSE"
+    },
+    "license_contents": {
+        "@zlib//": "version 1.2.11, January 15th, 2017\n\nCopyright (C) 1995-2017 Jean-loup Gailly and Mark Adler\n\nThis software is provided 'as-is', without any express or implied\nwarranty.  In no event will the authors be held liable for any damages\narising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose,\nincluding commercial applications, and to alter it and redistribute it\nfreely, subject to the following restrictions:\n\n1. The origin of this software must not be misrepresented; you must not\n    claim that you wrote the original software. If you use this software\n    in a product, an acknowledgment in the product documentation would be\n    appreciated but is not required.\n2. Altered source versions must be plainly marked as such, and must not be\n    misrepresented as being the original software.\n3. This notice may not be removed or altered from any source distribution.",
+        "@com_github_madler_zlib//": "version 1.2.11, January 15th, 2017\n\nCopyright (C) 1995-2017 Jean-loup Gailly and Mark Adler\n\nThis software is provided 'as-is', without any express or implied\nwarranty.  In no event will the authors be held liable for any damages\narising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose,\nincluding commercial applications, and to alter it and redistribute it\nfreely, subject to the following restrictions:\n\n1. The origin of this software must not be misrepresented; you must not\n    claim that you wrote the original software. If you use this software\n    in a product, an acknowledgment in the product documentation would be\n    appreciated but is not required.\n2. Altered source versions must be plainly marked as such, and must not be\n    misrepresented as being the original software.\n3. This notice may not be removed or altered from any source distribution.",
+        "@com_github_h2o_picohttpparser//": "Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase,\n                        Shigeo Mitsunari\n\nThe software is licensed under either the MIT License (below) or the Perl\nlicense.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to\ndeal in the Software without restriction, including without limitation the\nrights to use, copy, modify, merge, publish, distribute, sublicense, and/or\nsell copies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\nIN THE SOFTWARE."
+    }
+}