diff --git a/check_releasepy.bash b/check_releasepy.bash index 921153017..9d5c100c4 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -1,9 +1,22 @@ #!/bin/bash -e -test_dir=$(mktemp -d) -mkdir -p ${test_dir}/{focal,jammy,ubuntu}/debian -export _RELEASEPY_TEST_RELEASE_REPO=${test_dir} export _RELEASEPY_DEBUG=1 +test_dir=$(mktemp -d) +export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release" +mkdir -p ${_RELEASEPY_TEST_RELEASE_REPO}/{focal,jammy,ubuntu}/debian +export _RELEASEPY_TEST_SOURCE_REPO="${test_dir}/src" +mkdir -p ${_RELEASEPY_TEST_SOURCE_REPO} +# Fake packages.xml to make the vendor package script happy +cat > "${_RELEASEPY_TEST_SOURCE_REPO}/package.xml" <<-EOF + + + gz-foo + 0.0.0 + test + Testing maintainer + Foo License + +EOF exec_releasepy_test() { @@ -12,7 +25,7 @@ exec_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - gz-foo 1.2.3 token ${test_params}"" + gz-foo 1.2.3 token ${test_params} } exec_ignition_releasepy_test() @@ -22,7 +35,7 @@ exec_ignition_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-foo 1.2.3 token ${test_params}"" + ign-foo 1.2.3 token ${test_params} } exec_ignition_gazebo_releasepy_test() @@ -32,7 +45,18 @@ exec_ignition_gazebo_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-gazebo 1.2.3 token ${test_params}"" + ign-gazebo 1.2.3 token ${test_params} +} + +exec_releasepy_with_real_gz() +{ + gz_pkg=${1} major_version=${2} + ./release.py \ + --dry-run \ + --no-sanity-checks \ + --source-repo-uri http://github.com/gazebosim/gz-common \ + --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \ + "${gz_pkg}" "${major_version}.x.y" token } expect_job_run() @@ -73,7 +97,26 @@ expect_param() echo "${param} not found in test output" exit 1 fi +} + +expect_vendor_repo() +{ + output="${1}" repo="${2}" + + if ! grep -q "Github ${repo}" <<< "${output}"; then + echo "${repo} not found in test output" + exit 1 + fi +} + +expect_no_vendor() +{ + output="${1}" + if grep -q 'in ROS 2' <<< "${output}"; then + echo "ROS 2 string found in output" + exit 1 + fi } source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") @@ -81,6 +124,7 @@ expect_job_run "${source_repo_uri_test}" "gz-foo-source" expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder" expect_number_of_jobs "${source_repo_uri_test}" "1" expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" +expect_no_vendor "${source_repo_uri_test}" # non existing package source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder" @@ -88,6 +132,7 @@ expect_job_run "${source_tarball_uri_test}" "generic-release-homebrew_pull_reque expect_job_not_run "${source_tarball_uri_test}" "gz-foo-source" expect_number_of_jobs "${source_tarball_uri_test}" "7" expect_param "${source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" +expect_no_vendor "${source_tarball_uri_test}" nightly_test=$(exec_releasepy_test "--nightly-src-branch my-nightly-branch3 --upload-to-repo nightly") expect_job_run "${nightly_test}" "gz-foo-debbuilder" @@ -95,6 +140,7 @@ expect_job_not_run "${nightly_test}" "generic-release-homebrew_pull_request_upda expect_job_not_run "${nightly_test}" "gz-foo-source" expect_number_of_jobs "${nightly_test}" "2" expect_param "${nightly_test}" "SOURCE_TARBALL_URI=my-nightly-branch3" +expect_no_vendor "${nightly_test}" bump_linux_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz --only-bump-revision-linux -r 2") expect_job_run "${bump_linux_test}" "gz-foo-debbuilder" @@ -102,6 +148,7 @@ expect_job_not_run "${bump_linux_test}" "generic-release-homebrew_pull_request_u expect_job_not_run "${bump_linux_test}" "gz-foo-source" expect_number_of_jobs "${bump_linux_test}" "6" expect_param "${bump_linux_test}" "RELEASE_VERSION=2" +expect_no_vendor "${bump_linux_test}" ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${ignition_test}" "gz-foo-source" @@ -128,3 +175,12 @@ expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7" expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz" expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo" expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo" + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-fuel-tools 9) +expect_vendor_repo "${ros_vendor_test}" gazebo-release/gz_fuel_tools_vendor + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-cmake 2) +expect_no_vendor "${ros_vendor_test}" + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-ionic 3) +expect_no_vendor "${ros_vendor_test}" diff --git a/release.py b/release.py index 265796308..262851709 100755 --- a/release.py +++ b/release.py @@ -2,6 +2,7 @@ from __future__ import print_function from argparse import RawTextHelpFormatter +from typing import Tuple import subprocess import sys import tempfile @@ -10,6 +11,7 @@ import urllib.request import argparse import shutil +import venv USAGE = 'release.py ' try: @@ -22,6 +24,7 @@ LINUX_DISTROS = ['ubuntu', 'debian'] SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' +ROS_VENDOR = {'harmonic': 'jazzy', 'ionic': 'rolling'} OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing none" @@ -48,6 +51,10 @@ class ErrorNoOutput(Exception): pass +class ErrorAlreadyExists(Exception): + pass + + def error(msg): print("\n !! " + msg + "\n") sys.exit(1) @@ -151,6 +158,9 @@ def parse_args(argv): parser.add_argument('--only-bump-revision-linux', dest='bump_rev_linux_only', action='store_true', default=False, help='Bump only revision number. Do not upload new tarball.') + parser.add_argument('--only-bump-ros-vendor-package', dest='bump_ros_vendor_only', + action='store_true', default=False, + help='Only process the ROS vendor package (if any).') args = parser.parse_args() @@ -375,13 +385,13 @@ def discover_distros(repo_dir): return distro_arch_list -def check_call(cmd, ignore_dry_run=False): +def check_call(cmd, ignore_dry_run=False, cwd=None): if DRY_RUN and not ignore_dry_run: print_only_dbg('Dry-run running:\n %s\n' % (' '.join(cmd))) return b'', b'' else: print_only_dbg('Running:\n %s' % (' '.join(cmd))) - po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) out, err = po.communicate() if po.returncode != 0: # bitbucket for the first one, github for the second @@ -391,6 +401,8 @@ def check_call(cmd, ignore_dry_run=False): raise ErrorNoPermsRepo() if b"abort: no username supplied" in err: raise ErrorNoUsernameSupplied() + if b"already exists:" in err: + raise ErrorAlreadyExists() if not out and not err: # assume that call is only for getting return code raise ErrorNoOutput() @@ -515,9 +527,170 @@ def display_help_job_chain_for_source_calls(args): f'{releasepy_check_url}') +def get_collections_for_package(package_name, version) -> list: + script_directory = os.path.dirname(os.path.abspath(sys.argv[0])) + helper_script = f'{script_directory}/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py' + collection_yaml = f'{script_directory}/jenkins-scripts/dsl/gz-collections.yaml' + cmd = [helper_script, + get_canonical_package_name(package_name), + version, + collection_yaml] + try: + _out, _err = check_call(cmd, IGNORE_DRY_RUN) + except ErrorNoOutput: + # no output is a valid result + _out = b"" + _err = "" + else: + if _err: + print(f"An error happened running get_collections_from_package_and_version: {_err}") + sys.exit(1) + + collection_list = _out.decode().strip().split(' ') + return collection_list + + +def get_vendor_github_repo(package_name) -> str: + canonical_name = get_canonical_package_name(package_name) + return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor" + + +def get_vendor_repo_url(package_name) -> str: + # Clone needs ssh for real pushing operations. In simulation prefer https to avoid + # unexpected pushes and facilitate testing + protocol = 'https://github.com/' if DRY_RUN else 'ssh://git@github.com:' + return f"{protocol}{get_vendor_github_repo(package_name)}" + + +def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> Tuple[str, str, str]: + gz_vendor_tool = os.path.join(ws_dir, "gz_vendor") + # Create virtualenv for vendor dependencies + venv_dir = os.path.join(ws_dir, "venv") + venv.create(venv_dir, system_site_packages=True, with_pip=True) + subprocess.run([os.path.join(venv_dir, 'bin', 'pip3'), 'install', '-q', + 'jinja2==3.1.2', + 'catkin_pkg==1.0.0']) + cmd = ['git', 'clone', '-q', + 'https://github.com/gazebo-tooling/gz_vendor/', + gz_vendor_tool] + _, _err_tool = check_call(cmd, IGNORE_DRY_RUN) + gz_vendor_repo = os.path.join(ws_dir, 'gz_vendor_repo') + cmd = ['git', 'clone', '-q', + get_vendor_repo_url(package_name), + gz_vendor_repo] + _, _err_repo = check_call(cmd, IGNORE_DRY_RUN) + if _err_tool or _err_repo: + print("Problems with cloning vendor and tool repos:") + print(f"{_err_tool} {_err_repo}") + sys.exit(1) + + return gz_vendor_tool, gz_vendor_repo, venv_dir + + +def execute_update_vendor_package_tool(vendor_tool_path, + vendor_repo_path, + vendor_venv) -> None: + # The source repository when releasing matches the + src_repo = os.getcwd() + try: + src_repo = os.environ['_RELEASEPY_TEST_SOURCE_REPO'] + except KeyError: + pass + + run_cmd = [os.path.join(vendor_venv, 'bin', 'python3'), + f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", + f"{os.path.join(src_repo, 'package.xml')}", + '--output_dir', vendor_repo_path] + _, _err_run = check_call(run_cmd, IGNORE_DRY_RUN) + if _err_run: + print("Problems running the create_vendor_package.py script:") + print(_err_run.decode()) + sys.exit(1) + + +def create_pr_for_vendor_package(args, repo_path, base_branch) -> str: + cmd_diff = ['git', "-C", repo_path, 'diff'] + _out, _ = check_call(cmd_diff, IGNORE_DRY_RUN) + if not _out.decode(): + return 'vendor tool did not produce any change, avoid the PR' + + branch_name = f'releasepy/{args.version}' + vendor_repo = get_vendor_repo_url(args.package) + branch_cmd = ['git', "-C", repo_path, + 'checkout', '-b', branch_name] + _, _ = check_call(branch_cmd, IGNORE_DRY_RUN) + commit_cmd = ['git', "-C", repo_path, + 'commit', + '-m', f'Bump version to {args.version}', + '--all'] + _, _ = check_call(commit_cmd) + push_cmd = ['git', "-C", repo_path, + 'push', '--force', + vendor_repo, branch_name] + _, _ = check_call(push_cmd) + pr_cmd = ['gh', 'pr', 'create', + '--base', base_branch, + '--head', branch_name, + '--title', f'Bump version to {args.version}', + '--body', 'PR automatically created by release.py'] + try: + _out, _err = check_call(pr_cmd, cwd=repo_path) + except ErrorAlreadyExists: + return f'there is already a PR for the branch: {branch_name} .'\ + 'Please check it out manuallly.' + + if _err: + print("Problems creating the PR for the vendor package:") + print(_err.decode()) + sys.exit(1) + + if DRY_RUN: + return ' (skipped the creation on --dry-run)' + + return _out.decode() + + +def create_pr_in_gz_vendor_repo(args, ros_distro) -> str: + pr_msg = '' + with tempfile.TemporaryDirectory() as ws_dir: + ws_dir = tempfile.mkdtemp() + # Prepare the temporary workspace + vendor_tool_path, vendor_repo_path, venv_dir = \ + prepare_vendor_pr_temp_workspace(args.package, ws_dir) + # Run updating script on the temporary workspace + execute_update_vendor_package_tool( + vendor_tool_path, vendor_repo_path, venv_dir) + # Commits and PR creation + pr_msg = create_pr_for_vendor_package( + args, vendor_repo_path, ros_distro) + + return pr_msg + + +def process_ros_vendor_package(args): + print("ROS vendor packages that can be updated:") + if args.package.replace('gz-','') in ROS_VENDOR: + print(" - There are no gz metapackages in ROS") + return + for collection in get_collections_for_package(args.package, + args.version): + if collection in ROS_VENDOR: + ros_distro = ROS_VENDOR[collection] + print(f" * Github {get_vendor_github_repo(args.package)} " + f"part of {collection} in ROS 2 {ros_distro}") + print(" + Preparing a PR: ", end='', flush=True) + pr_url = create_pr_in_gz_vendor_repo(args, ros_distro) + print(pr_url) + + def go(argv): args = parse_args(argv) + # If only the process of ROS vendor package is set, just do it + if args.bump_ros_vendor_only: + process_ros_vendor_package(args) + sys.exit(0) + # Default to release 1 if not present if not args.release_version: args.release_version = 1 @@ -640,7 +813,8 @@ def go(argv): 'Source', args.version) display_help_job_chain_for_source_calls(args) - + # Process the possible update of an associated ROS vendor package + process_ros_vendor_package(args) if __name__ == '__main__': go(sys.argv)