Merge topic 'ci-sdk-upload'

d6c30cd Use the `tar` that comes with cmake
6b26668 Use xz format
3cc15d0 Fix new release notes
f269fdb Rename wheel_sdk => create_wheel_sdk_archive.py
06fcd41 Set VTK_INSTALL_SDK for wheel builds
97c97d4 Update wheel SDK tarball name
f0da391 Add release notes for wheel sdks
5e4eb95 Use python script to make sdk tarball
...

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !9144

.gitlab-ci.yml

@@ -448,6 +448,65 @@ wheel:vtk:upload:
         - wheel-windows-python310-x86_64:build
         - wheel-windows-python310-x86_64:test
 
+wheel-sdk:upload:
+    extends:
+        - .rsync_upload_wheel_sdk
+        - .weekly_upload_only
+    dependencies:
+        - wheel-linux-python36-x86_64:build
+        - wheel-linux-python37-x86_64:build
+        - wheel-linux-python38-x86_64:build
+        - wheel-linux-python39-x86_64:build
+        - wheel-linux-python310-x86_64:build
+        - wheel-macos-python36-x86_64:build
+        - wheel-macos-python37-x86_64:build
+        - wheel-macos-python38-x86_64:build
+        - wheel-macos-python39-arm64:build
+        - wheel-macos-python39-x86_64:build
+        - wheel-macos-python310-arm64:build
+        - wheel-macos-python310-x86_64:build
+        - wheel-windows-python36-x86_64:build
+        - wheel-windows-python37-x86_64:build
+        - wheel-windows-python38-x86_64:build
+        - wheel-windows-python39-x86_64:build
+        - wheel-windows-python310-x86_64:build
+    needs:
+        - wheel-linux-python36-x86_64:build
+        - wheel-linux-python37-x86_64:build
+        - wheel-linux-python37-x86_64:test
+        - wheel-linux-python38-x86_64:build
+        - wheel-linux-python38-x86_64:test
+        - wheel-linux-python39-x86_64:build
+        - wheel-linux-python39-x86_64:test
+        - wheel-linux-python310-x86_64:build
+        - wheel-linux-python310-x86_64:test
+        - wheel-macos-python36-x86_64:build
+        - wheel-macos-python36-x86_64:test
+        - wheel-macos-python37-x86_64:build
+        - wheel-macos-python37-x86_64:test
+        - wheel-macos-python38-x86_64:build
+        - wheel-macos-python38-x86_64:test
+        - wheel-macos-python39-arm64:build
+        - wheel-macos-python39-arm64:test
+        - wheel-macos-python39-x86_64:build
+        - wheel-macos-python39-x86_64:test
+        - wheel-macos-python310-arm64:build
+        - wheel-macos-python310-arm64:test
+        - wheel-macos-python310-x86_64:build
+        - wheel-macos-python310-x86_64:test
+        - wheel-windows-python36-x86_64:build
+        - wheel-windows-python37-x86_64:build
+        - wheel-windows-python37-x86_64:test
+        - wheel-windows-python38-x86_64:build
+        - wheel-windows-python38-x86_64:test
+        - wheel-windows-python39-x86_64:build
+        - wheel-windows-python39-x86_64:test
+        - wheel-windows-python310-x86_64:build
+        - wheel-windows-python310-x86_64:test
+    variables:
+        RSYNC_SOURCE: build/vtk-wheel-sdk-*.tar.xz
+        RSYNC_DESTINATION: wheel-sdks/
+
 ## Static analysis builds
 
 fedora34-tidy:build:

.gitlab/artifacts.yml

@@ -108,6 +108,7 @@
             - wheel_output.log
 
             - build/dist/*.whl
+            - build/vtk-wheel-sdk-*.tar.xz
 
 .cmake_release_artifacts:
     artifacts:

.gitlab/ci/configure_wheel.cmake

@@ -21,6 +21,8 @@ if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "macos")
 endif ()
 
 set(VTK_WHEEL_BUILD ON CACHE BOOL "")
+set(VTK_INSTALL_SDK ON CACHE BOOL "")
+
 set(CMAKE_PREFIX_PATH "$ENV{PYTHON_PREFIX}" CACHE STRING "")
 set(Python3_EXECUTABLE "$ENV{PYTHON_PREFIX}/${python_subdir}python$ENV{PYTHON_VERSION_SUFFIX}" CACHE FILEPATH "")
 # We always want the Python specified here, not the system one.

.gitlab/ci/create_wheel_sdk_archive.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+import subprocess
+
+dist_path = Path('dist')
+install_path = Path('install')
+
+wheel_file = next(dist_path.glob('*.whl'))
+sdk_name = wheel_file.stem.replace('vtk-', 'vtk-wheel-sdk-')
+tarball_name = f'{sdk_name}.tar.xz'
+
+install_path.rename(sdk_name)
+try:
+    cmd = ['cmake', '-E', 'tar', 'cJf', tarball_name, sdk_name]
+    subprocess.check_call(cmd)
+finally:
+    Path(sdk_name).rename(install_path)

.gitlab/issue_templates/new-release.md

@@ -67,8 +67,10 @@ git commit -m 'Update version number to @VERSION@@RC@' CMake/vtkVersion.cmake
     - [ ] Source (from the `build:source` CI job in the tag pipeline)
     - [ ] Documentation (from the `release-prep:documentation` CI job in the tag pipeline)
     - [ ] Wheels (from the `build:wheel-*` jobs).
+    - [ ] Wheel SDKs (from the `build:wheel-*` jobs (`vtk-wheel-sdk-*.tar.xz`)).
   - Upload assets to `vtk.org`
-    - [ ] `rsync -rptv $tarballs $wheels user@host:vtk_release/@MAJOR@.@MINOR@/`
+    - [ ] `rsync -rptv $tarballs $wheels $wheel_sdks user@host:vtk_release/@MAJOR@.@MINOR@/`
+    - [ ] `rsync -rptv $wheel_sdks user@host:wheel-sdks/`
   - [ ] Update `vtk.org/download` with the new release (email
         `comm@kitware.com` with filenames and hashes)
   - Software process updates (these can all be done independently)

.gitlab/os-linux.yml

@@ -12,6 +12,7 @@
 
     variables:
         GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci
+        VTK_INSTALL: 1
 
 .wheel_linux36_x86_64:
     extends: .manylinux2014_x86_64
@@ -394,6 +395,8 @@
         - auditwheel repair dist/*.whl >> $GIT_CLONE_PATH/wheel_output.log
         - mv -v dist/ dist-orig/
         - mv -v wheelhouse/ dist/
+        - "$PYTHON_PREFIX/bin/python $GIT_CLONE_PATH/.gitlab/ci/create_wheel_sdk_archive.py"
+
     interruptible: true
 
 .cmake_test_wheel_linux:

.gitlab/os-macos.yml

@@ -12,6 +12,9 @@
 .wheel_macos_arm64:
     extends: .macos
 
+    variables:
+        VTK_INSTALL: 1
+
 .wheel_macos39_arm64:
     extends: .wheel_macos_arm64
 
@@ -31,6 +34,9 @@
 .wheel_macos_x86_64:
     extends: .macos
 
+    variables:
+        VTK_INSTALL: 1
+
 .wheel_macos36_x86_64:
     extends: .wheel_macos_x86_64
 
@@ -213,6 +219,7 @@
         - "$PYTHON_PREFIX/bin/python$PYTHON_VERSION_SUFFIX -m venv venv"
         - venv/bin/pip install wheel
         - venv/bin/python3 setup.py bdist_wheel
+        - venv/bin/python3 $GIT_CLONE_PATH/.gitlab/ci/create_wheel_sdk_archive.py
         # XXX: `delocate` can't handle `@loader_path` or do recursive
         # dependency scanning. Ignore it and just do it manually because
         # apparently VTK is the most complicated wheel in the world.

.gitlab/os-windows.yml

@@ -14,6 +14,7 @@
         VCVARSALL: "${VS160COMNTOOLS}\\..\\..\\VC\\Auxiliary\\Build\\vcvarsall.bat"
         VCVARSPLATFORM: "x64"
         VCVARSVERSION: "14.28.29333"
+        VTK_INSTALL: 1
 
 .wheel_windows36_x86_64:
     extends: .wheel_windows_x86_64
@@ -255,6 +256,7 @@
         - . .\venv\Scripts\Activate.ps1
         - pip install wheel
         - python setup.py bdist_wheel
+        - python "$env:GIT_CLONE_PATH\.gitlab\ci\create_wheel_sdk_archive.py"
         # XXX(delvewheel): It is finding 32bit Visual Studio libraries and
         # bailing instead of continuing to search for 64bit versions of the
         # DLL.

.gitlab/upload.yml

@@ -17,6 +17,24 @@
         - ssh-keygen -y -f $RSYNC_BINARY_KEY > $RSYNC_BINARY_KEY.pub
         - rsync -tv --recursive -e "ssh -i $RSYNC_BINARY_KEY -o StrictHostKeyChecking=no" build/$SRC_SUBDIR kitware@web.kitware.com:$RSYNC_DESTINATION/
 
+.rsync_upload_wheel_sdk:
+    image: "fedora:34"
+    stage: upload
+    tags:
+        - docker
+        - linux
+        - build
+    environment:
+        name: rsync-upload-wheel-sdk
+
+    script:
+        - ls build/
+        - dnf install -y --setopt=install_weak_deps=False rsync openssh-clients
+        - chmod 400 $RSYNC_WHEEL_SDK_KEY
+        - ssh-keygen -y -f $RSYNC_WHEEL_SDK_KEY > $RSYNC_WHEEL_SDK_KEY.pub
+        - rsync -tv --recursive -e "ssh -i $RSYNC_WHEEL_SDK_KEY -o StrictHostKeyChecking=no" $RSYNC_SOURCE kitware@vtk.org:$RSYNC_DESTINATION/
+
+
 .pypi_upload:
     image: "fedora:34"
     stage: upload

Documentation/release/dev/python-wheel-sdks.md

@@ -0,0 +1,4 @@
+## Python Wheel SDKs
+
+SDKs (install trees) for each wheel are now uploaded as artifacts, and
+weekly SDKs for each wheel are uploaded to vtk.org.