build wheels on CI (#619)

This MR includes the commits from #617, so that one should be merged first.

The purpose of this MR is to build and test wheels on RAPIDS CI infrastructure. Previously, cuCIM was using a custom manual wheel building process based on running:
```
./run build_python_package
```
which used `dockcross`/`cibuildwheel`. We would then manually upload the resulting wheel to PyPI using `twine`.

The changes in this MR are to enable automated wheel builds for future RAPIDS releases in a similar way to other RAPIDS projects.

### Misc issues/questions for discussion
- The `build_wheels.sh` script installs the YASM and the openslide libraries via system package managers. This is currently hard-coded in that script rather than using the `dependencies.yaml` mechanism being used elsewhere. Is this reasonable or is there a better way?
- Following cuML's example, I added a separate `ci/release/apply_wheel_modifications.sh` script that updates the pyproject.toml as needed (to fixup version string and the CUDA version strings in the cucim package name and CuPy dependency)
- I updated the cuCIM test suite to make a handful of tests that relied on `imagecodecs` to be skipped if it is not installed. This is because I did not find a pre-compiled wheel for it on aarch64 and when it tried to compile from source instead, it failed due to various missing system packages needed for the compilation.
- Unlike most RAPIDS projects, which seem to use scikit-build, this project performs a separate CMake build via `run build_local` commands and copies the C++ libraries into the Python source tree. The `pyproject.toml` then just does a standard pure-python build, but will package the previously compiled libraries along with the other files.
- The wheel tests are currently taking approximately 35-50 minutes to run. It should be possible to reduce that time by an order of magnitude if the CuPy kernels are cached across runs. Does any RAPIDS projects cache the compiled CuPy kernels in this way? (cached kernels end up in `~/.cupy/kernel_cache/` by default, although an alternative location can be specified via `CUPY_CACHE_DIR`)

Authors:
  - Gregory Lee (https://github.com/grlee77)
  - https://github.com/jakirkham

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Divye Gala (https://github.com/divyegala)
  - Ray Douglass (https://github.com/raydouglass)
  - https://github.com/jakirkham

URL: #619

.github/workflows/build.yaml

@@ -66,3 +66,22 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-build:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel.sh
+  wheel-publish:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cucim

.github/workflows/pr.yaml

@@ -17,6 +17,8 @@ jobs:
       - conda-python-build
       - conda-python-tests
       - docs-build
+      - wheel-build
+      - wheel-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-23.12
   checks:
@@ -50,3 +52,17 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
+  wheel-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12
+    with:
+      build_type: pull-request
+      script: ci/build_wheel.sh
+  wheel-tests:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12
+    with:
+      build_type: pull-request
+      script: ci/test_wheel.sh

.github/workflows/test.yaml

@@ -22,3 +22,12 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      script: ci/test_wheel.sh

.pre-commit-config.yaml

@@ -14,7 +14,6 @@ repos:
     rev: 23.10.1
     hooks:
       - id: black
-        files: (python|legate)/.*
         args: ["--config", "python/cucim/pyproject.toml"]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.1.3

CMakeLists.txt

@@ -24,6 +24,8 @@ cmake_minimum_required(VERSION 3.18)
 # Set VERSION and BUILD
 unset(VERSION CACHE)
 file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION)
+# strip alpha version info
+string(REGEX REPLACE "a.*$" "" VERSION ${VERSION})
 set(PROJECT_VERSION_BUILD dev)
 
 # Append local cmake module path

CONTRIBUTING.md

@@ -263,19 +263,21 @@ conda install -y -c ${CONDA_BLD_DIR} -c conda-forge \
 
 ## Building a package (for distribution. Including a wheel package for pip)
 
-**Build**
-
-You can execute the following command to build a wheel file for pip.
+**Wheel Build**
 
+If you are using CUDA 12.x, please update pyproject.toml as follows before building the wheel
 ```bash
-./run build_package
+sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cucim/pyproject.toml
 ```
+This will switch the CuPy dependency to one based on CUDA 12.x instead of 11.x.
 
-The command would use `./temp` folder as a local build folder and build a distribution package into `dist` folder using [dockcross](https://github.com/dockcross/dockcross)'s manylinux2014 docker image.
+The wheel can then be built using:
 
-`./run build_package` will reuse local `./temp` folder to reduce the build time.
+```bash
+python -m pip wheel python/cucim/ -w dist -vvv --no-deps --disable-pip-version-check
+```
 
-If C++ code or dependent packages are updated so the build is failing somehow, please retry it after deleting the `temp` folder under the repository root.
+**Note:** It is possible to build the wheel in this way even without compiling the C++ library first, but in that case the `cucim.clara` module will not be importable.
 
 **Install**
 

VERSION

@@ -1 +1 @@
-23.12.00
+23.12.00a56

ci/build_cpp.sh

@@ -9,8 +9,10 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+version=$(rapids-generate-version)
+
 rapids-logger "Begin cpp build"
 
-rapids-conda-retry mambabuild conda/recipes/libcucim
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcucim
 
 rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -9,13 +9,24 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+package_name="cucim"
+package_dir="python/cucim"
+package_src_dir="${package_dir}/src/${package_name}"
+
+version=$(rapids-generate-version)
+
+commit=$(git rev-parse HEAD)
+
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py"
+
 rapids-logger "Begin py build"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
-rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/cucim

ci/build_wheel.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="cucim"
+package_dir="python/cucim"
+package_src_dir="${package_dir}/src/${package_name}"
+
+CMAKE_BUILD_TYPE="release"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# Patch project metadata files to include the CUDA version suffix and version override.
+pyproject_file="${package_dir}/pyproject.toml"
+
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+# update package name to have the cuda suffix
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py"
+
+if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then
+    # change pyproject.toml to use CUDA 12.x version of cupy
+    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
+fi
+
+# Install pip build dependencies (not yet using pyproject.toml)
+rapids-dependency-file-generator \
+  --file_key "py_build" \
+  --output "requirements" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt
+pip install -r build_requirements.txt
+
+# First build the C++ lib using CMake via the run script
+./run build_local all ${CMAKE_BUILD_TYPE}
+
+cd "${package_dir}"
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

ci/release/update-version.sh

@@ -34,12 +34,9 @@ function sed_runner() {
 sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py
 sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py
 
-sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cucim/pyproject.toml
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" VERSION
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION
-sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cucim/src/cucim/__init__.pyi
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION
+# Centralized version file update
+echo "${NEXT_FULL_TAG}" > VERSION
+
 sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md
 sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md
 sed_runner "s#cucim.kit.cuslide@${CURRENT_LONG_TAG}.so#cucim.kit.cuslide@${NEXT_FULL_TAG}.so#g" python/cucim/docs/getting_started/index.md

ci/test_wheel.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -eou pipefail
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install $(echo ./dist/cucim*.whl)[test]
+
+# Run smoke tests for aarch64 pull requests
+if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
+    python ./ci/wheel_smoke_test.py
+else
+    # TODO: revisit enabling imagecodecs package during testing
+    python -m pytest ./python/cucim
+fi

ci/wheel_smoke_test.py

@@ -0,0 +1,21 @@
+import cupy as cp
+
+import cucim
+import cucim.skimage
+
+
+if __name__ == "__main__":
+    # verify that all top-level modules are available
+    assert cucim.is_available("clara")
+    assert cucim.is_available("core")
+    assert cucim.is_available("skimage")
+
+    # generate a synthetic image and apply a filter
+    img = cucim.skimage.data.binary_blobs(length=512, n_dim=2)
+    assert isinstance(img, cp.ndarray)
+    assert img.dtype.kind == "b"
+    assert img.shape == (512, 512)
+
+    eroded = cucim.skimage.morphology.binary_erosion(
+        img, cp.ones((3, 3), dtype=bool)
+    )

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -24,13 +24,15 @@ dependencies:
 - libnvjpeg-dev=11.6.0.55
 - libnvjpeg=11.6.0.55
 - libwebp-base
+- matplotlib-base
 - nbsphinx
 - ninja
 - numpy>=1.21.3
 - numpydoc
 - nvcc_linux-64=11.8
 - openslide-python>=1.1.2
 - pip
+- pooch>=1.6.0
 - pre-commit
 - psutil>=5.8.0
 - pydata-sphinx-theme

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -24,12 +24,14 @@ dependencies:
 - libnvjpeg-dev
 - libnvjpeg-static
 - libwebp-base
+- matplotlib-base
 - nbsphinx
 - ninja
 - numpy>=1.21.3
 - numpydoc
 - openslide-python>=1.1.2
 - pip
+- pooch>=1.6.0
 - pre-commit
 - psutil>=5.8.0
 - pydata-sphinx-theme

conda/recipes/cucim/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2021-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set py_version = environ['CONDA_PY'] %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
@@ -12,7 +12,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

conda/recipes/libcucim/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2021-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
 {% set date_string = environ['RAPIDS_DATE_STRING'] %}
@@ -11,7 +11,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

cpp/cmake/deps/openslide.cmake

@@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide)
         set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so")
     elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so)
         set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so)
-    else () # CentOS 6
+    elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so)
+        set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so)
+    else () # CentOS (x86_64)
         set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so)
     endif ()
 

cpp/plugins/cucim.kit.cumed/CMakeLists.txt

@@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18)
 
 # Set VERSION
 unset(VERSION CACHE)
-file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION)
+file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION VERSION)
+# strip alpha version info
+string(REGEX REPLACE "a.*$" "" VERSION ${VERSION})
 
 # Append local cmake module path
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")

cpp/plugins/cucim.kit.cuslide/CMakeLists.txt

@@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18)
 
 # Set VERSION
 unset(VERSION CACHE)
-file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION)
+file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION VERSION)
+# strip alpha version info
+string(REGEX REPLACE "a.*$" "" VERSION ${VERSION})
 
 # Append local cmake module path
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")

cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake

@@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide)
         set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so")
     elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so)
         set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so)
-    else () # CentOS 6
+    elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so)
+        set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so)
+    else () # CentOS (x86_64)
         set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so)
     endif ()