From 07289a1ce926a2f5dbd1496f540923c6f4e3e8d6 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Wed, 11 May 2022 15:48:29 +0200 Subject: [PATCH] Use conda to build python packages during GPU tests (#480) This PR convert the `from sources` build we are doing in GPU test job to a `conda build`. This is done for the following reasons: - This is required step to improve the Ops CI/CD setup to a more convenient pipeline - This is required to start using `conda` compilers and `mamba` to build RAPIDS packages - This prevent us from manually managing and installing the dependencies in GPU job - This ensure the packages can be installed - This ensure the tests are running and working against the package content and not the build results. Currently the Python packages are not tested. This may increase the global pipeline time, but the usage of `mamba` should resolve this as `mamba` is faster than `conda` to build packages Authors: - Jordan Jacobelli (https://github.com/Ethyling) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Brad Rees (https://github.com/BradReesWork) URL: https://github.com/rapidsai/cusignal/pull/480 --- ci/gpu/build.sh | 18 ++++++------------ conda/recipes/cusignal/meta.yaml | 2 +- python/cusignal/test/__init__.py | 12 ------------ python/cusignal/test/test_acoustics.py | 2 +- python/cusignal/test/test_bsplines.py | 2 +- python/cusignal/test/test_convolution.py | 2 +- python/cusignal/test/test_filter_design.py | 2 +- python/cusignal/test/test_filtering.py | 2 +- python/cusignal/test/test_peak_finding.py | 2 +- python/cusignal/test/test_radartools.py | 2 +- python/cusignal/test/test_spectral_analysis.py | 2 +- python/cusignal/test/test_waveforms.py | 2 +- python/cusignal/test/test_wavelets.py | 2 +- python/cusignal/test/test_windows.py | 2 +- python/cusignal/testing/__init__.py | 0 python/cusignal/{test => testing}/utils.py | 0 16 files changed, 18 insertions(+), 36 deletions(-) delete mode 100644 python/cusignal/test/__init__.py create mode 100644 python/cusignal/testing/__init__.py rename python/cusignal/{test => testing}/utils.py (100%) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index bf5c1ed2..2aef7423 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -24,6 +24,7 @@ export HOME="$WORKSPACE" cd "$WORKSPACE" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` +unset GIT_DESCRIBE_TAG ################################################################################ # SETUP - Check environment @@ -38,14 +39,6 @@ nvidia-smi gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids -gpuci_conda_retry install -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge \ - cudatoolkit=${CUDA_REL} \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rapids-notebook-env=$MINOR_VERSION." - -# https://docs.rapids.ai/maintainers/depmgmt/ -# conda remove -f rapids-build-env rapids-notebook-env -# conda install "your-pkg=1.0.0" gpuci_logger "Check versions" python --version @@ -61,8 +54,11 @@ conda list --show-channel-urls # BUILD - Build cusignal ################################################################################ -gpuci_logger "Build cusignal" -"$WORKSPACE/build.sh" clean cusignal +gpuci_logger "Build and install cusignal" +cd "${WORKSPACE}" +CONDA_BLD_DIR="${WORKSPACE}/.conda-bld" +gpuci_conda_retry build --croot "${CONDA_BLD_DIR}" conda/recipes/cusignal --python=${PYTHON} +gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" cusignal ################################################################################ # TEST - Run GoogleTest and py.tests for cusignal @@ -89,5 +85,3 @@ pytest --cache-clear --junitxml="$WORKSPACE/junit-cusignal.xml" -v -s -m "not cp python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log return ${EXITCODE} - - diff --git a/conda/recipes/cusignal/meta.yaml b/conda/recipes/cusignal/meta.yaml index 9247c24b..b8cceca7 100644 --- a/conda/recipes/cusignal/meta.yaml +++ b/conda/recipes/cusignal/meta.yaml @@ -9,7 +9,7 @@ package: version: {{ version }} source: - path: ../../.. + git_url: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/python/cusignal/test/__init__.py b/python/cusignal/test/__init__.py deleted file mode 100644 index 64ea1011..00000000 --- a/python/cusignal/test/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019-2020, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/cusignal/test/test_acoustics.py b/python/cusignal/test/test_acoustics.py index fed6aa07..336e952b 100644 --- a/python/cusignal/test/test_acoustics.py +++ b/python/cusignal/test/test_acoustics.py @@ -16,7 +16,7 @@ import pytest import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_bsplines.py b/python/cusignal/test/test_bsplines.py index 7a0ff2b5..6d2c8e08 100644 --- a/python/cusignal/test/test_bsplines.py +++ b/python/cusignal/test/test_bsplines.py @@ -16,7 +16,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_convolution.py b/python/cusignal/test/test_convolution.py index 008fdb05..9e3c46d0 100644 --- a/python/cusignal/test/test_convolution.py +++ b/python/cusignal/test/test_convolution.py @@ -16,7 +16,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_filter_design.py b/python/cusignal/test/test_filter_design.py index 6cc20584..c875f1c9 100644 --- a/python/cusignal/test/test_filter_design.py +++ b/python/cusignal/test/test_filter_design.py @@ -16,7 +16,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_filtering.py b/python/cusignal/test/test_filtering.py index bd1c1d61..a937db2e 100644 --- a/python/cusignal/test/test_filtering.py +++ b/python/cusignal/test/test_filtering.py @@ -18,7 +18,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_peak_finding.py b/python/cusignal/test/test_peak_finding.py index 5d6b993c..a3c7c76a 100644 --- a/python/cusignal/test/test_peak_finding.py +++ b/python/cusignal/test/test_peak_finding.py @@ -17,7 +17,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_radartools.py b/python/cusignal/test/test_radartools.py index f05f2fb6..f690e96a 100644 --- a/python/cusignal/test/test_radartools.py +++ b/python/cusignal/test/test_radartools.py @@ -3,7 +3,7 @@ from numpy import vectorize from cusignal.radartools import ca_cfar, cfar_alpha -from cusignal.test.utils import array_equal +from cusignal.testing.utils import array_equal class TestCaCfar: diff --git a/python/cusignal/test/test_spectral_analysis.py b/python/cusignal/test/test_spectral_analysis.py index 7d599b0c..85e179e1 100644 --- a/python/cusignal/test/test_spectral_analysis.py +++ b/python/cusignal/test/test_spectral_analysis.py @@ -17,7 +17,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_waveforms.py b/python/cusignal/test/test_waveforms.py index 1ec81879..9d1c9801 100644 --- a/python/cusignal/test/test_waveforms.py +++ b/python/cusignal/test/test_waveforms.py @@ -17,7 +17,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_wavelets.py b/python/cusignal/test/test_wavelets.py index b350f1c1..91724c4e 100644 --- a/python/cusignal/test/test_wavelets.py +++ b/python/cusignal/test/test_wavelets.py @@ -17,7 +17,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/test/test_windows.py b/python/cusignal/test/test_windows.py index 315c9fd9..2f23841e 100644 --- a/python/cusignal/test/test_windows.py +++ b/python/cusignal/test/test_windows.py @@ -16,7 +16,7 @@ from scipy import signal import cusignal -from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal +from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal gpubenchmark = _check_rapids_pytest_benchmark() diff --git a/python/cusignal/testing/__init__.py b/python/cusignal/testing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/cusignal/test/utils.py b/python/cusignal/testing/utils.py similarity index 100% rename from python/cusignal/test/utils.py rename to python/cusignal/testing/utils.py