Use conda to build python packages during GPU tests (#480)

This PR convert the `from sources` build we are doing in GPU test job to a `conda build`. This is done for the following reasons: - This is required step to improve the Ops CI/CD setup to a more convenient pipeline - This is required to start using `conda` compilers and `mamba` to build RAPIDS packages - This prevent us from manually managing and installing the dependencies in GPU job - This ensure the packages can be installed - This ensure the tests are running and working against the package content and not the build results. Currently the Python packages are not tested. This may increase the global pipeline time, but the usage of `mamba` should resolve this as `mamba` is faster than `conda` to build packages Authors: - Jordan Jacobelli (https://github.com/Ethyling) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Brad Rees (https://github.com/BradReesWork) URL: #480
rapidsai · May 11, 2022 · 07289a1 · 07289a1
1 parent 600eb63
commit 07289a1
Show file tree

Hide file tree

Showing 16 changed files with 18 additions and 36 deletions.
diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
@@ -24,6 +24,7 @@ export HOME="$WORKSPACE"
 cd "$WORKSPACE"
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 ################################################################################
 # SETUP - Check environment
@@ -38,14 +39,6 @@ nvidia-smi
 gpuci_logger "Activate conda env"
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
-gpuci_conda_retry install -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge \
-    cudatoolkit=${CUDA_REL} \
-    "rapids-build-env=$MINOR_VERSION.*" \
-    "rapids-notebook-env=$MINOR_VERSION."
-
-# https://docs.rapids.ai/maintainers/depmgmt/ 
-# conda remove -f rapids-build-env rapids-notebook-env
-# conda install "your-pkg=1.0.0"
 
 gpuci_logger "Check versions"
 python --version
@@ -61,8 +54,11 @@ conda list --show-channel-urls
 # BUILD - Build cusignal
 ################################################################################
 
-gpuci_logger "Build cusignal"
-"$WORKSPACE/build.sh" clean cusignal
+gpuci_logger "Build and install cusignal"
+cd "${WORKSPACE}"
+CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
+gpuci_conda_retry build --croot "${CONDA_BLD_DIR}" conda/recipes/cusignal --python=${PYTHON}
+gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" cusignal
 
 ################################################################################
 # TEST - Run GoogleTest and py.tests for cusignal
@@ -89,5 +85,3 @@ pytest --cache-clear --junitxml="$WORKSPACE/junit-cusignal.xml" -v -s -m "not cp
 python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log
 
 return ${EXITCODE}
-
-
diff --git a/conda/recipes/cusignal/meta.yaml b/conda/recipes/cusignal/meta.yaml
@@ -9,7 +9,7 @@ package:
   version: {{ version }}
 
 source:
-  path: ../../..
+  git_url: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

diff --git a/python/cusignal/test/__init__.py b/python/cusignal/test/__init__.py
diff --git a/python/cusignal/test/test_acoustics.py b/python/cusignal/test/test_acoustics.py
@@ -16,7 +16,7 @@
 import pytest
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_bsplines.py b/python/cusignal/test/test_bsplines.py
@@ -16,7 +16,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_convolution.py b/python/cusignal/test/test_convolution.py
@@ -16,7 +16,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_filter_design.py b/python/cusignal/test/test_filter_design.py
@@ -16,7 +16,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_filtering.py b/python/cusignal/test/test_filtering.py
@@ -18,7 +18,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_peak_finding.py b/python/cusignal/test/test_peak_finding.py
@@ -17,7 +17,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_radartools.py b/python/cusignal/test/test_radartools.py
@@ -3,7 +3,7 @@
 from numpy import vectorize
 
 from cusignal.radartools import ca_cfar, cfar_alpha
-from cusignal.test.utils import array_equal
+from cusignal.testing.utils import array_equal
 
 
 class TestCaCfar:

diff --git a/python/cusignal/test/test_spectral_analysis.py b/python/cusignal/test/test_spectral_analysis.py
@@ -17,7 +17,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_waveforms.py b/python/cusignal/test/test_waveforms.py
@@ -17,7 +17,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_wavelets.py b/python/cusignal/test/test_wavelets.py
@@ -17,7 +17,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/test/test_windows.py b/python/cusignal/test/test_windows.py
@@ -16,7 +16,7 @@
 from scipy import signal
 
 import cusignal
-from cusignal.test.utils import _check_rapids_pytest_benchmark, array_equal
+from cusignal.testing.utils import _check_rapids_pytest_benchmark, array_equal
 
 gpubenchmark = _check_rapids_pytest_benchmark()
 

diff --git a/python/cusignal/testing/__init__.py b/python/cusignal/testing/__init__.py
diff --git a/python/cusignal/test/utils.py → python/cusignal/testing/utils.py b/python/cusignal/test/utils.py → python/cusignal/testing/utils.py