From e5839faa90912829f193b9806376ae37844bf645 Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Thu, 8 Aug 2019 15:59:23 +0300 Subject: [PATCH 1/2] AZP: port buildlib/ from master --- buildlib/azure-pipelines.md | 78 ++++++++++++++++++++++++++++ buildlib/azure-pipelines.yml | 99 ++++++++++++++++++++++++++++++++++++ buildlib/centos7.Dockerfile | 17 +++++++ buildlib/fedora.Dockerfile | 22 ++++++++ 4 files changed, 216 insertions(+) create mode 100644 buildlib/azure-pipelines.md create mode 100644 buildlib/azure-pipelines.yml create mode 100644 buildlib/centos7.Dockerfile create mode 100644 buildlib/fedora.Dockerfile diff --git a/buildlib/azure-pipelines.md b/buildlib/azure-pipelines.md new file mode 100644 index 00000000000..b086d754cd5 --- /dev/null +++ b/buildlib/azure-pipelines.md @@ -0,0 +1,78 @@ +# Introduction + +This project uses Azure Pipelines a GitHub check to validate pull requests +prior to merging. Each time a pull request is updated AZP will spawn VMs and +run compiles and tests based on the instructions in the +buildlib/azure-pipelines.yml file. + +The test console output is linked from the GitHub check integration. + +Azure Pipelines is linked to the UCF Consortium's Azure Tenant: + + https://portal.azure.com + +And runs inside the Azure Dev Ops Organization: + + https://dev.azure.com/ucfconsort + +As the UCX project: + + https://dev.azure.com/ucfconsort/ucx + +# Containers + +Most of the build steps are done inside Docker containers. The container +allows direct control and customization over the operating system environment +to achieve the required test. + +UCF hosts a private docker registry on the Azure Container Registry at +ucfconsort.azurecr.io: + + https://portal.azure.com/#@jgunthorpegmail.onmicrosoft.com/resource/subscriptions/b8ff5e38-a317-4bbd-9831-b73d3887df30/resourceGroups/PipelinesRG/providers/Microsoft.ContainerRegistry/registries/ucfconsort/overview + +The Azure Pipelines VM's have high speed access to this registry and can boot +containers failure quickly. + +## Dockerfiles + +Each container is described by a docker file in buildlib/. Dockerfiles can be +built locally using the build command at the top of the Dockerfile. Every +container has a unique name and tag reflecting its content. So that builds +continue to work on any stable branches the container version number should be +incremented when a build-incompatible change is made. + +Once built the docker container needs to be pushed to the ACR, using the +following steps: + +```shell +$ az login +$ az acr login --name ucfconsort +$ docker push ucfconsort.azurecr.io/ucx/centos7:1 +``` + +See https://docs.microsoft.com/en-us/cli/azure for details on how to get the +command line tools. + +## Alternate to 'docker push' + +If network connectivity is too poor for push, then the container can be built +on a VM inside Azure using this command: + +```shell +$ az acr build --registry ucfconsort -t ucfconsort.azurecr.io/ucx/centos7:1 -f buildlib/centos7.Dockerfile buildlib/ +``` + +## Testing Containers Locally + +The local container can be entered and checked out using a command sequence +similar to: + +```shell +$ cd ..../ucx +$ docker run --rm -ti -v `pwd`:`pwd` -w `pwd` ucfconsort.azurecr.io/ucx/centos7:1 /bin/bash +# mkdir build-centos7 && cd build-centos7 +# ../configure +# make +``` + +This will duplicate what will happen when running inside AZP. diff --git a/buildlib/azure-pipelines.yml b/buildlib/azure-pipelines.yml new file mode 100644 index 00000000000..fd1c723b843 --- /dev/null +++ b/buildlib/azure-pipelines.yml @@ -0,0 +1,99 @@ +# See https://aka.ms/yaml + +trigger: + - master +pr: + - master + +resources: + containers: + - container: centos7 + image: ucfconsort.azurecr.io/ucx/centos7:1 + endpoint: ucfconsort_registry + - container: fedora + image: ucfconsort.azurecr.io/ucx/fedora:1 + endpoint: ucfconsort_registry + +stages: + - stage: Build + jobs: + - job: latest_cc + displayName: Latest CCs and CppCheck + container: fedora + steps: + - bash: ./autogen.sh + displayName: Setup autotools + + - bash: | + set -eE + + mkdir build-gcc && cd build-gcc + gcc --version + # cscppc wraps gcc to use its output for cppcheck + export PATH="`cscppc --print-path-to-wrap`:$PATH" + ../contrib/configure-release + make -j`nproc` 2>&1 | tee cc.log + displayName: GCC + + - bash: | + set -eE + + cd build-gcc + cppcheck --version + + cppcheck_err="cppcheck.err" + # use cs* tools to keep only UCX related issues + cslinker --quiet cc.log \ + | csgrep --mode=json --path $(dirname $PWD) --strip-path-prefix $(dirname $PWD) \ + | csgrep --mode=json --invert-match --path 'conftest.c' \ + | csgrep --mode=grep --invert-match --event "internal warning" --prune-events=1 \ + > $cppcheck_err + + if [ -s $cppcheck_err ]; then + echo "CppCheck found errors:" + cat $cppcheck_err + exit 100 + fi + + echo "No errors reported by cppcheck" + + displayName: CppCheck + + - bash: | + set -eE + mkdir build-clang && cd build-clang + clang --version + ../contrib/configure-release CC=clang CXX=clang++ + displayName: Configure for Clang + + - bash: | + set -eE + cd build-clang + make -j`nproc` + displayName: Clang + + # Perform test builds on relevant distributions. + - job: Distros + displayName: Build for + strategy: + matrix: + centos7: + CONTAINER: centos7 + CONFIGURE_OPTS: + container: $[ variables['CONTAINER'] ] + steps: + - bash: ./autogen.sh + displayName: Setup autotools + + - bash: | + set -eE + mkdir build && cd build + ../configure $(CONFIGURE_OPTS) + displayName: Configure + + - bash: | + set -eE + cd build + gcc -v + make -s -j `nproc` + displayName: Build for $(CONTAINER) diff --git a/buildlib/centos7.Dockerfile b/buildlib/centos7.Dockerfile new file mode 100644 index 00000000000..5cf22ec7546 --- /dev/null +++ b/buildlib/centos7.Dockerfile @@ -0,0 +1,17 @@ +# docker build -t ucfconsort.azurecr.io/ucx/centos7:1 -f buildlib/centos7.Dockerfile buildlib/ +FROM centos:7 + +RUN yum install -y \ + autoconf \ + automake \ + doxygen \ + file \ + gcc-c++ \ + git \ + glibc-devel \ + libtool \ + make \ + maven \ + numactl-devel \ + rdma-core-devel \ + && yum clean dbcache packages diff --git a/buildlib/fedora.Dockerfile b/buildlib/fedora.Dockerfile new file mode 100644 index 00000000000..953bf2ca7d2 --- /dev/null +++ b/buildlib/fedora.Dockerfile @@ -0,0 +1,22 @@ +# docker build -t ucfconsort.azurecr.io/ucx/fedora:1 -f buildlib/fedora.Dockerfile buildlib/ +FROM fedora:30 + +RUN dnf install -y \ + autoconf \ + automake \ + clang \ + cppcheck \ + cscppc \ + csmock-common \ + doxygen \ + file \ + gcc-c++ \ + git \ + glibc-devel \ + libtool \ + make \ + maven \ + numactl-devel \ + rdma-core-devel \ + rpm-build \ + && dnf clean dbcache packages From 188b2946ef927c1c742fd0226febc94cd7dba41b Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Thu, 8 Aug 2019 15:59:54 +0300 Subject: [PATCH 2/2] AZP: enable for v1.6.x branch --- buildlib/azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildlib/azure-pipelines.yml b/buildlib/azure-pipelines.yml index fd1c723b843..4f5b12cbe86 100644 --- a/buildlib/azure-pipelines.yml +++ b/buildlib/azure-pipelines.yml @@ -2,8 +2,10 @@ trigger: - master + - v1.6.x pr: - master + - v1.6.x resources: containers: