Merge pull request #4015 from amaslenn/topic/azp-v1.6

AZP: port buildlib/ from master — v1.6

buildlib/azure-pipelines.md

@@ -0,0 +1,78 @@
+# Introduction
+
+This project uses Azure Pipelines a GitHub check to validate pull requests
+prior to merging. Each time a pull request is updated AZP will spawn VMs and
+run compiles and tests based on the instructions in the
+buildlib/azure-pipelines.yml file.
+
+The test console output is linked from the GitHub check integration.
+
+Azure Pipelines is linked to the UCF Consortium's Azure Tenant:
+
+   https://portal.azure.com
+
+And runs inside the Azure Dev Ops Organization:
+
+   https://dev.azure.com/ucfconsort
+
+As the UCX project:
+
+   https://dev.azure.com/ucfconsort/ucx
+
+# Containers
+
+Most of the build steps are done inside Docker containers. The container
+allows direct control and customization over the operating system environment
+to achieve the required test.
+
+UCF hosts a private docker registry on the Azure Container Registry at
+ucfconsort.azurecr.io:
+
+  https://portal.azure.com/#@jgunthorpegmail.onmicrosoft.com/resource/subscriptions/b8ff5e38-a317-4bbd-9831-b73d3887df30/resourceGroups/PipelinesRG/providers/Microsoft.ContainerRegistry/registries/ucfconsort/overview
+
+The Azure Pipelines VM's have high speed access to this registry and can boot
+containers failure quickly.
+
+## Dockerfiles
+
+Each container is described by a docker file in buildlib/. Dockerfiles can be
+built locally using the build command at the top of the Dockerfile. Every
+container has a unique name and tag reflecting its content. So that builds
+continue to work on any stable branches the container version number should be
+incremented when a build-incompatible change is made.
+
+Once built the docker container needs to be pushed to the ACR, using the
+following steps:
+
+```shell
+$ az login
+$ az acr login --name ucfconsort
+$ docker push ucfconsort.azurecr.io/ucx/centos7:1
+```
+
+See https://docs.microsoft.com/en-us/cli/azure for details on how to get the
+command line tools.
+
+## Alternate to 'docker push'
+
+If network connectivity is too poor for push, then the container can be built
+on a VM inside Azure using this command:
+
+```shell
+$ az acr build --registry ucfconsort -t ucfconsort.azurecr.io/ucx/centos7:1 -f buildlib/centos7.Dockerfile buildlib/
+```
+
+## Testing Containers Locally
+
+The local container can be entered and checked out using a command sequence
+similar to:
+
+```shell
+$ cd ..../ucx
+$ docker run --rm -ti -v `pwd`:`pwd` -w `pwd` ucfconsort.azurecr.io/ucx/centos7:1 /bin/bash
+# mkdir build-centos7 && cd build-centos7
+# ../configure
+# make
+```
+
+This will duplicate what will happen when running inside AZP.

buildlib/azure-pipelines.yml

@@ -0,0 +1,101 @@
+# See https://aka.ms/yaml
+
+trigger:
+  - master
+  - v1.6.x
+pr:
+  - master
+  - v1.6.x
+
+resources:
+  containers:
+    - container: centos7
+      image: ucfconsort.azurecr.io/ucx/centos7:1
+      endpoint: ucfconsort_registry
+    - container: fedora
+      image: ucfconsort.azurecr.io/ucx/fedora:1
+      endpoint: ucfconsort_registry
+
+stages:
+  - stage: Build
+    jobs:
+      - job: latest_cc
+        displayName: Latest CCs and CppCheck
+        container: fedora
+        steps:
+          - bash: ./autogen.sh
+            displayName: Setup autotools
+
+          - bash: |
+              set -eE
+
+              mkdir build-gcc && cd build-gcc
+              gcc --version
+              # cscppc wraps gcc to use its output for cppcheck
+              export PATH="`cscppc --print-path-to-wrap`:$PATH"
+              ../contrib/configure-release
+              make -j`nproc` 2>&1 | tee cc.log
+            displayName: GCC
+
+          - bash: |
+              set -eE
+
+              cd build-gcc
+              cppcheck --version
+
+              cppcheck_err="cppcheck.err"
+              # use cs* tools to keep only UCX related issues
+              cslinker --quiet cc.log \
+                | csgrep --mode=json --path $(dirname $PWD) --strip-path-prefix $(dirname $PWD) \
+                | csgrep --mode=json --invert-match --path 'conftest.c' \
+                | csgrep --mode=grep --invert-match --event "internal warning" --prune-events=1 \
+                > $cppcheck_err
+
+                if [ -s $cppcheck_err ]; then
+                  echo "CppCheck found errors:"
+                  cat $cppcheck_err
+                  exit 100
+                fi
+
+                echo "No errors reported by cppcheck"
+
+            displayName: CppCheck
+
+          - bash: |
+              set -eE
+              mkdir build-clang && cd build-clang
+              clang --version
+              ../contrib/configure-release CC=clang CXX=clang++
+            displayName: Configure for Clang
+
+          - bash: |
+              set -eE
+              cd build-clang
+              make -j`nproc`
+            displayName: Clang
+
+      # Perform test builds on relevant distributions.
+      - job: Distros
+        displayName: Build for
+        strategy:
+          matrix:
+            centos7:
+              CONTAINER: centos7
+              CONFIGURE_OPTS:
+        container: $[ variables['CONTAINER'] ]
+        steps:
+          - bash: ./autogen.sh
+            displayName: Setup autotools
+
+          - bash: |
+              set -eE
+              mkdir build && cd build
+              ../configure $(CONFIGURE_OPTS)
+            displayName: Configure
+
+          - bash: |
+              set -eE
+              cd build
+              gcc -v
+              make -s -j `nproc`
+            displayName: Build for $(CONTAINER)

buildlib/centos7.Dockerfile

@@ -0,0 +1,17 @@
+# docker build -t ucfconsort.azurecr.io/ucx/centos7:1 -f buildlib/centos7.Dockerfile buildlib/
+FROM centos:7
+
+RUN yum install -y \
+    autoconf \
+    automake \
+    doxygen \
+    file \
+    gcc-c++ \
+    git \
+    glibc-devel \
+    libtool \
+    make \
+    maven \
+    numactl-devel \
+    rdma-core-devel \
+    && yum clean dbcache packages

buildlib/fedora.Dockerfile

@@ -0,0 +1,22 @@
+# docker build -t ucfconsort.azurecr.io/ucx/fedora:1 -f buildlib/fedora.Dockerfile buildlib/
+FROM fedora:30
+
+RUN dnf install -y \
+    autoconf \
+    automake \
+    clang \
+    cppcheck \
+    cscppc \
+    csmock-common \
+    doxygen \
+    file \
+    gcc-c++ \
+    git \
+    glibc-devel \
+    libtool \
+    make \
+    maven \
+    numactl-devel \
+    rdma-core-devel \
+    rpm-build \
+    && dnf clean dbcache packages