Merge #899 #905

899: Better error diagnostics on targets without std. r=Emilgardis a=Alexhuszagh Provides a warning if a docker command fails and `rust-std` is not available for the target. For example, for `x86_64-unknown-dragonfly`, the added output would be: ```bash [cross] warning: rust-std is not available for x86_64-unknown-dragonfly [cross] note: you may need to build components for the target via `-Z build-std=<components>` or in your cross configuration specify `target.x86_64-unknown-dragonfly.build-std` the available components are core, std, alloc, and proc_macro ``` This is done solely if the command fails and if the target is a built-in, so we don't get misleading warnings for custom targets, and there is no overhead if the build succeeds. Closes #94. Closes #167. 905: Numerous bug fixes for musl and other images. r=Emilgardis a=Alexhuszagh Provide `qemu-runner` that allows native or non-native for any architecture, and install `qemu-user` binaries for x86 musl images. In addition, create symlinks so dynamically-linked binaries can be run, including for C++, with musl images. Adding `qemu-user` runners is also useful for running dynamically-linked x86 binaries, since otherwise they may be unable to find the linked musl libc, and we cannot install it in `/lib` or `/usr/lib` due to conflicts with the system. Co-authored-by: Alex Huszagh <ahuszagh@gmail.com>
cross-rs · Jul 3, 2022 · 4c091de · 4c091de
3 parents 4f0b147 + 10b0eb1 + ed5ce0f
commit 4c091de
Show file tree

Hide file tree

Showing 26 changed files with 334 additions and 97 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -158,7 +158,7 @@ jobs:
             - { target: aarch64-unknown-linux-gnu,        os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user qemu-system }
             - { target: arm-unknown-linux-gnueabi,        os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
             - { target: arm-unknown-linux-gnueabihf,      os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
-            - { target: armv7-unknown-linux-gnueabi,    os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user qemu-system }
+            - { target: armv7-unknown-linux-gnueabi,      os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user }
             - { target: armv7-unknown-linux-gnueabihf,    os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user qemu-system }
             - { target: thumbv7neon-unknown-linux-gnueabihf, os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user qemu-system }
             - { target: i586-unknown-linux-gnu,           os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
@@ -175,27 +175,27 @@ jobs:
             - { target: riscv64gc-unknown-linux-gnu,      os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
             - { target: s390x-unknown-linux-gnu,          os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-system }
             - { target: sparc64-unknown-linux-gnu,        os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-system }
-            - { target: aarch64-unknown-linux-musl,       os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: arm-unknown-linux-musleabihf,     os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: arm-unknown-linux-musleabi,       os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: armv5te-unknown-linux-gnueabi,    os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: armv5te-unknown-linux-musleabi,   os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: armv7-unknown-linux-musleabi,   os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: armv7-unknown-linux-musleabihf,   os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: i586-unknown-linux-musl,          os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: i686-unknown-linux-musl,          os: ubuntu-latest,                    std: 1, run: 1 }
-            - { target: mips-unknown-linux-musl,          os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
-            - { target: mipsel-unknown-linux-musl,        os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
+            - { target: aarch64-unknown-linux-musl,       os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: arm-unknown-linux-musleabihf,     os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: arm-unknown-linux-musleabi,       os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: armv5te-unknown-linux-gnueabi,    os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: armv5te-unknown-linux-musleabi,   os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: armv7-unknown-linux-musleabi,     os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: armv7-unknown-linux-musleabihf,   os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: i586-unknown-linux-musl,          os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user }
+            - { target: i686-unknown-linux-musl,          os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1, runners: qemu-user }
+            - { target: mips-unknown-linux-musl,          os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
+            - { target: mipsel-unknown-linux-musl,        os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, run: 1 }
             - { target: aarch64-linux-android,            os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
             - { target: arm-linux-androideabi,            os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
             - { target: armv7-linux-androideabi,          os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
-            - { target: thumbv7neon-linux-androideabi,          os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
+            - { target: thumbv7neon-linux-androideabi,    os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
             - { target: i686-linux-android,               os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
             - { target: x86_64-linux-android,             os: ubuntu-latest,  cpp: 1,           std: 1, run: 1, runners: qemu-user }
             - { target: x86_64-pc-windows-gnu,            os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
             - { target: i686-pc-windows-gnu,              os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
             # Disabled for now, see https://github.com/rust-lang/rust/issues/98216
-            #- { target: asmjs-unknown-emscripten,         os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
+            #-{ target: asmjs-unknown-emscripten,         os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
             - { target: wasm32-unknown-emscripten,        os: ubuntu-latest,  cpp: 1,           std: 1, run: 1 }
             - { target: x86_64-unknown-dragonfly,         os: ubuntu-latest,  cpp: 1, dylib: 1, std: 1, build-std: 1 }
             - { target: i686-unknown-freebsd,             os: ubuntu-latest,          dylib: 1, std: 1 }

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- #905 - added `qemu-runner` for musl images, allowing use of native or emulated runners.
+- #905 - added qemu emulation to `i586-unknown-linux-gnu`, `i686-unknown-linux-musl`, and `i586-unknown-linux-gnu`, so they can run on an `x86` CPU, rather than an `x86_64` CPU.
 - #900 - add the option to skip copying build artifacts back to host when using remote cross via `CROSS_REMOTE_SKIP_BUILD_ARTIFACTS`.
 - #891 - support custom user namespace overrides by setting the `CROSS_CONTAINER_USER_NAMESPACE` environment variable. 
 - #890 - support rootless docker via the `CROSS_ROOTLESS_CONTAINER_ENGINE` environment variable.
@@ -20,6 +22,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+- #905 - fixed running dynamically-linked libraries for all musl targets except `x86_64-unknown-linux-musl`.
 - #904 - ensure `cargo metadata` works by using the same channel.
 - #904 - fixed the path for workspace volumes and passthrough volumes with docker-in-docker.
 - #898 - fix the path to the mount root with docker-in-docker if mounting volumes.

diff --git a/docker/Dockerfile.aarch64-unknown-linux-musl b/docker/Dockerfile.aarch64-unknown-linux-musl
@@ -16,17 +16,18 @@ RUN /qemu.sh aarch64
 COPY musl.sh /
 RUN /musl.sh TARGET=aarch64-linux-musl
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/aarch64-linux-musl/lib/libc.so \
-    /usr/local/aarch64-linux-musl/lib/ld-musl-aarch64.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/aarch64-linux-musl
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT aarch64
 
 COPY aarch64-linux-musl-gcc.sh /usr/bin/
 
+COPY qemu-runner /
+
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc.sh \
-    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER="/qemu-runner aarch64" \
     CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc \
     CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_aarch64_unknown_linux_musl="--sysroot=/usr/local/aarch64-linux-musl" \
-    QEMU_LD_PREFIX=/usr/local/aarch64-linux-musl \
+    BINDGEN_EXTRA_CLANG_ARGS_aarch64_unknown_linux_musl="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.arm-unknown-linux-gnueabi b/docker/Dockerfile.arm-unknown-linux-gnueabi
@@ -16,8 +16,10 @@ RUN apt-get update && apt-get install --assume-yes --no-install-recommends \
     libc6-dev-armel-cross && \
     /qemu.sh arm
 
+COPY qemu-runner /
+
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_LINKER=arm-linux-gnueabi-gcc \
-    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm \
+    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_RUNNER="/qemu-runner arm" \
     CC_arm_unknown_linux_gnueabi=arm-linux-gnueabi-gcc \
     CXX_arm_unknown_linux_gnueabi=arm-linux-gnueabi-g++ \
     BINDGEN_EXTRA_CLANG_ARGS_arm_unknown_linux_gnueabi="--sysroot=/usr/arm-linux-gnueabi" \

diff --git a/docker/Dockerfile.arm-unknown-linux-gnueabihf b/docker/Dockerfile.arm-unknown-linux-gnueabihf
@@ -19,8 +19,10 @@ ENV PATH /x-tools/arm-unknown-linux-gnueabihf/bin/:$PATH
 COPY qemu.sh /
 RUN /qemu.sh arm
 
+COPY qemu-runner /
+
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-unknown-linux-gnueabihf-gcc \
-    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm \
+    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER="/qemu-runner arm" \
     CC_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-gcc \
     CXX_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-g++ \
     BINDGEN_EXTRA_CLANG_ARGS_arm_unknown_linux_gnueabihf="--sysroot=/x-tools/arm-unknown-linux-gnueabihf/arm-unknown-linux-gnueabihf/sysroot/" \

diff --git a/docker/Dockerfile.arm-unknown-linux-musleabi b/docker/Dockerfile.arm-unknown-linux-musleabi
@@ -20,15 +20,16 @@ RUN /musl.sh \
                       --with-float=soft \
                       --with-mode=arm"
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/arm-linux-musleabi/lib/libc.so \
-    /usr/local/arm-linux-musleabi/lib/ld-musl-arm.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/arm-linux-musleabi
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT arm
+
+COPY qemu-runner /
 
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABI_LINKER=arm-linux-musleabi-gcc \
-    CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABI_RUNNER=qemu-arm \
+    CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABI_RUNNER="/qemu-runner arm" \
     CC_arm_unknown_linux_musleabi=arm-linux-musleabi-gcc \
     CXX_arm_unknown_linux_musleabi=arm-linux-musleabi-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_arm_unknown_linux_musleabi="--sysroot=/usr/local/arm-linux-musleabi" \
-    QEMU_LD_PREFIX=/usr/local/arm-linux-musleabi \
+    BINDGEN_EXTRA_CLANG_ARGS_arm_unknown_linux_musleabi="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.arm-unknown-linux-musleabihf b/docker/Dockerfile.arm-unknown-linux-musleabihf
@@ -21,15 +21,16 @@ RUN /musl.sh \
                       --with-float=hard \
                       --with-mode=arm"
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/arm-linux-musleabihf/lib/libc.so \
-    /usr/local/arm-linux-musleabihf/lib/ld-musl-armhf.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/arm-linux-musleabihf
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT armhf
+
+COPY qemu-runner /
 
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_LINKER=arm-linux-musleabihf-gcc \
-    CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_RUNNER=qemu-arm \
+    CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_RUNNER="/qemu-runner arm" \
     CC_arm_unknown_linux_musleabihf=arm-linux-musleabihf-gcc \
     CXX_arm_unknown_linux_musleabihf=arm-linux-musleabihf-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_arm_unknown_linux_musleabihf="--sysroot=/usr/local/arm-linux-musleabihf" \
-    QEMU_LD_PREFIX=/usr/local/arm-linux-musleabihf \
+    BINDGEN_EXTRA_CLANG_ARGS_arm_unknown_linux_musleabihf="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.armv5te-unknown-linux-gnueabi b/docker/Dockerfile.armv5te-unknown-linux-gnueabi
@@ -17,8 +17,10 @@ RUN apt-get update && apt-get install --assume-yes --no-install-recommends \
     libc6-dev-armel-cross && \
     /qemu.sh arm
 
+COPY qemu-runner /
+
 ENV CARGO_TARGET_ARMV5TE_UNKNOWN_LINUX_GNUEABI_LINKER=arm-linux-gnueabi-gcc \
-    CARGO_TARGET_ARMV5TE_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm \
+    CARGO_TARGET_ARMV5TE_UNKNOWN_LINUX_GNUEABI_RUNNER="/qemu-runner arm" \
     CC_armv5te_unknown_linux_gnueabi=arm-linux-gnueabi-gcc \
     CXX_armv5te_unknown_linux_gnueabi=arm-linux-gnueabi-g++ \
     BINDGEN_EXTRA_CLANG_ARGS_armv5te_unknown_linux_gnueabi="--sysroot=/usr/arm-linux-gnueabi" \

diff --git a/docker/Dockerfile.armv5te-unknown-linux-musleabi b/docker/Dockerfile.armv5te-unknown-linux-musleabi
@@ -20,15 +20,16 @@ RUN /musl.sh \
                       --with-float=soft \
                       --with-mode=arm"
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/arm-linux-musleabi/lib/libc.so \
-    /usr/local/arm-linux-musleabi/lib/ld-musl-arm.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/arm-linux-musleabi
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT arm
+
+COPY qemu-runner /
 
 ENV CARGO_TARGET_ARMV5TE_UNKNOWN_LINUX_MUSLEABI_LINKER=arm-linux-musleabi-gcc \
-    CARGO_TARGET_ARMV5TE_UNKNOWN_LINUX_MUSLEABI_RUNNER=qemu-arm \
+    CARGO_TARGET_ARMV5TE_UNKNOWN_LINUX_MUSLEABI_RUNNER="/qemu-runner arm" \
     CC_armv5te_unknown_linux_musleabi=arm-linux-musleabi-gcc \
     CXX_armv5te_unknown_linux_musleabi=arm-linux-musleabi-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_armv5te_unknown_linux_musleabi="--sysroot=/usr/arm-linux-musleabi" \
-    QEMU_LD_PREFIX=/usr/local/arm-linux-musleabi \
+    BINDGEN_EXTRA_CLANG_ARGS_armv5te_unknown_linux_musleabi="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.armv7-unknown-linux-gnueabi b/docker/Dockerfile.armv7-unknown-linux-gnueabi
@@ -17,8 +17,10 @@ RUN apt-get install --assume-yes --no-install-recommends \
 COPY qemu.sh /
 RUN /qemu.sh arm
 
+COPY qemu-runner /
+
 ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABI_LINKER=arm-linux-gnueabi-gcc \
-    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm \
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABI_RUNNER="/qemu-runner armv7" \
     CC_armv7_unknown_linux_gnueabi=arm-linux-gnueabi-gcc \
     CXX_armv7_unknown_linux_gnueabi=arm-linux-gnueabi-g++ \
     BINDGEN_EXTRA_CLANG_ARGS_armv7_unknown_linux_gnueabi="--sysroot=/usr/arm-linux-gnueabi" \

diff --git a/docker/Dockerfile.armv7-unknown-linux-musleabi b/docker/Dockerfile.armv7-unknown-linux-musleabi
@@ -21,15 +21,16 @@ RUN /musl.sh \
                       --with-mode=thumb \
                       --with-mode=arm"
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/arm-linux-musleabi/lib/libc.so \
-    /usr/local/arm-linux-musleabi/lib/ld-musl-arm.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/arm-linux-musleabi
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT arm
+
+COPY qemu-runner /
 
 ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABI_LINKER=arm-linux-musleabi-gcc \
-    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABI_RUNNER=qemu-arm \
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABI_RUNNER="/qemu-runner armv7" \
     CC_armv7_unknown_linux_musleabi=arm-linux-musleabi-gcc \
     CXX_armv7_unknown_linux_musleabi=arm-linux-musleabi-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_armv7_unknown_linux_musleabi="--sysroot=/usr/local/arm-linux-musleabi" \
-    QEMU_LD_PREFIX=/usr/local/arm-linux-musleabi \
+    BINDGEN_EXTRA_CLANG_ARGS_armv7_unknown_linux_musleabi="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.armv7-unknown-linux-musleabihf b/docker/Dockerfile.armv7-unknown-linux-musleabihf
@@ -21,15 +21,16 @@ RUN /musl.sh \
                       --with-mode=thumb \
                       --with-fpu=vfp"
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/arm-linux-musleabihf/lib/libc.so \
-    /usr/local/arm-linux-musleabihf/lib/ld-musl-armhf.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/arm-linux-musleabihf
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT armhf
+
+COPY qemu-runner /
 
 ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_LINKER=arm-linux-musleabihf-gcc \
-    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_RUNNER=qemu-arm \
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_RUNNER="/qemu-runner armv7" \
     CC_armv7_unknown_linux_musleabihf=arm-linux-musleabihf-gcc \
     CXX_armv7_unknown_linux_musleabihf=arm-linux-musleabihf-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_armv7_unknown_linux_musleabihf="--sysroot=/usr/local/arm-linux-musleabihf" \
-    QEMU_LD_PREFIX=/usr/local/arm-linux-musleabihf \
+    BINDGEN_EXTRA_CLANG_ARGS_armv7_unknown_linux_musleabihf="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.i586-unknown-linux-gnu b/docker/Dockerfile.i586-unknown-linux-gnu
@@ -13,4 +13,10 @@ RUN /xargo.sh
 RUN apt-get update && apt-get install --assume-yes --no-install-recommends \
     g++-multilib
 
-ENV PKG_CONFIG_PATH="/usr/lib/i386-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"
+COPY qemu.sh /
+RUN /qemu.sh i386
+
+COPY qemu-runner /
+
+ENV CARGO_TARGET_I586_UNKNOWN_LINUX_GNU_RUNNER="/qemu-runner i586" \
+    PKG_CONFIG_PATH="/usr/lib/i386-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"
diff --git a/docker/Dockerfile.i586-unknown-linux-musl b/docker/Dockerfile.i586-unknown-linux-musl
@@ -13,7 +13,18 @@ RUN /xargo.sh
 COPY musl.sh /
 RUN /musl.sh TARGET=i586-linux-musl
 
+COPY qemu.sh /
+RUN /qemu.sh i386
+
+ENV CROSS_MUSL_SYSROOT=/usr/local/i586-linux-musl
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT i386
+
+COPY qemu-runner /
+
 ENV CARGO_TARGET_I586_UNKNOWN_LINUX_MUSL_LINKER=i586-linux-musl-gcc \
+    CARGO_TARGET_I586_UNKNOWN_LINUX_MUSL_RUNNER="/qemu-runner i586" \
     CC_i586_unknown_linux_musl=i586-linux-musl-gcc \
     CXX_i586_unknown_linux_musl=i586-linux-musl-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_i586_unknown_linux_musl="--sysroot=/usr/local/i586-linux-musl"
+    BINDGEN_EXTRA_CLANG_ARGS_i586_unknown_linux_musl="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT
diff --git a/docker/Dockerfile.i686-unknown-linux-musl b/docker/Dockerfile.i686-unknown-linux-musl
@@ -13,7 +13,18 @@ RUN /xargo.sh
 COPY musl.sh /
 RUN /musl.sh TARGET=i686-linux-musl
 
+COPY qemu.sh /
+RUN /qemu.sh i386
+
+ENV CROSS_MUSL_SYSROOT=/usr/local/i686-linux-musl
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT i386
+
+COPY qemu-runner /
+
 ENV CARGO_TARGET_I686_UNKNOWN_LINUX_MUSL_LINKER=i686-linux-musl-gcc \
+    CARGO_TARGET_I686_UNKNOWN_LINUX_MUSL_RUNNER="/qemu-runner i686" \
     CC_i686_unknown_linux_musl=i686-linux-musl-gcc \
     CXX_i686_unknown_linux_musl=i686-linux-musl-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_i686_unknown_linux_musl="--sysroot=/usr/local/i686-linux-musl"
+    BINDGEN_EXTRA_CLANG_ARGS_i686_unknown_linux_musl="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT
diff --git a/docker/Dockerfile.mips-unknown-linux-musl b/docker/Dockerfile.mips-unknown-linux-musl
@@ -18,15 +18,16 @@ RUN /musl.sh \
     TARGET=mips-linux-muslsf \
     "COMMON_CONFIG += -with-arch=mips32r2"
 
-# Allows qemu run dynamic linked binaries
-RUN ln -sf \
-    /usr/local/mips-linux-muslsf/lib/libc.so \
-    /usr/local/mips-linux-muslsf/lib/ld-musl-mips-sf.so.1
+ENV CROSS_MUSL_SYSROOT=/usr/local/mips-linux-muslsf
+COPY musl-symlink.sh /
+RUN /musl-symlink.sh $CROSS_MUSL_SYSROOT mips-sf
+
+COPY qemu-runner /
 
 ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_MUSL_LINKER=mips-linux-muslsf-gcc \
-    CARGO_TARGET_MIPS_UNKNOWN_LINUX_MUSL_RUNNER=qemu-mips \
+    CARGO_TARGET_MIPS_UNKNOWN_LINUX_MUSL_RUNNER="/qemu-runner mips" \
     CC_mips_unknown_linux_musl=mips-linux-muslsf-gcc \
     CXX_mips_unknown_linux_musl=mips-linux-muslsf-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_mips_unknown_linux_musl="--sysroot=/usr/local/mips-linux-muslsf" \
-    QEMU_LD_PREFIX=/usr/local/mips-linux-muslsf \
+    BINDGEN_EXTRA_CLANG_ARGS_mips_unknown_linux_musl="--sysroot=$CROSS_MUSL_SYSROOT" \
+    QEMU_LD_PREFIX=$CROSS_MUSL_SYSROOT \
     RUST_TEST_THREADS=1
diff --git a/docker/Dockerfile.mips64-unknown-linux-gnuabi64 b/docker/Dockerfile.mips64-unknown-linux-gnuabi64
@@ -16,8 +16,10 @@ RUN apt-get update && apt-get install --assume-yes --no-install-recommends \
     libc6-dev-mips64-cross && \
     /qemu.sh mips64
 
+COPY qemu-runner /
+
 ENV CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_LINKER=mips64-linux-gnuabi64-gcc \
-    CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64 \
+    CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_RUNNER="/qemu-runner mips64" \
     CC_mips64_unknown_linux_gnuabi64=mips64-linux-gnuabi64-gcc \
     CXX_mips64_unknown_linux_gnuabi64=mips64-linux-gnuabi64-g++ \
     BINDGEN_EXTRA_CLANG_ARGS_mips64_unknown_linux_gnuabi64="--sysroot=/usr/mips64-linux-gnuabi64" \