From 1cbd96316158431cad883e3c6e7cdf7b33903a6b Mon Sep 17 00:00:00 2001 From: youben11 Date: Wed, 23 Oct 2024 10:24:42 +0100 Subject: [PATCH] fix(ci): lint all dockerfiles and lower failure threshold --- .hadolint.yaml | 2 +- docker/Dockerfile.concrete-compiler-env | 23 +++++++++++------------ docker/Dockerfile.concrete-python | 5 ++++- docker/Dockerfile.cuda-118-env | 22 ++++++++++++---------- docker/Dockerfile.cuda-123-env | 22 ++++++++++++---------- docker/Dockerfile.hpx-env | 16 ++++++++-------- 6 files changed, 48 insertions(+), 42 deletions(-) diff --git a/.hadolint.yaml b/.hadolint.yaml index f8cbb9da2..699d521ab 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1 +1 @@ -failure-threshold: error +failure-threshold: none diff --git a/docker/Dockerfile.concrete-compiler-env b/docker/Dockerfile.concrete-compiler-env index 9a86fef00..77a0ce86b 100644 --- a/docker/Dockerfile.concrete-compiler-env +++ b/docker/Dockerfile.concrete-compiler-env @@ -2,25 +2,24 @@ FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc # epel-release is for install ccache RUN dnf clean all -RUN dnf install -y epel-release +RUN dnf install -y epel-release && dnf clean all RUN dnf update -y -RUN dnf install -y ninja-build hwloc-devel ccache ncurses-devel -RUN dnf install -y openssh-clients graphviz graphviz-devel +# hadolint ignore=DL3041 +RUN dnf install -y ninja-build hwloc-devel ccache ncurses-devel openssh-clients graphviz graphviz-devel && dnf clean all RUN mkdir -p ~/.ssh/ && ssh-keyscan -t ecdsa github.com >> ~/.ssh/known_hosts # Setup gcc-11 (required for cuda11.8) -RUN dnf install -y gcc-toolset-11 +RUN dnf install -y gcc-toolset-11 && dnf clean all ENV CC_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/gcc ENV CXX_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/c++ ENV PATH "/opt/rh/gcc-toolset-11/root/usr/bin:$PATH" ENV LD_LIBRARY_PATH "/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib:/opt/rh/gcc-toolset-11/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-11/root/usr/lib/dyninst:$LD_LIBRARY_PATH" # setup ccache with an unlimited amount of files and storage -RUN ccache -M 0 -RUN ccache -F 0 +RUN ccache -M 0 && ccache -F 0 # Install Rust +SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH=/root/.cargo/bin:$PATH RUN rustup install nightly-2024-09-30 -SHELL ["/bin/bash", "-c"] # Install boost ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz RUN tar -xzvf /boost_1_71_0.tar.gz @@ -35,16 +34,16 @@ ENV PATH "$PATH:/usr/local/cuda-11.8/bin" # Set the python path. Options: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] # Links and env would be available to use the appropriate python version ARG python_tag=cp38-cp38 -RUN ln -s /opt/python/${python_tag}/bin/pip /bin/pip -RUN ln -s /opt/python/${python_tag}/bin/python /bin/python +RUN ln -s /opt/python/${python_tag}/bin/pip /bin/pip && ln -s /opt/python/${python_tag}/bin/python /bin/python ENV PYTHON_EXEC=/opt/python/${python_tag}/bin/python # Install python deps -RUN pip install numpy pybind11==2.8 PyYAML pytest wheel auditwheel mypy +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir numpy pybind11==2.8 PyYAML pytest wheel auditwheel mypy # Setup and build compiler COPY / /workdir WORKDIR /workdir/compilers/concrete-compiler/compiler -RUN mkdir -p /build -RUN make DATAFLOW_EXECUTION_ENABLED=ON BUILD_DIR=/build CCACHE=ON \ +RUN mkdir -p /build && \ + make DATAFLOW_EXECUTION_ENABLED=ON BUILD_DIR=/build CCACHE=ON \ Python3_EXECUTABLE=${PYTHON_EXEC} \ concretecompiler python-bindings ENV PYTHONPATH "$PYTHONPATH:/build/tools/concretelang/python_packages/concretelang_core" diff --git a/docker/Dockerfile.concrete-python b/docker/Dockerfile.concrete-python index ca293a317..fb89b41a3 100644 --- a/docker/Dockerfile.concrete-python +++ b/docker/Dockerfile.concrete-python @@ -3,6 +3,9 @@ FROM python:3.10-slim ARG version # provide the `ld` binary required by the compiler -RUN apt update && apt install -y binutils graphviz +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install --no-install-recommends -y binutils graphviz \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.zama.ai/cpu/ concrete-python==${version} diff --git a/docker/Dockerfile.cuda-118-env b/docker/Dockerfile.cuda-118-env index 466b0a7d3..ab0232ba5 100644 --- a/docker/Dockerfile.cuda-118-env +++ b/docker/Dockerfile.cuda-118-env @@ -1,14 +1,16 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc +FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc as build -RUN dnf install -y kernel-devel kernel-headers +RUN dnf install -y kernel-devel kernel-headers && dnf clean all RUN curl https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda-repo-rhel8-11-8-local-11.8.0_520.61.05-1.x86_64.rpm -o cuda.rpm -RUN rpm -i cuda.rpm -RUN dnf clean all -RUN dnf install -y epel-release -RUN dnf update -y -RUN dnf -y module install nvidia-driver:latest-dkms -RUN dnf -y install cuda +RUN rpm -i cuda.rpm \ + && dnf install -y epel-release \ + && dnf clean all +# hadolint ignore=DL3041 +RUN dnf update -y \ + && dnf -y module install nvidia-driver:latest-dkms \ + && dnf -y install cuda \ + && dnf clean all FROM scratch -COPY --from=0 /usr/local/cuda-11.8/ /usr/local/cuda-11.8/ -COPY --from=0 /usr/lib64/libcuda.so* /usr/lib64/ +COPY --from=build /usr/local/cuda-11.8/ /usr/local/cuda-11.8/ +COPY --from=build /usr/lib64/libcuda.so* /usr/lib64/ diff --git a/docker/Dockerfile.cuda-123-env b/docker/Dockerfile.cuda-123-env index 33389fc2d..5de78e418 100644 --- a/docker/Dockerfile.cuda-123-env +++ b/docker/Dockerfile.cuda-123-env @@ -1,14 +1,16 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc +FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc as build -RUN dnf install -y kernel-devel kernel-headers +RUN dnf install -y kernel-devel kernel-headers && dnf clean all RUN curl https://developer.download.nvidia.com/compute/cuda/12.3.2/local_installers/cuda-repo-rhel8-12-3-local-12.3.2_545.23.08-1.x86_64.rpm -o cuda.rpm -RUN rpm -i cuda.rpm -RUN dnf clean all -RUN dnf install -y epel-release -RUN dnf update -y -RUN dnf -y module install nvidia-driver:latest-dkms -RUN dnf -y install cuda +RUN rpm -i cuda.rpm \ + && dnf install -y epel-release \ + && dnf clean all +# hadolint ignore=DL3041 +RUN dnf update -y \ + && dnf -y module install nvidia-driver:latest-dkms \ + && dnf -y install cuda \ + && dnf clean all FROM scratch -COPY --from=0 /usr/local/cuda-12.3/ /usr/local/cuda-12.3/ -COPY --from=0 /usr/lib64/libcuda.so* /usr/lib64/ +COPY --from=build /usr/local/cuda-12.3/ /usr/local/cuda-12.3/ +COPY --from=build /usr/lib64/libcuda.so* /usr/lib64/ diff --git a/docker/Dockerfile.hpx-env b/docker/Dockerfile.hpx-env index 6bc449ff4..0b480dd0f 100644 --- a/docker/Dockerfile.hpx-env +++ b/docker/Dockerfile.hpx-env @@ -1,9 +1,9 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc +FROM quay.io/pypa/manylinux_2_28_x86_64:2024-02-08-a1b4ddc as build RUN dnf update -y -RUN dnf install -y ninja-build hwloc-devel +RUN dnf install -y ninja-build hwloc-devel && dnf clean all # Setup gcc 11 (to be compatible with concrete-compiler image) -RUN dnf install -y gcc-toolset-11 +RUN dnf install -y gcc-toolset-11 && dnf clean all ENV CC_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/gcc ENV CXX_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/c++ # Install boost @@ -15,17 +15,17 @@ RUN ./bootstrap.sh && ./b2 --with-filesystem install RUN git clone https://github.com/STEllAR-GROUP/hpx.git /hpx WORKDIR /hpx RUN git checkout v1.9.1 -RUN mkdir build # empty HPX_WITH_MAX_CPU_COUNT = dynamic # ref https://github.com/STEllAR-GROUP/hpx/blob/1.7.1/CMakeLists.txt#L759 -RUN cd build && cmake \ +WORKDIR /hpx/build +RUN cmake \ -DCMAKE_C_COMPILER=${CC_COMPILER} \ -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DHPX_WITH_MAX_CPU_COUNT="" \ -DHPX_WITH_FETCH_ASIO=on \ -DHPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY=ON \ - -DHPX_WITH_MALLOC=system .. -RUN cd build && make -j2 + -DHPX_WITH_MALLOC=system .. \ + && make -j2 FROM scratch -COPY --from=0 /hpx/ /hpx/ +COPY --from=build /hpx/ /hpx/