diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35fd1ef639d60b53e08fc9337cad03e45c29e744..499b238bca526f914a7f9b82381ea3c817903acc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,11 +16,15 @@ include: build_containers: stage: build - # Note: the docker-builder image has to be build and pushed manually once to bootstrap this job. - image: ${CI_REGISTRY_IMAGE}/docker-builder:master + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] rules: - !reference [.primary_ref_jobs, rules] - !reference [.merge_request_jobs, rules] + before_script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json script: # Find the directories that contain a file named 'Dockerfile', and (re)build the ones # that contain changed files. @@ -33,6 +37,10 @@ build_containers: then echo "Detected a changed file inside ./${DOCKER_DIR}/. (Re)build the container." IMAGE_NAME=$(echo "${DOCKER_DIR}" | sed -r 's|^dockerfiles/||') - buildimage "${DOCKER_DIR}" "${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}" + /kaniko/executor \ + --context "${CI_PROJECT_DIR}/${DOCKER_DIR}" \ + --dockerfile "${CI_PROJECT_DIR}/${DOCKER_DIR}/Dockerfile" \ + --destination "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}" \ + --cleanup fi done diff --git a/README.rst b/README.rst index 9a14ad83202440ebc9ff4cdc54e4b9bf6124db7f..3b37cb65e4633cae439cf0da37ca607a4b0e474c 100644 --- a/README.rst +++ b/README.rst @@ -122,7 +122,7 @@ The complete list of variables is given below. Name Required Protected Default value Purpose ======================= ======== ========= ========================== ================================================================ CONDA_BUILD_EXTRA_ARGS No No Optional extra arguments to conda-build. -CONDA_BUILD_COMMAND No No build Specify 'mambabuild' to use alternative build command from boa. +CONDA_BUILD_COMMAND No No conda mambabuild Specify 'boa build' to use alternative build command from boa. CONDARC No No ${CI_PROJECT_DIR}/.condarc Location of conda config file. CONDA_RECIPE_DIR No No conda-recipe Directory containing package recipe(s). CONDA_OUTPUT_FOLDER No No .conda-bld Directory where built package(s) are stored. diff --git a/dockerfiles/ci-tools/README.rst b/dockerfiles/ci-tools/README.rst index c26e63bca5e257757145f7d293bc21402a8b1055..ef44f0fd5669532690a5b6c11aab7d761661d69c 100644 --- a/dockerfiles/ci-tools/README.rst +++ b/dockerfiles/ci-tools/README.rst @@ -7,25 +7,25 @@ A Docker image containing various tools and utilities that come in handy during List of tools installed: - * git - * openssh-client - * rsync - * `s-nail`_ - * `gettext`_ - * `curl`_ - * `jq`_ - * `black`_ - * `autopep8`_ - * `release-cli`_ - * `python-gitlab`_ - * `junitparser`_ - * `anybadge`_ - * `python-compare-ast`_ - * `coverage-fixpaths`_ - * `git-scripts`_ - * ssh-addkey - * report-badge - * report-diff +* git +* openssh-client +* rsync +* `s-nail`_ +* `gettext`_ +* `curl`_ +* `jq`_ +* `black`_ +* `autopep8`_ +* `release-cli`_ +* `python-gitlab`_ +* `junitparser`_ +* `anybadge`_ +* `python-compare-ast`_ +* `coverage-fixpaths`_ +* `git-scripts`_ +* ssh-addkey +* report-badge +* report-diff ssh-addkey ========== diff --git a/dockerfiles/conda-builder/Dockerfile b/dockerfiles/conda-builder/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..29f86c2484af4effeb4f1020d4da06887e1748f6 --- /dev/null +++ b/dockerfiles/conda-builder/Dockerfile @@ -0,0 +1,64 @@ +FROM centos:7 + +LABEL description="Builds linux-64 conda packages. CentOS 7 + mambaforge + conda-build." + +# Configuration options. +ARG CONDA_INSTALL_DIR=/opt/conda +ARG CONDA_FORGE_PINNING=2022.02.15.10.00.06 + +# Set a UTF-8 locale. Useful for running Python 3 programs. +ENV LANG en_US.utf-8 +ENV LC_ALL en_US.utf-8 + +# Fetch updates and install dependencies. +# bzip2 is required for installing conda. +# git and openssh-clients are required for cloning git repositories in recipes. +# patch is required for applying patches in recipes. +RUN yum -y update \ + && yum -y install bzip2 git openssh-clients patch \ + && yum clean all \ + && rm -rf /var/cache/yum + +# Install the latest Miniconda with Python 3 and update everything. +RUN curl -L https://github.com/conda-forge/miniforge/releases/download/4.11.0-0/Mambaforge-4.11.0-0-Linux-x86_64.sh -o mambaforge.sh \ + && bash mambaforge.sh -b -p ${CONDA_INSTALL_DIR} \ + && rm mambaforge.sh \ + && ${CONDA_INSTALL_DIR}/bin/conda config --set show_channel_urls True \ + && ${CONDA_INSTALL_DIR}/bin/conda config --set channel_priority strict \ + # There are just too many packages that clobber paths. So don't set to + # path_conflict to `prevent` -- else we'll be overriding this setting to + # `warn` in lots of projects. + && ${CONDA_INSTALL_DIR}/bin/conda config --set path_conflict warn \ + && ${CONDA_INSTALL_DIR}/bin/conda config --set notify_outdated_conda false \ + # Set the variant file to the one from conda-forge-pinning. + # The conda-forge-pinning package (installed below) puts its variant + # file in $CONDA_INSTALL_DIR. Pointing conda-build to that variant file + # ensures we're compiling against libraries compatible with the wider + # conda-forge ecosystem. + && ${CONDA_INSTALL_DIR}/bin/conda config --set conda_build.config_file ${CONDA_INSTALL_DIR}/conda_build_config.yaml \ + # Explicitly fail when overlinking shared libraries. This will prevent dynamic library linking issues, + # and will be the default setting in conda build 4.0 anyway. + && ${CONDA_INSTALL_DIR}/bin/conda config --set conda_build.error_overlinking True \ + # Explicitly install a specific build of requests because of a chardet incompability. + # See https://github.com/conda-forge/requests-feedstock/pull/54. \ + # TODO: when the metadata of the affected requests packages has been patched, + # remove the `requests` line. + && ${CONDA_INSTALL_DIR}/bin/conda install --yes --update-all \ + conda-build \ + conda-verify \ + mamba \ + boa \ + coverage \ + coverage-fixpaths \ + conda-forge-pinning=${CONDA_FORGE_PINNING} \ + requests=2.28.0=pyhd8ed1ab_1 \ + && ${CONDA_INSTALL_DIR}/bin/conda clean -tipy \ + && ${CONDA_INSTALL_DIR}/bin/conda-build purge-all \ + && ${CONDA_INSTALL_DIR}/bin/conda init bash + +# Add a shell script that activates conda ... +COPY entrypoint.sh /opt/docker/bin/entrypoint.sh +# ... and make it the Docker entrypoint so that conda is available when we run a container. +ENTRYPOINT [ "/bin/bash", "/opt/docker/bin/entrypoint.sh" ] +# Provide a default command (`bash`), which will start if the user doesn't specify one. +CMD [ "/bin/bash" ] diff --git a/dockerfiles/conda-builder/README.rst b/dockerfiles/conda-builder/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..c06c9c35d770226f8845e902ac12d6e05815c836 --- /dev/null +++ b/dockerfiles/conda-builder/README.rst @@ -0,0 +1,25 @@ +============= +conda-builder +============= + +Docker image based upon CentOS 7 for building linux-64 conda packages on top of conda-forge. + +List of tools installed: + +* `conda`_ +* `conda-build`_ +* `conda-verify`_ +* `mamba`_ +* `boa`_ +* `coverage`_ +* `coverage-fixpaths`_ +* `conda-forge-pinning`_ + +.. _conda: https://docs.conda.io +.. _conda-build: https://docs.conda.io/projects/conda-build +.. _conda-verify: https://github.com/conda/conda-verify +.. _mamba: https://github.com/mamba-org/mamba +.. _boa: https://github.com/mamba-org/boa +.. _coverage: https://coverage.readthedocs.io +.. _coverage-fixpaths: https://github.com/omegacen/coverage-fixpaths +.. _conda-forge-pinning: https://github.com/conda-forge/conda-forge-pinning-feedstock diff --git a/dockerfiles/conda-builder/entrypoint.sh b/dockerfiles/conda-builder/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..190be4b75106321f4a564ff4906a9a7e6353b207 --- /dev/null +++ b/dockerfiles/conda-builder/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# GitLab runner doesn't source .bashrc, so we do it ourselves. +. ~/.bashrc + +# Run whatever the user wants to. +exec "$@" \ No newline at end of file diff --git a/dockerfiles/docker-builder/Dockerfile b/dockerfiles/docker-builder/Dockerfile deleted file mode 100644 index 5fed2fbd76ad6117a2e78b02880571a179da1b1a..0000000000000000000000000000000000000000 --- a/dockerfiles/docker-builder/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM gcr.io/kaniko-project/executor:debug - -LABEL description="Convenience wrapper around kaniko for building images in GitLab CI" - -RUN mkdir -p /kaniko/.docker -COPY entrypoint.sh /usr/local/bin/entrypoint -COPY buildimage.sh /usr/local/bin/buildimage - -ENTRYPOINT [ "/bin/sh", "/usr/local/bin/entrypoint" ] -CMD [ "/bin/sh" ] diff --git a/dockerfiles/docker-builder/README.rst b/dockerfiles/docker-builder/README.rst deleted file mode 100644 index 83ac72bdedbdde7ce593fd2e35d93e204c7f4455..0000000000000000000000000000000000000000 --- a/dockerfiles/docker-builder/README.rst +++ /dev/null @@ -1,17 +0,0 @@ -============== -docker-builder -============== - -Convenience wrapper around kaniko for building images in GitLab CI. - -Usage in GitLab CI: - -.. code-block:: yaml - - build_docker_image: - image: ${CI_REGISTRY}/omegacen/ci-templates/docker-builder - script: - - buildimage <dockerfile> [<image_subname>:]<tag> - -This will automatically upload the build image to the container registry of -the project this snippet is used in. diff --git a/dockerfiles/docker-builder/buildimage.sh b/dockerfiles/docker-builder/buildimage.sh deleted file mode 100755 index db9c8aa97c818fa7b79907ca47b0ebec0d3f86c1..0000000000000000000000000000000000000000 --- a/dockerfiles/docker-builder/buildimage.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -# Wrapper for building OCI images with Kaniko in GitLab CI. Pushes to -# $CI_REGISTRY_IMAGE/IMAGE_SUBNAME:TAG or to -# $CI_REGISTRY_IMAGE:TAG, depending on the second argument. -# Usage: -# -# buildimage.sh DOCKERFILE_OR_CONTEXT [IMAGE_SUBNAME:]TAG [EXTRA_KANIKO_ARGS] -# - -ABSOLUTE_PATH=$(readlink -f "$1") -IMAGE_NAME_AND_OR_TAG="$2" -shift -shift - -if [ -f "${ABSOLUTE_PATH}" ]; -then - # First argument is a file. - CONTEXT=$(dirname "${ABSOLUTE_PATH}") - DOCKERFILE="${ABSOLUTE_PATH}" -else - # Assume first argument is a directory and contains a file 'Dockerfile'. - CONTEXT="${ABSOLUTE_PATH}" - DOCKERFILE="${ABSOLUTE_PATH}/Dockerfile" -fi - -if echo "${IMAGE_NAME_AND_OR_TAG}" | grep -q ":"; -then - # With ":" separator: assume subname and tag - DESTINATION="${CI_REGISTRY_IMAGE}/${IMAGE_NAME_AND_OR_TAG}" -else - # Without ":" separator: assume tag only - DESTINATION="${CI_REGISTRY_IMAGE}:${IMAGE_NAME_AND_OR_TAG}" -fi - -/kaniko/executor \ - --context "${CONTEXT}" \ - --dockerfile "${DOCKERFILE}" \ - --destination "${DESTINATION}" \ - "$@" diff --git a/dockerfiles/docker-builder/entrypoint.sh b/dockerfiles/docker-builder/entrypoint.sh deleted file mode 100644 index 7b2dba526b13b773d2cbac2b60e4a44424c8f7cf..0000000000000000000000000000000000000000 --- a/dockerfiles/docker-builder/entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# Store authentication for the GitLab registry. -# See https://github.com/GoogleContainerTools/kaniko#pushing-to-different-registries -# and https://docs.gitlab.com/ee/ci/docker/using_kaniko.html -AUTH=$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n') -cat << EOF > /kaniko/.docker/config.json -{ - "auths": { - "${CI_REGISTRY}": { - "auth": "${AUTH}" - } - } -} -EOF - -# Run whatever the user wants to. -exec "$@" diff --git a/templates/conda/_shared.yml b/templates/conda/_shared.yml index 84fa19748583323b365c481b2bfb0af347d51d3c..91e9f3810246a8dd97f5393ba69ff6321df5b32c 100644 --- a/templates/conda/_shared.yml +++ b/templates/conda/_shared.yml @@ -5,4 +5,4 @@ variables: CONDA_RECIPE_DIR: conda-recipe CONDA_OUTPUT_FOLDER: .conda-bld CONDARC: "${CI_PROJECT_DIR}/.condarc" - CONDA_BUILD_COMMAND: build + CONDA_BUILD_COMMAND: conda mambabuild diff --git a/templates/conda/build.yml b/templates/conda/build.yml index e096c4cd3852c5f5e7e834c31a62846c5ae39957..24bf0ac9c6de41bddb58ea986c5e7bd146af2012 100644 --- a/templates/conda/build.yml +++ b/templates/conda/build.yml @@ -7,7 +7,7 @@ include: # .abstract_conda_build_test: - image: omegacen/conda-builder + image: ${CI_AWE_IMAGE_BASE}/conda-builder:${CI_AWE_IMAGE_TAG} rules: - if: $CI_AWE_SKIP_CONDA_BUILD_TEST when: never @@ -50,7 +50,7 @@ conda_build: export _CP_VERSION_PRE_SET=1 fi # The actual build command. - - conda ${CONDA_BUILD_COMMAND} ${CONDA_RECIPE_DIR} --no-test --no-anaconda-upload --output-folder ${CONDA_OUTPUT_FOLDER} ${CONDA_BUILD_EXTRA_ARGS} + - ${CONDA_BUILD_COMMAND} ${CONDA_RECIPE_DIR} --no-test --no-anaconda-upload --output-folder ${CONDA_OUTPUT_FOLDER} ${CONDA_BUILD_EXTRA_ARGS} # Derive and expose the package name, version, and build string. # We do this by looking at the filename of the conda package, # cf. https://docs.conda.io/projects/conda-build/en/latest/concepts/package-naming-conv.html. @@ -111,7 +111,7 @@ conda_test: # Run the actual test. - | env PYTEST_ADDOPTS="--junitxml=report.xml --override-ini junit_family=legacy" \ - conda ${CONDA_BUILD_COMMAND} ${CONDA_OUTPUT_FOLDER}/*/*.tar.bz2 --test --keep-old-work ${CONDA_BUILD_EXTRA_ARGS} \ + ${CONDA_BUILD_COMMAND} ${CONDA_OUTPUT_FOLDER}/*/*.tar.bz2 --test --keep-old-work ${CONDA_BUILD_EXTRA_ARGS} \ | tee conda_build_test.log after_script: # After the tests have run, we try to automatically extract the coverage and test reports.