diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e90415cde50f06d0a22e97f712c27a98aa3d101a..35fd1ef639d60b53e08fc9337cad03e45c29e744 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,38 @@ -image: node:lts-alpine +# Note: there's no job to lint the .yml files, because: +# +# - The GitLab API for linting does not support local includes +# and .extends is not fully supported [1]. +# - Local linting with e.g. check-jsonschema [2] +# does not support GitLab's custom !reference tag [3] [4]. +# +# [1] https://docs.gitlab.com/ee/api/lint.html#yaml-expansion +# [2] https://github.com/python-jsonschema/check-jsonschema +# [3] https://gitlab.com/gitlab-org/gitlab/-/issues/348666#note_804097628 +# [4] https://github.com/SchemaStore/schemastore/issues/1476 -lint: +include: + - local: '/templates/shared/all.yml' + - local: '/templates/changedfiles/all.yml' + +build_containers: + stage: build + # Note: the docker-builder image has to be build and pushed manually once to bootstrap this job. + image: ${CI_REGISTRY_IMAGE}/docker-builder:master + rules: + - !reference [.primary_ref_jobs, rules] + - !reference [.merge_request_jobs, rules] script: - - npm install -g gitlab-ci-lint - - find . \( -name '*.yaml' -o -name '*.yml' \) -print -exec gitlab-ci-lint --url https://gitlab.astro-wise.org "{}" \; + # Find the directories that contain a file named 'Dockerfile', and (re)build the ones + # that contain changed files. + - | + find . -name Dockerfile -type f | while read FILE; + do + # Strip leading './' and trailing '/<filename>' + DOCKER_DIR=$(echo "${FILE}" | sed -r 's|^\./||' | xargs dirname) + if grep -q "^${DOCKER_DIR}" changed_files.log; + then + echo "Detected a changed file inside ./${DOCKER_DIR}/. (Re)build the container." + IMAGE_NAME=$(echo "${DOCKER_DIR}" | sed -r 's|^dockerfiles/||') + buildimage "${DOCKER_DIR}" "${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}" + fi + done diff --git a/README.rst b/README.rst index 3a73a05d921e64665630ef9dc87c49a6092e0f88..7b06f6ffb422c1c2feb4da36abb0c6abf2189be4 100644 --- a/README.rst +++ b/README.rst @@ -10,17 +10,17 @@ Ready to include templates for common CI jobs. Usage ===== -autopep8.yml ------------- +Autopep8 +-------- By including this template in your project, its Python files will be linted with autopep8 for each Merge Request: - * Only the changed files will be checked with black. - * If they are not properly formatted according to black, the CI will fail. - * An assist-MR will be created to automatically fix the formatting. - * Upon merging this assist-MR, the changed files are checked again - and the CI should pass. +* Only the changed files will be checked with autopep8. +* If they are not properly formatted according to autopep8, the CI will fail. +* An assist-MR will be created to automatically fix the formatting. +* Upon merging this assist-MR, the changed files are checked again + and the CI should pass. To use the autopep8 CI template, follow these steps: @@ -32,8 +32,8 @@ To use the autopep8 CI template, follow these steps: include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'autopep8.yml' + ref: v5 + file: '/templates/autoformat/autopep8.yml' The variables that can be changed are: @@ -46,17 +46,17 @@ AUTOPEP8_IGNORE No No E203,E26 This CI template cannot be used together with the black CI template. -black.yml ---------- +Black +----- By including this template in your project, its Python files will be linted with `black`_ 19.10b0 for each Merge Request: - * Only the changed files will be checked with black. - * If they are not properly formatted according to black, the CI will fail. - * An assist-MR will be created to automatically fix the formatting. - * Upon merging this assist-MR, the changed files are checked again - and the CI should pass. +* Only the changed files will be checked with black. +* If they are not properly formatted according to black, the CI will fail. +* An assist-MR will be created to automatically fix the formatting. +* Upon merging this assist-MR, the changed files are checked again + and the CI should pass. The default line length is 120, although this can be changed via the ``BLACK_LINE_LENGTH`` variable. @@ -71,8 +71,8 @@ To use the black CI template, follow these steps: include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'black.yml' + ref: v5 + file: '/templates/autoformat/black.yml' The variables that can be changed are: @@ -84,8 +84,8 @@ BLACK_LINE_LENGTH No No 120 How many characters pe This CI template cannot be used together with the autopep8 CI template. -conda.yml ---------- +Conda +----- By including this template, the CI will automatically build one or more conda recipes in the repository and upload the built package our conda channel, @@ -113,8 +113,8 @@ Next, include the following snippet in your ``.gitlab-ci.yml`` file: include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'conda.yml' + ref: v5 + file: '/templates/conda/all.yml' The complete list of variables is given below. @@ -148,8 +148,8 @@ by setting ``CONDA_BUILD_COMMAND`` to ``mambabuild`` or by using the experimental conda feature by setting ``CONDA_BUILD_EXTRA_ARGS`` to ``--experimental-solver=libmamba``. -conda-build.yml ---------------- +Conda build +----------- It is possible to only include the template for the build stage, in case you do not want to automatically upload your build recipe but only want to test it. @@ -160,12 +160,12 @@ In that case, you must include the ``conda-build.yml`` template instead of include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'conda-build.yml' + ref: v5 + file: '/templates/conda/build.yml' -monthlymerge.yml ----------------- +Monthly Merge +------------- This template performs an automatic release merge and tag based on a schedule (usually monthly). It performs the following steps: @@ -173,6 +173,10 @@ This template performs an automatic release merge and tag based on a schedule * Merge the ``develop`` branch into the ``master`` branch, * Bump the version in specified files according to CalVer_ ``YYYY.MM.MICRO``, * Create a tag on the ``master`` branch. +* Create a `GitLab release`_ associated to the tag. + +The last step is done via the CI template described below, which is included +in the Monthly Merge template. To use it, follow these steps: @@ -182,8 +186,8 @@ To use it, follow these steps: include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'monthlymerge.yml' + ref: v5 + file: '/templates/monthlymerge.yml' #. Next, create a `pipeline schedule`_ in your project. Set it to the first of the month. This pipeline schedule should ideally be owned by `CI Bot`_. @@ -199,19 +203,47 @@ To use it, follow these steps: The complete list of variables is given below. -=============== ======== ========= ==================================== =================================================================== - Name Required Protected Default value Purpose -=============== ======== ========= ==================================== =================================================================== -MM_PRIVATE_KEY Yes Yes The private SSH key of the user that own the pipeline schedule. -MM_RUN_JOB Yes No Must be set for the job to run. Add this to the schedule variables. -MM_SOURCE No No develop The source branch to be merged into the target branch. -MM_TARGET No No master The target branch into which the source branch will be merged. -MM_BUMP_FILES No No `__init__.py conda-recipe/meta.yaml` A space delimited list of files whose versions will be bumped. -=============== ======== ========= ==================================== =================================================================== +======================== ======== ========= ==================================== =================================================================== + Name Required Protected Default value Purpose +======================== ======== ========= ==================================== =================================================================== +CI_AWE_RUN_MONTHLY_MERGE Yes No Must be set for the job to run. Add this to the schedule variables. +MM_PRIVATE_KEY Yes Yes The private SSH key of the user that own the pipeline schedule. +MM_SOURCE No No develop The source branch to be merged into the target branch. +MM_TARGET No No master The target branch into which the source branch will be merged. +MM_BUMP_FILES No No `__init__.py conda-recipe/meta.yaml` A space delimited list of files whose versions will be bumped. +======================== ======== ========= ==================================== =================================================================== +Create GitLab releases on tags +------------------------------ -sonarqube.yml -------------- +This template automatically creates a `GitLab release`_ on tags that match +either a `CalVer`_ pattern or a `SemVer`_ pattern. For CalVer tags matching +the pattern ``^\d{4}\.\d{2}\.\d+$`` you must include the following snippet in +your ``.gitlab-ci.yml`` file: + +.. code-block:: yaml + + include: + - project: 'omegacen/ci-templates' + ref: v5 + file: '/templates/release/calver.yml' + +Whereas for SemVer tags mathching the pattern ``^\d+\.\d+\.\d+$``, you must +include the following snippet: + +.. code-block:: yaml + + include: + - project: 'omegacen/ci-templates' + ref: v5 + file: '/templates/release/semver.yml' + +The automatically created release has the same name as the associated tag, +and its release notes contain a list of merged MRs to the default branch +since the previous release. + +SonarQube +--------- By including this template, SonarQube source code analysis will automatically be run on the default branch and on merge requests. @@ -226,8 +258,8 @@ To use it, follow these steps: include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'sonarqube.yml' + ref: v5 + file: '/templates/sonarqube.yml' #. Optionally, you can add one or more `Project badges`_. In your GitLab project, go to *Settings* -> *General* -> *Badges*. Then add the following: @@ -253,8 +285,8 @@ SONAR_PYLINT_RULES No No `C0326:MINOR:1,C0328:MINOR:1,[...], ========================= ======== ========= =========================================================== ============================================================== -latex.yml ---------- +LaTeX +----- This template builds a pdf file from latex sources. For feature branches and merge requests it will also generate a pdf that highlights the differences @@ -266,8 +298,8 @@ To use it, include the following snippet in your ``.gitlab-ci.yml`` file: include: - project: 'omegacen/ci-templates' - ref: v4 - file: 'latex.yml' + ref: v5 + file: '/templates/latex.yml' The filename of the main tex file is determined automatically. This can be overruled by specifying the ``FILENAME_TEX`` variable. @@ -281,6 +313,158 @@ FILENAME_TEX No No `automatically determined` Manual specification ============ ======== ========= ========================== ====================================== +Test report badge +----------------- + +This template generates a badge that shows the percentage of successful tests in the +test suite. It requires another job earlier in the pipeline that generates a JUnit test +report. The ``conda_test`` job does generates such a report, if the conda recipe +includes tests. + +To use this template, include the following snippet in your ``.gitlab-ci.yml`` file: + +.. code-block:: yaml + + include: + - project: 'omegacen/ci-templates' + ref: v5 + file: '/templates/testreport/badge.yml' + +Next, add badge to the `Project badges`_. In your GitLab project, go to +*Settings* -> *General* -> *Badges*. Then add the following: + +* Name: ``Test Success Rate`` +* Link: ``https://gitlab.astro-wise.org/%{project_path}/-/pipelines/%{default_branch}/latest`` +* Badge image URL: ``https://gitlab.astro-wise.org/%{project_path}/-/jobs/artifacts/%{default_branch}/raw/report.svg?job=test_report_badge`` + +The complete list of variables for this CI template is given below. + +========================= ======== ========= ============= ===================================== + Name Required Protected Default value Purpose +========================= ======== ========= ============= ===================================== +TEST_REPORT_ARTIFACT_FILE No No `report.xml` Test report file to create badge for. +========================= ======== ========= ============= ===================================== + +Test report change detection +---------------------------- + +This template compares the test report of the current pipeline to the report of +the previous pipeline. If there are tests that passed before but fail now, +this job fails. This is particularly useful if the test suite has failing tests +that are not easily fixable (like for example ``astro`` has), but you're still +interested in regressions. + +This template requires another job earlier in the pipeline that generates a JUnit test +report. The ``conda_test`` job does generates such a report, if the conda recipe +includes tests. + +To use this template, include the following snippet in your ``.gitlab-ci.yml`` file: + +.. code-block:: yaml + + include: + - project: 'omegacen/ci-templates' + ref: v5 + file: '/templates/testreport/diff.yml' + +The complete list of variables for this CI template is given below. + +========================= ======== ========= ============= ========================================== + Name Required Protected Default value Purpose +========================= ======== ========= ============= ========================================== +TEST_REPORT_ARTIFACT_FILE No No `report.xml` Test report file to create badge for. +TEST_REPORT_JOB No No `conda_test` Name of the job that generates the report. +========================= ======== ========= ============= ========================================== + +List changed files +------------------ + +This template generates list of the changed files in a Merge Request or since the last +push to a branch. + +To use this template, include the following snippet in your ``.gitlab-ci.yml`` file: + +.. code-block:: yaml + + include: + - project: 'omegacen/ci-templates' + ref: v5 + file: '/templates/changedfiles/all.yml' + +Jobs in later stages of the pipeline will then have access to the ``changed_files.log`` +artifacts. This file contains a list of all changed files in either the MR or since +that last push to the branch. + +Controlling when jobs runs +========================== + +By including a CI template, the jobs in it get automatically run under the right circumstances. +E.g., by including the conda template, a conda package is build and tested for each MR to the +main branches, and for each commit on the main branches a conda packages is build, tested, +and released. + +However, if you want more control over when a particular job runs, you can use the +``CI_AWE_RUN_<JOBNAME>`` and ``CI_AWE_SKIP_<JOBNAME>`` variables to run or to skip +a job, respectively. The full list of variables is as follows: + +========================= ============================= ============================ ========================================= + Job name Skip variable Run variable Template +========================= ============================= ============================ ========================================= +`autopep8` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/autopep8.yml` +`black` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/black.yml` +`changed_files_mr` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/mergerequest.yml` +`changed_files_push` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/push.yml` +`conda_build` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml` +`conda_test` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml` +`conda_upload` CI_AWE_SKIP_CONDA_UPLOAD CI_AWE_RUN_CONDA_UPLOAD `templates/conda/release.yml` +`latex_pdf` CI_AWE_SKIP_LATEX_PDF CI_AWE_RUN_LATEX_PDF `templates/latex.yml` +`latex_pdf_diff` CI_AWE_SKIP_LATEX_PDF_DIFF CI_AWE_RUN_LATEX_PDF_DIFF `templates/latex.yml` +`sonar_branch` CI_AWE_SKIP_SONAR_BRANCH CI_AWE_RUN_SONAR_BRANCH `templates/sonarqube.yml` +`sonar_mr` CI_AWE_SKIP_SONAR_MR CI_AWE_RUN_SONAR_MR `templates/sonarqube.yml` +`test_report_badge` CI_AWE_SKIP_TEST_REPORT_BADGE CI_AWE_RUN_TEST_REPORT_BADGE `templates/testreport/badge.yml` +`test_report_diff_mr` CI_AWE_SKIP_TEST_REPORT_DIFF CI_AWE_RUN_TEST_REPORT_DIFF `templates/testreport/diff.yml` +`test_report_diff_branch` CI_AWE_SKIP_TEST_REPORT_DIFF CI_AWE_RUN_TEST_REPORT_DIFF `templates/testreport/diff.yml` +========================= ============================= ============================ ========================================= + +In addition, you can use the ``CI_AWE_SKIP_ALL`` and ``CI_AWE_RUN_ALL`` variables to +control whether any or all of these jobs run. + +For example, say you want to run the conda builds also on all +merge requests (not just on MRs to the main branches). You can then add the +following to your ``.gitlab-ci.yml`: + +.. code-block:: yaml + + workflow: + rules: + - if: $CI_MERGE_REQUEST_ID + variables: + CI_AWE_RUN_CONDA_BUILD_TEST: '1' + - when: always + +Note that the last ``when: always`` is needed in this example because otherwise +no jobs would run when there is no merge request. + +Or, when you only want to run the SonarQube job (and nothing else) when the target +of the merge request is the default branch: + +.. code-block:: yaml + + workflow: + rules: + - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH + variables: + CI_AWE_RUN_SONAR_MR: '1' + CI_AWE_SKIP_ALL: '1' + - when: always + +The variable precedence is as follows (highest first): + +1. ``CI_AWE_SKIP_<JOBNAME>`` +2. ``CI_AWE_RUN_<JOBNAME>`` +3. ``CI_AWE_SKIP_ALL`` +4. ``CI_AWE_RUN_ALL`` + Template versioning =================== @@ -298,6 +482,8 @@ changes every now and then. .. _conda-wise documentation: http://omegacen.pages.astro-wise.org/conda/maintainers.html .. _CI variables: https://docs.gitlab.com/ce/ci/variables/#variables .. _CalVer: http://calver.org/ +.. _SemVer: https://semver.org/ .. _pipeline schedule: https://docs.gitlab.com/ce/user/project/pipelines/schedules.html .. _CI Bot: https://gitlab.astro-wise.org/ci-bot .. _Project badges: https://docs.gitlab.com/ce/user/project/badges.html +.. _GitLab release: https://docs.gitlab.com/ee/user/project/releases/ diff --git a/conda-release.yml b/conda-release.yml deleted file mode 100644 index 9e068c7265bd04b1e74d009650c9fca2d207c909..0000000000000000000000000000000000000000 --- a/conda-release.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Shared configuration. -# -# Note: this cannot be placed in a seperate file because then it will be -# included twice by GitLab in the main conda.yml file. And including -# a .yml file twice yields an error (at least with GitLab 11.9). -# - -stages: - - lint - - build - - test - - quality - - release - -variables: - CONDA_RECIPE_DIR: conda-recipe - CONDA_OUTPUT_FOLDER: .conda-bld - CONDARC: "${CI_PROJECT_DIR}/.condarc" - -# -# Release -# - -# Upload packages. Only for main branches. -conda_upload: - stage: release - image: omegacen/gitlabci-easyssh - dependencies: - - conda_build - rules: - - if: $CI_PIPELINE_SOURCE != 'schedule' && $CI_COMMIT_TAG - - if: $CI_PIPELINE_SOURCE != 'schedule' && $CI_COMMIT_BRANCH == 'master' - - if: $CI_PIPELINE_SOURCE != 'schedule' && $CI_COMMIT_BRANCH == 'develop' - - if: $CI_PIPELINE_SOURCE != 'schedule' && $CI_COMMIT_BRANCH =~ /^release\/.*$/ - - if: $CI_PIPELINE_SOURCE != 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - before_script: - - if [ -z "${CONDA_UPLOAD_KEY}" ]; then echo "CI variable 'CONDA_UPLOAD_KEY' is not set, exiting."; false; fi - script: - - ssh-addkey "${CONDA_UPLOAD_KEY}" - - rsync -chavz --include="*/" --include="*.tar.bz2" --exclude="*" - ${CONDA_OUTPUT_FOLDER}/ conda@129.125.6.100:~/public_html diff --git a/conda.yml b/conda.yml deleted file mode 100644 index be6ee4fc597b4c0976a15fce211d21583fd83f02..0000000000000000000000000000000000000000 --- a/conda.yml +++ /dev/null @@ -1,3 +0,0 @@ -include: - - local: '/conda-build.yml' - - local: '/conda-release.yml' diff --git a/dockerfiles/ci-tools/Dockerfile b/dockerfiles/ci-tools/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d1b2b49ff55a39716a70befde1a23a1bd421eb0c --- /dev/null +++ b/dockerfiles/ci-tools/Dockerfile @@ -0,0 +1,43 @@ +FROM python:3 + +LABEL description="Various tools that come in handy during (GitLab) CI" + +# Install via apt +RUN apt-get update && apt-get -y install \ + jq \ + rsync \ + s-nail \ + gettext \ + && apt-get clean +# Install via pip +# https://github.com/weiwei/junitparser/pull/89 is in junitparser 2.6.0. \ +# https://github.com/weiwei/junitparser/pull/90 is not yet merged :(. +RUN pip install \ + black==22.3.0 \ + autopep8 \ + python-gitlab \ + python-compare-ast \ + coverage-fixpaths \ + anybadge \ + "junitparser>=2.6" + +# Install Gitlab release-cli +RUN curl --location --output /usr/local/bin/release-cli "https://gitlab.com/gitlab-org/release-cli/-/releases/permalink/latest/downloads/bin/release-cli-linux-amd64" \ + && chmod +x /usr/local/bin/release-cli + +# Install git-scripts +RUN git clone https://gitlab.astro-wise.org/omegacen/git-scripts.git ~/git-scripts \ + && mv ~/git-scripts/bin/* /usr/local/bin/ \ + && rm -rf ~/git-scripts + +# Install ssh-addkey +RUN mkdir -p ~/.ssh +RUN echo "Host *\n\tStrictHostKeyChecking no" > ~/.ssh/config +COPY ssh-addkey.sh /usr/local/bin/ssh-addkey +COPY python-gitlab-set-private-token.sh /usr/local/bin/python-gitlab-set-private-token +COPY report_badge.py /usr/local/bin/report_badge +COPY report_diff.py /usr/local/bin/report_diff + +COPY entrypoint.sh /usr/local/bin/entrypoint +ENTRYPOINT [ "/bin/bash", "/usr/local/bin/entrypoint" ] +CMD [ "/bin/bash" ] diff --git a/dockerfiles/ci-tools/README.rst b/dockerfiles/ci-tools/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..c26e63bca5e257757145f7d293bc21402a8b1055 --- /dev/null +++ b/dockerfiles/ci-tools/README.rst @@ -0,0 +1,85 @@ +======== +ci-tools +======== + +A Docker image containing various tools and utilities that come in handy during +(GitLab) CI jobs. + +List of tools installed: + + * git + * openssh-client + * rsync + * `s-nail`_ + * `gettext`_ + * `curl`_ + * `jq`_ + * `black`_ + * `autopep8`_ + * `release-cli`_ + * `python-gitlab`_ + * `junitparser`_ + * `anybadge`_ + * `python-compare-ast`_ + * `coverage-fixpaths`_ + * `git-scripts`_ + * ssh-addkey + * report-badge + * report-diff + +ssh-addkey +========== + +Easily add private SSH keys during your GitLab CI jobs. + +This utility reduces the hassle of adding a private SSH key to running +Docker containers to one command. This is particularly useful for deploy +jobs in GitLab CI, hence the name. + +While running a container (interactively), you can add a private key as follows: + +.. code-block:: + + $ ssh-addkey "${SSH_PRIVATE_KEY}" + +where the ``SSH_PRIVATE_KEY`` variable contains your private key. You can then +either ssh, rsync, or use git to sync to your favorite deploy server. + +report-badge +============ + +Generates a CI badge from a test report with the percentage of passed tests. +Usage: + +.. code-block:: + + $ report-badge <input JUnit report file> <output SVG badge file> + +report-diff +=========== + +Create a diff of two JUnit test reports. The resulting test report contains only +tests that are present in both of the input reports. In addition, their status has +to have changed in order for the tests to be included. The status of the last input +report is shown in the diff. + +Usage: + +.. code-block:: + + $ report-diff <JUnit report before> <JUnit report after> <JUnit diff report> + + +.. _s-nail: https://wiki.archlinux.org/title/S-nail +.. _gettext: https://www.gnu.org/software/gettext/ +.. _curl: https://curl.se/ +.. _jq: https://stedolan.github.io/jq/ +.. _black: https://black.readthedocs.io +.. _autopep8: https://github.com/hhatto/autopep8 +.. _release-cli: https://gitlab.com/gitlab-org/release-cli +.. _junitparser: https://github.com/weiwei/junitparser +.. _anybadge: https://github.com/jongracecox/anybadge +.. _python-gitlab: https://python-gitlab.readthedocs.io +.. _python-compare-ast: https://github.com/omegacen/python-compare-ast +.. _coverage-fixpaths: https://github.com/omegacen/coverage-fixpaths +.. _git-scripts: https://gitlab.astro-wise.org/omegacen/git-scripts diff --git a/dockerfiles/ci-tools/entrypoint.sh b/dockerfiles/ci-tools/entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..d6cc6d7aafd35bbee045581f60bdb08fd20728ad --- /dev/null +++ b/dockerfiles/ci-tools/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# Propagate GitLab CI variables to Git. +if [ -n "${GITLAB_USER_EMAIL}" ]; then + git config --global user.email "${GITLAB_USER_EMAIL}" +fi +if [ -n "${GITLAB_USER_NAME}" ]; then + git config --global user.name "${GITLAB_USER_NAME}" +fi + +# Set python-gitlab configuration. +if [ -n "${GITLAB_CI}" ]; then + cat << EOF > ~/.python-gitlab.cfg +[global] +default = current-ci-server + +[current-ci-server] +url = ${CI_SERVER_URL} +job_token = ${CI_JOB_TOKEN} +api_version = 4 +EOF +fi + +# Run whatever the user wants to. +exec "$@" \ No newline at end of file diff --git a/dockerfiles/ci-tools/python-gitlab-set-private-token.sh b/dockerfiles/ci-tools/python-gitlab-set-private-token.sh new file mode 100755 index 0000000000000000000000000000000000000000..d7a895d10b5ab344252c629f632aead05f109b32 --- /dev/null +++ b/dockerfiles/ci-tools/python-gitlab-set-private-token.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ -z "${GITLAB_CI}" ]; then + echo "Not running during GitLab CI, exiting." + exit 1 +fi + +cat << EOF > ~/.python-gitlab.cfg +[global] +default = current-ci-server + +[current-ci-server] +url = ${CI_SERVER_URL} +private_token = $1 +api_version = 4 +EOF \ No newline at end of file diff --git a/dockerfiles/ci-tools/report_badge.py b/dockerfiles/ci-tools/report_badge.py new file mode 100755 index 0000000000000000000000000000000000000000..993c4ffe8d83436aeecc27d99e2069d875f6d956 --- /dev/null +++ b/dockerfiles/ci-tools/report_badge.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse + +from anybadge import Badge +from junitparser import JUnitXml + + +def create_badge(report_path, badge_path): + report = JUnitXml.fromfile(report_path) + report.update_statistics() + total = report.tests - report.skipped + passed = total - report.errors - report.failures + percentage = 100 * passed / total + color = 'green' if passed == total else 'red' + badge = Badge(f'tests', f'{percentage:.1f}%', default_color=color) + badge.write_badge(badge_path) + + +def main(): + parser = argparse.ArgumentParser(description='Create a CI badge from a test report.') + parser.add_argument('report', metavar='XML', type=str, help='Path of input JUnit XML report file') + parser.add_argument('badge', metavar='SVG', type=str, help='Path of output SVG file') + args = parser.parse_args() + create_badge(args.report, args.badge) + + +if __name__ == '__main__': + main() diff --git a/dockerfiles/ci-tools/report_diff.py b/dockerfiles/ci-tools/report_diff.py new file mode 100755 index 0000000000000000000000000000000000000000..d67884f1b753ffd7d816bd39aa2934bcb6b27a7c --- /dev/null +++ b/dockerfiles/ci-tools/report_diff.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import argparse + +from junitparser import JUnitXml, TestSuite + + +def case_dictionary(report): + return { + (case.classname, case.name): case + for suite in report + for case in suite + } + + +def compare_reports(path_before, path_after, path_out): + before = JUnitXml.fromfile(path_before) + after = JUnitXml.fromfile(path_after) + before_cases = case_dictionary(before) + after_cases = case_dictionary(after) + + changed_cases = TestSuite('tests_with_changed_result') + + for key, after_case in after_cases.items(): + if key in before_cases: + before_case = before_cases[key] + before_results = [type(r) for r in before_case.result] + after_results = [type(r) for r in after_case.result] + if set(before_results) != set(after_results): + changed_cases.add_testcase(after_case) + + xml = JUnitXml() + xml.add_testsuite(changed_cases) + xml.update_statistics() + xml.write(path_out, to_console=False) + + +def main(): + parser = argparse.ArgumentParser( + description='Create a diff of two JUnit test reports. The resulting test report contains only ' + 'tests that are present in both of the input reports. In addition, their status has ' + 'to have changed in order for the tests to be included. ' + 'The status of the last input report is shown in the diff.' + ) + parser.add_argument("before", help="Path of the initial XML report to compare.") + parser.add_argument("after", help="Path of the re-run XML report to compare.") + parser.add_argument("output", help='Path to write diff report to.') + args = parser.parse_args() + compare_reports(args.before, args.after, args.output) + + +if __name__ == '__main__': + main() diff --git a/dockerfiles/ci-tools/ssh-addkey.sh b/dockerfiles/ci-tools/ssh-addkey.sh new file mode 100755 index 0000000000000000000000000000000000000000..5218ddb35cd84314c7e4b59062cb4ba76e0e8060 --- /dev/null +++ b/dockerfiles/ci-tools/ssh-addkey.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Find a non-existing filename for the private key +fileprefix="${HOME}/.ssh/id_" +i=0 +while [[ -e ${fileprefix}${i} ]] ; do + (( i++ )) +done +file="${fileprefix}${i}" + +# Put the key in the file and make it read-only +echo "$1" > "${file}" +chmod 600 "${file}" + +# Make SSH aware of the private key +echo -e "\tIdentityFile ${file}" >> ~/.ssh/config \ No newline at end of file diff --git a/dockerfiles/docker-builder/Dockerfile b/dockerfiles/docker-builder/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5fed2fbd76ad6117a2e78b02880571a179da1b1a --- /dev/null +++ b/dockerfiles/docker-builder/Dockerfile @@ -0,0 +1,10 @@ +FROM gcr.io/kaniko-project/executor:debug + +LABEL description="Convenience wrapper around kaniko for building images in GitLab CI" + +RUN mkdir -p /kaniko/.docker +COPY entrypoint.sh /usr/local/bin/entrypoint +COPY buildimage.sh /usr/local/bin/buildimage + +ENTRYPOINT [ "/bin/sh", "/usr/local/bin/entrypoint" ] +CMD [ "/bin/sh" ] diff --git a/dockerfiles/docker-builder/README.rst b/dockerfiles/docker-builder/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..83ac72bdedbdde7ce593fd2e35d93e204c7f4455 --- /dev/null +++ b/dockerfiles/docker-builder/README.rst @@ -0,0 +1,17 @@ +============== +docker-builder +============== + +Convenience wrapper around kaniko for building images in GitLab CI. + +Usage in GitLab CI: + +.. code-block:: yaml + + build_docker_image: + image: ${CI_REGISTRY}/omegacen/ci-templates/docker-builder + script: + - buildimage <dockerfile> [<image_subname>:]<tag> + +This will automatically upload the build image to the container registry of +the project this snippet is used in. diff --git a/dockerfiles/docker-builder/buildimage.sh b/dockerfiles/docker-builder/buildimage.sh new file mode 100755 index 0000000000000000000000000000000000000000..db9c8aa97c818fa7b79907ca47b0ebec0d3f86c1 --- /dev/null +++ b/dockerfiles/docker-builder/buildimage.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# Wrapper for building OCI images with Kaniko in GitLab CI. Pushes to +# $CI_REGISTRY_IMAGE/IMAGE_SUBNAME:TAG or to +# $CI_REGISTRY_IMAGE:TAG, depending on the second argument. +# Usage: +# +# buildimage.sh DOCKERFILE_OR_CONTEXT [IMAGE_SUBNAME:]TAG [EXTRA_KANIKO_ARGS] +# + +ABSOLUTE_PATH=$(readlink -f "$1") +IMAGE_NAME_AND_OR_TAG="$2" +shift +shift + +if [ -f "${ABSOLUTE_PATH}" ]; +then + # First argument is a file. + CONTEXT=$(dirname "${ABSOLUTE_PATH}") + DOCKERFILE="${ABSOLUTE_PATH}" +else + # Assume first argument is a directory and contains a file 'Dockerfile'. + CONTEXT="${ABSOLUTE_PATH}" + DOCKERFILE="${ABSOLUTE_PATH}/Dockerfile" +fi + +if echo "${IMAGE_NAME_AND_OR_TAG}" | grep -q ":"; +then + # With ":" separator: assume subname and tag + DESTINATION="${CI_REGISTRY_IMAGE}/${IMAGE_NAME_AND_OR_TAG}" +else + # Without ":" separator: assume tag only + DESTINATION="${CI_REGISTRY_IMAGE}:${IMAGE_NAME_AND_OR_TAG}" +fi + +/kaniko/executor \ + --context "${CONTEXT}" \ + --dockerfile "${DOCKERFILE}" \ + --destination "${DESTINATION}" \ + "$@" diff --git a/dockerfiles/docker-builder/entrypoint.sh b/dockerfiles/docker-builder/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..7b2dba526b13b773d2cbac2b60e4a44424c8f7cf --- /dev/null +++ b/dockerfiles/docker-builder/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Store authentication for the GitLab registry. +# See https://github.com/GoogleContainerTools/kaniko#pushing-to-different-registries +# and https://docs.gitlab.com/ee/ci/docker/using_kaniko.html +AUTH=$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n') +cat << EOF > /kaniko/.docker/config.json +{ + "auths": { + "${CI_REGISTRY}": { + "auth": "${AUTH}" + } + } +} +EOF + +# Run whatever the user wants to. +exec "$@" diff --git a/_autoformat.yml b/templates/autoformat/_shared.yml similarity index 92% rename from _autoformat.yml rename to templates/autoformat/_shared.yml index 299519ba35e5c70b20c28086fedd7022e544a388..00424e93c1ab63660aa94edd8936504667f84faa 100644 --- a/_autoformat.yml +++ b/templates/autoformat/_shared.yml @@ -1,26 +1,13 @@ -stages: - - lint - - build - - test - - quality - - release +include: + - local: '/templates/shared/all.yml' compare_ast: stage: lint - image: omegacen/autoformat + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} rules: - - if: $AUTOFORMAT_COMPARE_AST && $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^assist\/autoformat\/.*$/ + - if: $CI_AWE_RUN_COMPARE_AST && $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^assist\/autoformat\/.*$/ script: - - | - cat << EOF > ~/.python-gitlab.cfg - [global] - default = astro-wise - - [astro-wise] - url = ${CI_SERVER_URL} - private_token = ${AUTOFORMAT_TOKEN} - api_version = 4 - EOF + - python-gitlab-set-private-token ${AUTOFORMAT_TOKEN} - | AST_OUTPUT=$(python-compare-ast 2>&1) && AST_RESULT=0 || AST_RESULT=1 echo "$AST_OUTPUT" @@ -43,27 +30,20 @@ compare_ast: .autoformat_mr: stage: lint - image: omegacen/autoformat + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} variables: AUTOFORMAT_MR_LABELS: autoformat AUTOFORMAT_COMMENT_MARKER: autoformat-comment AUTOFORMAT_NAME: abstractautoformatter rules: - - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/ + - if: $CI_AWE_SKIP_AUTOFORMAT + when: never + - if: $CI_AWE_RUN_AUTOFORMAT + - !reference [.merge_request_jobs, rules] script: - export -f autoformat-check - export -f autoformat-change - # Save configuration for python-gitlab. Somehow we can't use simple environment variables for this $%@#$@. - - | - cat << EOF > ~/.python-gitlab.cfg - [global] - default = astro-wise - - [astro-wise] - url = ${CI_SERVER_URL} - private_token = ${AUTOFORMAT_TOKEN} - api_version = 4 - EOF + - python-gitlab-set-private-token ${AUTOFORMAT_TOKEN} - | # Run the formatter only on changed files. CHANGED_FILES=$(git diff --diff-filter=d --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME...$CI_COMMIT_SHA -- | grep '\.py$' || true) diff --git a/autopep8.yml b/templates/autoformat/autopep8.yml similarity index 95% rename from autopep8.yml rename to templates/autoformat/autopep8.yml index f652af1a85ead0bce365831ab855a91ef1471dbc..cd3cc6fcea98552043e1521aa50f70e9faea2158 100644 --- a/autopep8.yml +++ b/templates/autoformat/autopep8.yml @@ -1,8 +1,8 @@ include: - - local: '_autoformat.yml' + - local: '/templates/autoformat/_shared.yml' variables: - AUTOFORMAT_COMPARE_AST: 1 + CI_AWE_RUN_COMPARE_AST: 1 autopep8: extends: .autoformat_mr diff --git a/black.yml b/templates/autoformat/black.yml similarity index 87% rename from black.yml rename to templates/autoformat/black.yml index ad5f8b0a3ee09e48f7c606c7676cf8e92b52c6a2..4f900768c34199f06382fe9dd6da89c4f70a8b9e 100644 --- a/black.yml +++ b/templates/autoformat/black.yml @@ -1,7 +1,7 @@ include: - - local: '_autoformat.yml' + - local: '/templates/autoformat/_shared.yml' -autopep8: +black: extends: .autoformat_mr variables: AUTOFORMAT_MR_LABELS: autoformat,black diff --git a/templates/changedfiles/_shared.yml b/templates/changedfiles/_shared.yml new file mode 100644 index 0000000000000000000000000000000000000000..87f6e396fb9d4ce9443eaf1b8f47c86176fd9fc0 --- /dev/null +++ b/templates/changedfiles/_shared.yml @@ -0,0 +1,28 @@ +# Stores the list of changed files in a CI artifact `changed_files.log`. +# Useful for subsequent jobs that need such a list but cannot generate it +# themselves (e.g. because they don't have access to git, such as jobs in +# the kaniko image). + +# Storing the list of changed files in a dotenv file (and subsequently exposing +# it as an environment variable) would have been nicer. But we cannot store the +# list of changed files in dotenv file because at the time +# of writing GitLab does not support multiline dotenv variables [1]. +# +# [1] https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsdotenv + +include: + - local: '/templates/shared/all.yml' + +.changed_files: + variables: + GIT_DEPTH: 0 + image: + name: alpine/git + entrypoint: [""] + stage: pre + script: + - git diff --name-only ${CHANGED_FILES_COMPARE_REF}...${CI_COMMIT_SHA} > changed_files.log + artifacts: + paths: + - changed_files.log + expire_in: 1 day diff --git a/templates/changedfiles/all.yml b/templates/changedfiles/all.yml new file mode 100644 index 0000000000000000000000000000000000000000..8e1e773e02b1b409e4ce9631f00e59125f4b631b --- /dev/null +++ b/templates/changedfiles/all.yml @@ -0,0 +1,3 @@ +include: + - local: '/templates/changedfiles/mergerequest.yml' + - local: '/templates/changedfiles/push.yml' diff --git a/templates/changedfiles/mergerequest.yml b/templates/changedfiles/mergerequest.yml new file mode 100644 index 0000000000000000000000000000000000000000..1020554705776cb1c2db25ea6b947a4e897ee6b8 --- /dev/null +++ b/templates/changedfiles/mergerequest.yml @@ -0,0 +1,12 @@ +include: + - local: '/templates/changedfiles/_shared.yml' + +changed_files_mr: + extends: .changed_files + variables: + CHANGED_FILES_COMPARE_REF: origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + rules: + - if: $CI_AWE_SKIP_CHANGED_FILES + when: never + - if: $CI_AWE_RUN_CHANGED_FILES + - !reference [.merge_request_jobs, rules] diff --git a/templates/changedfiles/push.yml b/templates/changedfiles/push.yml new file mode 100644 index 0000000000000000000000000000000000000000..bea5621d80ec493e1f2a26d7dc68d60cd71981f2 --- /dev/null +++ b/templates/changedfiles/push.yml @@ -0,0 +1,12 @@ +include: + - local: '/templates/changedfiles/_shared.yml' + +changed_files_push: + extends: .changed_files + variables: + CHANGED_FILES_COMPARE_REF: $CI_COMMIT_BEFORE_SHA + rules: + - if: $CI_AWE_SKIP_CHANGED_FILES || $CI_PIPELINE_SOURCE != 'push' + when: never + - if: $CI_AWE_RUN_CHANGED_FILES + - !reference [.primary_ref_jobs, rules] diff --git a/templates/conda/_shared.yml b/templates/conda/_shared.yml new file mode 100644 index 0000000000000000000000000000000000000000..84fa19748583323b365c481b2bfb0af347d51d3c --- /dev/null +++ b/templates/conda/_shared.yml @@ -0,0 +1,8 @@ +include: + - local: '/templates/shared/all.yml' + +variables: + CONDA_RECIPE_DIR: conda-recipe + CONDA_OUTPUT_FOLDER: .conda-bld + CONDARC: "${CI_PROJECT_DIR}/.condarc" + CONDA_BUILD_COMMAND: build diff --git a/templates/conda/all.yml b/templates/conda/all.yml new file mode 100644 index 0000000000000000000000000000000000000000..6e23cb701a03701994325cb11484445168ca960e --- /dev/null +++ b/templates/conda/all.yml @@ -0,0 +1,3 @@ +include: + - local: '/templates/conda/build.yml' + - local: '/templates/conda/release.yml' diff --git a/conda-build.yml b/templates/conda/build.yml similarity index 91% rename from conda-build.yml rename to templates/conda/build.yml index c217670df4fbc6a64b3b29bff2b1a08ffa94d2bf..e096c4cd3852c5f5e7e834c31a62846c5ae39957 100644 --- a/conda-build.yml +++ b/templates/conda/build.yml @@ -1,23 +1,6 @@ -# -# Shared configuration. -# -# Note: this cannot be placed in a seperate file because then it will be -# included twice by GitLab in the main conda.yml file. And including -# a .yml file twice yields an error (at least with GitLab 11.9). -# - -stages: - - lint - - build - - test - - quality - - release - -variables: - CONDA_RECIPE_DIR: conda-recipe - CONDA_OUTPUT_FOLDER: .conda-bld - CONDARC: "${CI_PROJECT_DIR}/.condarc" - CONDA_BUILD_COMMAND: build +include: + - local: '/templates/shared/rules.yml' + - local: '/templates/conda/_shared.yml' # # Abstract jobs @@ -26,13 +9,11 @@ variables: .abstract_conda_build_test: image: omegacen/conda-builder rules: - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_BRANCH == 'master' - - if: $CI_COMMIT_BRANCH == 'develop' - - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/ - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/ - - if: $CI_PIPELINE_SOURCE == 'schedule' && $CONDA_BUILD_RUN_JOB + - if: $CI_AWE_SKIP_CONDA_BUILD_TEST + when: never + - if: $CI_AWE_RUN_CONDA_BUILD_TEST + - !reference [.primary_ref_jobs, rules] + - !reference [.merge_request_jobs, rules] before_script: - if [ -z "${CONDA_DOWNLOAD_KEY}" ]; then echo "CI variable 'CONDA_DOWNLOAD_KEY' is not set, exiting."; false; fi - conda config --prepend channels https://${CONDA_DOWNLOAD_KEY}@conda.astro-wise.org @@ -187,6 +168,7 @@ conda_test: - conda_build artifacts: expire_in: 1 day + when: always paths: - coverage.xml - report.xml diff --git a/templates/conda/release.yml b/templates/conda/release.yml new file mode 100644 index 0000000000000000000000000000000000000000..4758436712ca462b05f8513a76b2b337baf696be --- /dev/null +++ b/templates/conda/release.yml @@ -0,0 +1,26 @@ +include: + - local: '/templates/conda/_shared.yml' + +# +# Release +# + +# Upload packages. Only for main branches. +conda_upload: + stage: release + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} + dependencies: + - conda_build + rules: + - if: $CI_AWE_SKIP_CONDA_UPLOAD + when: never + - if: $CI_AWE_RUN_CONDA_UPLOAD + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + - !reference [.primary_ref_jobs, rules] + before_script: + - if [ -z "${CONDA_UPLOAD_KEY}" ]; then echo "CI variable 'CONDA_UPLOAD_KEY' is not set, exiting."; false; fi + script: + - ssh-addkey "${CONDA_UPLOAD_KEY}" + - rsync -chavz --include="*/" --include="*.tar.bz2" --exclude="*" + ${CONDA_OUTPUT_FOLDER}/ conda@129.125.6.100:~/public_html diff --git a/latex.yml b/templates/latex.yml similarity index 84% rename from latex.yml rename to templates/latex.yml index ccdabd4cb3583fb24d2edca5d863c9aa46d3e42d..d661038b6dc53b79d31d25ca9ec6585c0ee7a7ab 100644 --- a/latex.yml +++ b/templates/latex.yml @@ -1,6 +1,5 @@ -stages: - - build - - test +include: + - local: '/templates/shared/all.yml' .abstract_pdf_job: # Image from https://github.com/omegacen/docker-latex-builder @@ -36,13 +35,11 @@ latex_pdf: artifacts: untracked: true rules: - # Ensure that all jobs are detached so they show up on te MR page. - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_BRANCH == 'master' - - if: $CI_COMMIT_BRANCH == 'develop' - - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/ - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/ + - if: $CI_AWE_SKIP_LATEX_PDF + when: never + - if: $CI_AWE_RUN_LATEX_PDF + - !reference [.primary_ref_jobs, rules] + - !reference [.merge_request_jobs, rules] latex_pdf_diff: # Compile a pdf with the differences highlighted. @@ -50,7 +47,10 @@ latex_pdf_diff: extends: .abstract_pdf_job stage: test rules: - - if: $CI_MERGE_REQUEST_ID + - if: $CI_AWE_SKIP_LATEX_PDF_DIFF + when: never + - if: $CI_AWE_RUN_LATEX_PDF_DIFF + - !reference [.merge_request_jobs, rules] variables: # Setting GIT_DEPTH to 0 prevents a shallow clone and thus ensures that # the CI_DEFAULT_BRANCH is available. diff --git a/monthlymerge.yml b/templates/monthlymerge.yml similarity index 59% rename from monthlymerge.yml rename to templates/monthlymerge.yml index e9a26466afdc555eb2957638e42c054c79f0cacc..07ea3011713c6f22b8eefc143db4e657389af19e 100644 --- a/monthlymerge.yml +++ b/templates/monthlymerge.yml @@ -1,15 +1,12 @@ -stages: - - lint - - build - - test - - quality - - release +include: + - local: '/templates/shared/stages.yml' + - local: '/templates/release/calver.yml' release_merge_and_tag:on-schedule: stage: release - image: omegacen/ci-tools + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} rules: - - if: $CI_PIPELINE_SOURCE == 'schedule' && $MM_RUN_JOB + - if: $CI_AWE_RUN_MONTHLY_MERGE && $CI_PIPELINE_SOURCE == 'schedule' variables: MM_SOURCE: develop MM_TARGET: master @@ -53,30 +50,16 @@ release_merge_and_tag:on-schedule: fi done - git commit -m "Bump version to ${version}" ${MM_BUMP_FILES} - - git push origin + # Push the branch but don't run a pipeline. The pipeline will be run when + # pushing the tag at the next step, and we don't want duplicate pipelines + # (with duplicate releases). + # We could also give the option `-o ci.variable="CI_AWE_SKIP_ALL=1"` but + # it's not guaranteed all jobs respect that, so use `-o ci.skip` instead. + # This will list the pipeline as skipped, as opposed to being unlisted when + # using `CI_AWE_SKIP_ALL`. + # Note that pushing the branch and tag simultaneously with `git push --atomic ...` + # will cause GitLab to run pipelines on both the branch and the tag. + - git push origin -o ci.skip # 5. Tag this version. - git tag -a "${version}" -m "Automated tag of ${version}." - git push origin ${version} - -release_gitlab: - stage: release - image: omegacen/ci-tools - variables: - GIT_DEPTH: 0 - rules: - - if: $CI_COMMIT_TAG =~ /^\d{4}\.\d{2}\.\d+/ - script: - # Determine the previous tag. This is a bit brittle because we're sorting lexicographically by tag name - # instead of topographically by commits in the git tree. But since we're restricting to YYYY.MM.PATCH - # tags, this should be ok. - - PREVIOUS_TAG=$(git tag --merged ${CI_COMMIT_SHA} | grep -P '^\d{4}.\d{2}.\d+$' | sort | tail -n 2 | head -n 1) - - | - if [ -z "$PREVIOUS_TAG" ] || [ "$PREVIOUS_TAG" = "$CI_COMMIT_TAG" ]; then - echo "Could not find previous tag, exiting" - false - fi - - echo "Found previous tag ${PREVIOUS_TAG}" - - PREVIOUS_TAG_SHA=$(git rev-parse "${PREVIOUS_TAG}") - - CHANGES=$(git log-mr --output markdown --pretty oneline --target "$CI_DEFAULT_BRANCH" "$PREVIOUS_TAG_SHA..$CI_COMMIT_SHA" | sort --ignore-case) - - DESCRIPTION='##### Changes'$'\n'$'\n'"$CHANGES" - - release-cli create --name "$CI_COMMIT_TAG" --description "$DESCRIPTION" --tag-name "$CI_COMMIT_TAG" diff --git a/templates/release/_shared.yml b/templates/release/_shared.yml new file mode 100644 index 0000000000000000000000000000000000000000..95eb9630f446f2129fad7ea87bfdda61f1715f69 --- /dev/null +++ b/templates/release/_shared.yml @@ -0,0 +1,20 @@ +include: + - local: '/templates/shared/all.yml' + +.release_gitlab: + stage: release + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} + variables: + GIT_DEPTH: 0 + script: + - PREVIOUS_TAG=$(git topotag ${CI_COMMIT_SHA} | grep -P "${TAG_PATTERN}" | head -n 2 | tail -n 1) + - | + if [ -z "$PREVIOUS_TAG" ] || [ "$PREVIOUS_TAG" = "$CI_COMMIT_TAG" ]; then + echo "Could not find previous tag, exiting" + false + fi + - echo "Found previous tag ${PREVIOUS_TAG}" + - PREVIOUS_TAG_SHA=$(git rev-parse "${PREVIOUS_TAG}") + - CHANGES=$(git log-mr --output markdown --pretty oneline --target "$CI_DEFAULT_BRANCH" "$PREVIOUS_TAG_SHA..$CI_COMMIT_SHA" | sort --ignore-case) + - DESCRIPTION='##### Changes'$'\n'$'\n'"$CHANGES" + - release-cli create --name "$CI_COMMIT_TAG" --description "$DESCRIPTION" --tag-name "$CI_COMMIT_TAG" diff --git a/templates/release/calver.yml b/templates/release/calver.yml new file mode 100644 index 0000000000000000000000000000000000000000..1122851561895e979a073fa3bcacb7e6b6206419 --- /dev/null +++ b/templates/release/calver.yml @@ -0,0 +1,9 @@ +include: + - local: '/templates/release/_shared.yml' + +.release_gitlab_calver: + extends: .release_gitlab + variables: + TAG_PATTERN: '^\d{4}\.\d{2}\.\d+$' + rules: + - if: $CI_COMMIT_TAG =~ /^\d{4}\.\d{2}\.\d+$/ diff --git a/templates/release/semver.yml b/templates/release/semver.yml new file mode 100644 index 0000000000000000000000000000000000000000..83545ab45765149dd209d26de35b62399b868c36 --- /dev/null +++ b/templates/release/semver.yml @@ -0,0 +1,9 @@ +include: + - local: '/templates/release/_shared.yml' + +.release_gitlab_semver: + extends: .release_gitlab + variables: + TAG_PATTERN: '^\d+\.\d+\.\d+$' + rules: + - if: $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/ diff --git a/templates/shared/all.yml b/templates/shared/all.yml new file mode 100644 index 0000000000000000000000000000000000000000..f1e6373f57738861447304b058e8c3e938cc1290 --- /dev/null +++ b/templates/shared/all.yml @@ -0,0 +1,4 @@ +include: + - local: '/templates/shared/rules.yml' + - local: '/templates/shared/stages.yml' + - local: '/templates/shared/variables.yml' diff --git a/templates/shared/rules.yml b/templates/shared/rules.yml new file mode 100644 index 0000000000000000000000000000000000000000..86926c89b815222a10dbe20acc39844bbd66f53d --- /dev/null +++ b/templates/shared/rules.yml @@ -0,0 +1,34 @@ +.primary_ref_jobs: + rules: + - if: $CI_AWE_SKIP_ALL + when: never + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_REF_PROTECTED == 'true' + - if: $CI_COMMIT_BRANCH == 'master' + - if: $CI_COMMIT_BRANCH == 'main' + - if: $CI_COMMIT_BRANCH == 'develop' + - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/ + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_AWE_RUN_ALL + +.merge_request_jobs: + rules: + - if: $CI_AWE_SKIP_ALL + when: never + - if: $CI_MERGE_REQUEST_ID && + $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/ && + ( + $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'master' || + $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main' || + $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop' || + $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^release\/.*$/ || + $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH + ) && + $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != 'master' && + $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != 'main' && + $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != 'develop' && + $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^release\/.*$/ && + $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != $CI_DEFAULT_BRANCH + - if: $CI_AWE_RUN_ALL diff --git a/templates/shared/stages.yml b/templates/shared/stages.yml new file mode 100644 index 0000000000000000000000000000000000000000..ca7d04544027af100147cc200e2da9472f3b1892 --- /dev/null +++ b/templates/shared/stages.yml @@ -0,0 +1,8 @@ +stages: + - pre + - lint + - build + - test + - test_post + - quality + - release diff --git a/templates/shared/variables.yml b/templates/shared/variables.yml new file mode 100644 index 0000000000000000000000000000000000000000..11a526c5680a3e507538ad3035362ad2b6acbdd6 --- /dev/null +++ b/templates/shared/variables.yml @@ -0,0 +1,9 @@ +variables: + # Explicitly reference the release branch and the registry of this project. + # Because when including the templates from another GitLab project, variables + # such as $CI_REGISTRY_IMAGE and $CI_COMMIT_BRANCH refer to the including + # project, not this project (omegacen/ci-templates). + # We reference these things, so we can change them in one place in the future, + # avoiding shotgun surgery. + CI_AWE_IMAGE_TAG: v5 + CI_AWE_IMAGE_BASE: ${CI_REGISTRY}/omegacen/ci-templates diff --git a/sonarqube.yml b/templates/sonarqube.yml similarity index 91% rename from sonarqube.yml rename to templates/sonarqube.yml index 90fd36dd4ea833d5d82c86f8363a00bb4d1aa9b5..28b8f49cbcfb9dc82c02cb3950b969531c0e9dd8 100644 --- a/sonarqube.yml +++ b/templates/sonarqube.yml @@ -1,10 +1,5 @@ -stages: - - lint - - build - - test - - quality - - release - +include: + - local: '/templates/shared/stages.yml' .abstract_sonar: image: @@ -81,11 +76,16 @@ stages: } -sonar_default_branch: +sonar_branch: extends: .abstract_sonar allow_failure: true rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_AWE_SKIP_SONAR_BRANCH + when: never + - if: $CI_AWE_RUN_SONAR_BRANCH + - if: $CI_COMMIT_TAG + when: never + - !reference [.primary_ref_jobs, rules] script: # pylint strategy: we'll lint all files. - | @@ -101,7 +101,10 @@ sonar_default_branch: sonar_mr: extends: .abstract_sonar rules: - - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH + - if: $CI_AWE_SKIP_SONAR_MR + when: never + - if: $CI_AWE_RUN_SONAR_MR + - !reference [.merge_request_jobs, rules] script: # pylint strategy: we'll only lint changed files. - | diff --git a/templates/testreport/badge.yml b/templates/testreport/badge.yml new file mode 100644 index 0000000000000000000000000000000000000000..784a8b22b8b22003acb0882b238a22b557d3bb22 --- /dev/null +++ b/templates/testreport/badge.yml @@ -0,0 +1,19 @@ +include: + - local: '/templates/shared/all.yml' + +test_report_badge: + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} + stage: test_post + variables: + TEST_REPORT_ARTIFACT_FILE: report.xml + rules: + - if: $CI_AWE_SKIP_TEST_REPORT_BADGE + when: never + - if: $CI_AWE_RUN_TEST_REPORT_BADGE + - !reference [.primary_ref_jobs, rules] + script: + - report_badge "$TEST_REPORT_ARTIFACT_FILE" report.svg + artifacts: + paths: + - report.svg + expire_in: 30 days diff --git a/templates/testreport/diff.yml b/templates/testreport/diff.yml new file mode 100644 index 0000000000000000000000000000000000000000..b23b71cec6bd202d33a80f751270e268519e1290 --- /dev/null +++ b/templates/testreport/diff.yml @@ -0,0 +1,59 @@ +.test_report_diff: + image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG} + variables: + TEST_REPORT_JOB: conda_test + TEST_REPORT_ARTIFACT_FILE: report.xml + stage: test_post + rules: + - if: $CI_AWE_SKIP_TEST_REPORT_DIFF + when: never + - if: $CI_AWE_RUN_TEST_REPORT_DIFF + - !reference [.primary_ref_jobs, rules] + script: + # To download the test report of the previous pipeline, we cannot use the API + # as described at https://docs.gitlab.com/ee/api/job_artifacts.html#download-a-single-artifact-file-from-specific-tag-or-branch, + # because that only exposes the latest _successful_ pipeline. We want the latest test report + # of either the last succeeded pipeline or the last failed pipeline. + - python-gitlab-set-private-token ${TEST_REPORT_JOB_TOKEN} + # So, first determine the previous pipeline. + - LAST_PIPELINE_SUCCESS=$(gitlab --output json project-pipeline list --project-id $CI_PROJECT_ID --ref $TEST_REPORT_DIFF_REF --scope finished --status success | jq 'first.id') + - LAST_PIPELINE_FAILED=$(gitlab --output json project-pipeline list --project-id $CI_PROJECT_ID --ref $TEST_REPORT_DIFF_REF --scope finished --status failed | jq 'first.id') + - 'LAST_PIPELINE=$(( LAST_PIPELINE_SUCCESS > LAST_PIPELINE_FAILED ? LAST_PIPELINE_SUCCESS : LAST_PIPELINE_FAILED ))' + - echo "Previous pipeline on ${TEST_REPORT_DIFF_REF} was ${LAST_PIPELINE}." + # Determine the previous job. + - LAST_JOB=$(gitlab --output json project-pipeline-job list --project-id $CI_PROJECT_ID --pipeline-id $LAST_PIPELINE | jq ".[] | select(.name==\"${TEST_REPORT_JOB}\") | .id") + - echo "Previous ${TEST_REPORT_JOB} job was ${LAST_JOB}." + # Download the report of the previous pipeline. + # Can't use python-gitlab for that yet, see https://github.com/python-gitlab/python-gitlab/issues/1926. + - TEST_REPORT_ARTIFACT_URL="$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$LAST_JOB/artifacts/$TEST_REPORT_ARTIFACT_FILE" + - 'curl --output "$TEST_REPORT_ARTIFACT_FILE.old" --location --header "JOB-TOKEN: $CI_JOB_TOKEN" "$TEST_REPORT_ARTIFACT_URL"' + # Compare it to the report of this pipeline. + - report_diff "$TEST_REPORT_ARTIFACT_FILE.old" "$TEST_REPORT_ARTIFACT_FILE" report_diff.xml + - junitparser verify report_diff.xml + artifacts: + expire_in: 1 day + when: always + paths: + - report_diff.xml + reports: + junit: report_diff.xml + +test_report_diff_mr: + extends: .test_report_diff + variables: + TEST_REPORT_DIFF_REF: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + rules: + - if: $CI_AWE_SKIP_TEST_REPORT_DIFF + when: never + - if: $CI_AWE_RUN_TEST_REPORT_DIFF + - !reference [.merge_request_jobs, rules] + +test_report_diff_branch: + extends: .test_report_diff + variables: + TEST_REPORT_DIFF_REF: $CI_COMMIT_REF_NAME + rules: + - if: $CI_AWE_SKIP_TEST_REPORT_DIFF + when: never + - if: $CI_AWE_RUN_TEST_REPORT_DIFF + - !reference [.primary_ref_jobs, rules]