From 718ff24a9fffedb4559634fdb47598f51cc0b47d Mon Sep 17 00:00:00 2001
From: Teake Nutma <t.a.nutma@rug.nl>
Date: Thu, 16 Jun 2022 14:56:56 +0200
Subject: [PATCH] Add Docker READMEs and fix a bug
---
README.rst | 35 +++++-----
dockerfiles/ci-tools/README.rst | 85 ++++++++++++++++++++++++
dockerfiles/ci-tools/report_diff.py | 7 +-
dockerfiles/docker-builder/README.rst | 17 +++++
dockerfiles/docker-builder/buildimage.sh | 2 +-
templates/testreport/badge.yml | 2 +-
6 files changed, 127 insertions(+), 21 deletions(-)
create mode 100644 dockerfiles/ci-tools/README.rst
create mode 100644 dockerfiles/docker-builder/README.rst
diff --git a/README.rst b/README.rst
index 1474f73..7b06f6f 100644
--- a/README.rst
+++ b/README.rst
@@ -407,23 +407,24 @@ However, if you want more control over when a particular job runs, you can use t
``CI_AWE_RUN_<JOBNAME>`` and ``CI_AWE_SKIP_<JOBNAME>`` variables to run or to skip
a job, respectively. The full list of variables is as follows:
-==================== ============================= ============================ =========================================
- Job name Skip variable Run variable Template
-==================== ============================= ============================ =========================================
-`autopep8` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/autopep8.yml`
-`black` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/black.yml`
-`changed_files_mr` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/mergerequest.yml`
-`changed_files_push` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/push.yml`
-`conda_build` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
-`conda_test` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
-`conda_upload` CI_AWE_SKIP_CONDA_UPLOAD CI_AWE_RUN_CONDA_UPLOAD `templates/conda/release.yml`
-`latex_pdf` CI_AWE_SKIP_LATEX_PDF CI_AWE_RUN_LATEX_PDF `templates/latex.yml`
-`latex_pdf_diff` CI_AWE_SKIP_LATEX_PDF_DIFF CI_AWE_RUN_LATEX_PDF_DIFF `templates/latex.yml`
-`sonar_branch` CI_AWE_SKIP_SONAR_BRANCH CI_AWE_RUN_SONAR_BRANCH `templates/sonarqube.yml`
-`sonar_mr` CI_AWE_SKIP_SONAR_MR CI_AWE_RUN_SONAR_MR `templates/sonarqube.yml`
-`test_report_badge` CI_AWE_SKIP_TEST_REPORT_BADGE CI_AWE_RUN_TEST_REPORT_BADGE `templates/report/badge.yml`
-`test_report_diff` CI_AWE_SKIP_TEST_REPORT_DIFF CI_AWE_RUN_TEST_REPORT_DIFF `templates/report/diff.yml`
-==================== ============================= ============================ =========================================
+========================= ============================= ============================ =========================================
+ Job name Skip variable Run variable Template
+========================= ============================= ============================ =========================================
+`autopep8` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/autopep8.yml`
+`black` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/black.yml`
+`changed_files_mr` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/mergerequest.yml`
+`changed_files_push` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/push.yml`
+`conda_build` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
+`conda_test` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
+`conda_upload` CI_AWE_SKIP_CONDA_UPLOAD CI_AWE_RUN_CONDA_UPLOAD `templates/conda/release.yml`
+`latex_pdf` CI_AWE_SKIP_LATEX_PDF CI_AWE_RUN_LATEX_PDF `templates/latex.yml`
+`latex_pdf_diff` CI_AWE_SKIP_LATEX_PDF_DIFF CI_AWE_RUN_LATEX_PDF_DIFF `templates/latex.yml`
+`sonar_branch` CI_AWE_SKIP_SONAR_BRANCH CI_AWE_RUN_SONAR_BRANCH `templates/sonarqube.yml`
+`sonar_mr` CI_AWE_SKIP_SONAR_MR CI_AWE_RUN_SONAR_MR `templates/sonarqube.yml`
+`test_report_badge` CI_AWE_SKIP_TEST_REPORT_BADGE CI_AWE_RUN_TEST_REPORT_BADGE `templates/testreport/badge.yml`
+`test_report_diff_mr` CI_AWE_SKIP_TEST_REPORT_DIFF CI_AWE_RUN_TEST_REPORT_DIFF `templates/testreport/diff.yml`
+`test_report_diff_branch` CI_AWE_SKIP_TEST_REPORT_DIFF CI_AWE_RUN_TEST_REPORT_DIFF `templates/testreport/diff.yml`
+========================= ============================= ============================ =========================================
In addition, you can use the ``CI_AWE_SKIP_ALL`` and ``CI_AWE_RUN_ALL`` variables to
control whether any or all of these jobs run.
diff --git a/dockerfiles/ci-tools/README.rst b/dockerfiles/ci-tools/README.rst
new file mode 100644
index 0000000..c26e63b
--- /dev/null
+++ b/dockerfiles/ci-tools/README.rst
@@ -0,0 +1,85 @@
+========
+ci-tools
+========
+
+A Docker image containing various tools and utilities that come in handy during
+(GitLab) CI jobs.
+
+List of tools installed:
+
+ * git
+ * openssh-client
+ * rsync
+ * `s-nail`_
+ * `gettext`_
+ * `curl`_
+ * `jq`_
+ * `black`_
+ * `autopep8`_
+ * `release-cli`_
+ * `python-gitlab`_
+ * `junitparser`_
+ * `anybadge`_
+ * `python-compare-ast`_
+ * `coverage-fixpaths`_
+ * `git-scripts`_
+ * ssh-addkey
+ * report-badge
+ * report-diff
+
+ssh-addkey
+==========
+
+Easily add private SSH keys during your GitLab CI jobs.
+
+This utility reduces the hassle of adding a private SSH key to running
+Docker containers to one command. This is particularly useful for deploy
+jobs in GitLab CI, hence the name.
+
+While running a container (interactively), you can add a private key as follows:
+
+.. code-block::
+
+ $ ssh-addkey "${SSH_PRIVATE_KEY}"
+
+where the ``SSH_PRIVATE_KEY`` variable contains your private key. You can then
+either ssh, rsync, or use git to sync to your favorite deploy server.
+
+report-badge
+============
+
+Generates a CI badge from a test report with the percentage of passed tests.
+Usage:
+
+.. code-block::
+
+ $ report-badge <input JUnit report file> <output SVG badge file>
+
+report-diff
+===========
+
+Create a diff of two JUnit test reports. The resulting test report contains only
+tests that are present in both of the input reports. In addition, their status has
+to have changed in order for the tests to be included. The status of the last input
+report is shown in the diff.
+
+Usage:
+
+.. code-block::
+
+ $ report-diff <JUnit report before> <JUnit report after> <JUnit diff report>
+
+
+.. _s-nail: https://wiki.archlinux.org/title/S-nail
+.. _gettext: https://www.gnu.org/software/gettext/
+.. _curl: https://curl.se/
+.. _jq: https://stedolan.github.io/jq/
+.. _black: https://black.readthedocs.io
+.. _autopep8: https://github.com/hhatto/autopep8
+.. _release-cli: https://gitlab.com/gitlab-org/release-cli
+.. _junitparser: https://github.com/weiwei/junitparser
+.. _anybadge: https://github.com/jongracecox/anybadge
+.. _python-gitlab: https://python-gitlab.readthedocs.io
+.. _python-compare-ast: https://github.com/omegacen/python-compare-ast
+.. _coverage-fixpaths: https://github.com/omegacen/coverage-fixpaths
+.. _git-scripts: https://gitlab.astro-wise.org/omegacen/git-scripts
diff --git a/dockerfiles/ci-tools/report_diff.py b/dockerfiles/ci-tools/report_diff.py
index 4723439..d67884f 100755
--- a/dockerfiles/ci-tools/report_diff.py
+++ b/dockerfiles/ci-tools/report_diff.py
@@ -37,11 +37,14 @@ def compare_reports(path_before, path_after, path_out):
def main():
parser = argparse.ArgumentParser(
- description='Create a new report with testcases present before and after but with different status.'
+ description='Create a diff of two JUnit test reports. The resulting test report contains only '
+ 'tests that are present in both of the input reports. In addition, their status has '
+ 'to have changed in order for the tests to be included. '
+ 'The status of the last input report is shown in the diff.'
)
parser.add_argument("before", help="Path of the initial XML report to compare.")
parser.add_argument("after", help="Path of the re-run XML report to compare.")
- parser.add_argument("output", help='Path to write compare report to.')
+ parser.add_argument("output", help='Path to write diff report to.')
args = parser.parse_args()
compare_reports(args.before, args.after, args.output)
diff --git a/dockerfiles/docker-builder/README.rst b/dockerfiles/docker-builder/README.rst
new file mode 100644
index 0000000..83ac72b
--- /dev/null
+++ b/dockerfiles/docker-builder/README.rst
@@ -0,0 +1,17 @@
+==============
+docker-builder
+==============
+
+Convenience wrapper around kaniko for building images in GitLab CI.
+
+Usage in GitLab CI:
+
+.. code-block:: yaml
+
+ build_docker_image:
+ image: ${CI_REGISTRY}/omegacen/ci-templates/docker-builder
+ script:
+ - buildimage <dockerfile> [<image_subname>:]<tag>
+
+This will automatically upload the build image to the container registry of
+the project this snippet is used in.
diff --git a/dockerfiles/docker-builder/buildimage.sh b/dockerfiles/docker-builder/buildimage.sh
index 4a86ea1..db9c8aa 100755
--- a/dockerfiles/docker-builder/buildimage.sh
+++ b/dockerfiles/docker-builder/buildimage.sh
@@ -2,7 +2,7 @@
# Wrapper for building OCI images with Kaniko in GitLab CI. Pushes to
# $CI_REGISTRY_IMAGE/IMAGE_SUBNAME:TAG or to
-# $CI_REGISTRY_IMAGE:TAG, dependinng on the second argument.
+# $CI_REGISTRY_IMAGE:TAG, depending on the second argument.
# Usage:
#
# buildimage.sh DOCKERFILE_OR_CONTEXT [IMAGE_SUBNAME:]TAG [EXTRA_KANIKO_ARGS]
diff --git a/templates/testreport/badge.yml b/templates/testreport/badge.yml
index 27c56ee..784a8b2 100644
--- a/templates/testreport/badge.yml
+++ b/templates/testreport/badge.yml
@@ -12,7 +12,7 @@ test_report_badge:
- if: $CI_AWE_RUN_TEST_REPORT_BADGE
- !reference [.primary_ref_jobs, rules]
script:
- - report_badge "$TEST_REPORT_BADGE_ARTIFACT_FILE" report.svg
+ - report_badge "$TEST_REPORT_ARTIFACT_FILE" report.svg
artifacts:
paths:
- report.svg
--
GitLab