From 97f4a177fe2e49fd89cf4f8f1ea26c0e42704bca Mon Sep 17 00:00:00 2001
From: Teake Nutma <t.a.nutma@rug.nl>
Date: Wed, 15 Jun 2022 10:44:38 +0200
Subject: [PATCH] Rename report compare job, add badge documentation
---
.gitlab-ci.yml | 44 ++++++++--------
README.rst | 61 ++++++++++++++++------
dockerfiles/ci-tools/Dockerfile | 2 +-
dockerfiles/ci-tools/report_compare.py | 50 ------------------
dockerfiles/ci-tools/report_diff.py | 51 ++++++++++++++++++
templates/report/changes.yml | 29 ----------
templates/{report => testreport}/badge.yml | 10 ++--
templates/testreport/diff.yml | 25 +++++++++
8 files changed, 149 insertions(+), 123 deletions(-)
delete mode 100755 dockerfiles/ci-tools/report_compare.py
create mode 100755 dockerfiles/ci-tools/report_diff.py
delete mode 100644 templates/report/changes.yml
rename templates/{report => testreport}/badge.yml (58%)
create mode 100644 templates/testreport/diff.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a60f364..35fd1ef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,25 +14,25 @@ include:
- local: '/templates/shared/all.yml'
- local: '/templates/changedfiles/all.yml'
-#build_containers:
-# stage: build
-# # Note: the docker-builder image has to be build and pushed manually once to bootstrap this job.
-# image: ${CI_REGISTRY_IMAGE}/docker-builder:master
-# rules:
-# - !reference [.primary_ref_jobs, rules]
-# - !reference [.merge_request_jobs, rules]
-# script:
-# # Find the directories that contain a file named 'Dockerfile', and (re)build the ones
-# # that contain changed files.
-# - |
-# find . -name Dockerfile -type f | while read FILE;
-# do
-# # Strip leading './' and trailing '/<filename>'
-# DOCKER_DIR=$(echo "${FILE}" | sed -r 's|^\./||' | xargs dirname)
-# if grep -q "^${DOCKER_DIR}" changed_files.log;
-# then
-# echo "Detected a changed file inside ./${DOCKER_DIR}/. (Re)build the container."
-# IMAGE_NAME=$(echo "${DOCKER_DIR}" | sed -r 's|^dockerfiles/||')
-# buildimage "${DOCKER_DIR}" "${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}"
-# fi
-# done
+build_containers:
+ stage: build
+ # Note: the docker-builder image has to be build and pushed manually once to bootstrap this job.
+ image: ${CI_REGISTRY_IMAGE}/docker-builder:master
+ rules:
+ - !reference [.primary_ref_jobs, rules]
+ - !reference [.merge_request_jobs, rules]
+ script:
+ # Find the directories that contain a file named 'Dockerfile', and (re)build the ones
+ # that contain changed files.
+ - |
+ find . -name Dockerfile -type f | while read FILE;
+ do
+ # Strip leading './' and trailing '/<filename>'
+ DOCKER_DIR=$(echo "${FILE}" | sed -r 's|^\./||' | xargs dirname)
+ if grep -q "^${DOCKER_DIR}" changed_files.log;
+ then
+ echo "Detected a changed file inside ./${DOCKER_DIR}/. (Re)build the container."
+ IMAGE_NAME=$(echo "${DOCKER_DIR}" | sed -r 's|^dockerfiles/||')
+ buildimage "${DOCKER_DIR}" "${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}"
+ fi
+ done
diff --git a/README.rst b/README.rst
index b523dd8..0f0feec 100644
--- a/README.rst
+++ b/README.rst
@@ -281,6 +281,35 @@ FILENAME_TEX No No `automatically determined` Manual specification
============ ======== ========= ========================== ======================================
+Test report badge
+-----------------
+
+This template generates a badge that shows the percentage of successful tests in the
+test suite. It requires another job earlier in the pipeline that generates a JUnit test
+report. The ``conda_test`` job does generates such a report, if the conda recipe
+includes tests.
+
+To use this template, include the following snippet in your ``.gitlab-ci.yml`` file:
+
+.. code-block:: yaml
+
+ include:
+ - project: 'omegacen/ci-templates'
+ ref: v5
+ file: '/templates/report/badge.yml'
+
+Next, add badge to the `Project badges`_. In your GitLab project, go to
+*Settings* -> *General* -> *Badges*. Then add the following:
+
+ * Name: ``Test Success Rate``
+ * Link: ``https://gitlab.astro-wise.org/%{project_path}/-/pipelines/%{default_branch}/latest``
+ * Badge image URL: ``https://gitlab.astro-wise.org/%{project_path}/-/jobs/artifacts/%{default_branch}/raw/report.svg?job=test_report_badge``
+
+Test report change detection
+----------------------------
+
+TBW.
+
Controlling when jobs runs
==========================
@@ -293,22 +322,22 @@ However, if you want more control over when a particular job runs, you can use t
``CI_AWE_RUN_<JOBNAME>`` and ``CI_AWE_SKIP_<JOBNAME>`` variables to run or to skip
a job, respectively. The full list of variables is as follows:
-==================== ============================ =========================== =========================================
- Job name Skip variable Run variable Template
-==================== ============================ =========================== =========================================
-`autopep8` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/autopep8.yml`
-`black` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/black.yml`
-`changed_files_mr` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/mergerequest.yml`
-`changed_files_push` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/push.yml`
-`conda_build` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
-`conda_test` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
-`conda_upload` CI_AWE_SKIP_CONDA_UPLOAD CI_AWE_RUN_CONDA_UPLOAD `templates/conda/release.yml`
-`latex_pdf` CI_AWE_SKIP_LATEX_PDF CI_AWE_RUN_LATEX_PDF `templates/latex.yml`
-`latex_pdf_diff` CI_AWE_SKIP_LATEX_PDF_DIFF CI_AWE_RUN_LATEX_PDF_DIFF `templates/latex.yml`
-`sonar_branch` CI_AWE_SKIP_SONAR_BRANCH CI_AWE_RUN_SONAR_BRANCH `templates/sonarqube.yml`
-`sonar_mr` CI_AWE_SKIP_SONAR_MR CI_AWE_RUN_SONAR_MR `templates/sonarqube.yml`
-`report_badge` CI_AWE_SKIP_REPORT_BADGE CI_AWE_RUN_REPORT_BADGE `templates/report/badge.yml`
-`report_changes` CI_AWE_SKIP_REPORT_CHANGES CI_AWE_RUN_REPORT_CHANGES `templates/report/changes.yml`
+==================== ============================= ============================ =========================================
+ Job name Skip variable Run variable Template
+==================== ============================= ============================ =========================================
+`autopep8` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/autopep8.yml`
+`black` CI_AWE_SKIP_AUTOFORMAT CI_AWE_RUN_AUTOFORMAT `templates/autoformat/black.yml`
+`changed_files_mr` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/mergerequest.yml`
+`changed_files_push` CI_AWE_SKIP_CHANGED_FILES CI_AWE_RUN_CHANGED_FILES `templates/changedfiles/push.yml`
+`conda_build` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
+`conda_test` CI_AWE_SKIP_CONDA_BUILD_TEST CI_AWE_RUN_CONDA_BUILD_TEST `templates/conda/build.yml`
+`conda_upload` CI_AWE_SKIP_CONDA_UPLOAD CI_AWE_RUN_CONDA_UPLOAD `templates/conda/release.yml`
+`latex_pdf` CI_AWE_SKIP_LATEX_PDF CI_AWE_RUN_LATEX_PDF `templates/latex.yml`
+`latex_pdf_diff` CI_AWE_SKIP_LATEX_PDF_DIFF CI_AWE_RUN_LATEX_PDF_DIFF `templates/latex.yml`
+`sonar_branch` CI_AWE_SKIP_SONAR_BRANCH CI_AWE_RUN_SONAR_BRANCH `templates/sonarqube.yml`
+`sonar_mr` CI_AWE_SKIP_SONAR_MR CI_AWE_RUN_SONAR_MR `templates/sonarqube.yml`
+`test_report_badge` CI_AWE_SKIP_TEST_REPORT_BADGE CI_AWE_RUN_TEST_REPORT_BADGE `templates/report/badge.yml`
+`test_report_diff` CI_AWE_SKIP_TESt_REPORT_DIFF CI_AWE_RUN_TEST_REPORT_DIFF `templates/report/diff.yml`
In addition, you can use the ``CI_AWE_SKIP_ALL` and ``CI_AWE_RUN_ALL` variables to
control whether any or all of these jobs run.
diff --git a/dockerfiles/ci-tools/Dockerfile b/dockerfiles/ci-tools/Dockerfile
index b943026..d1b2b49 100644
--- a/dockerfiles/ci-tools/Dockerfile
+++ b/dockerfiles/ci-tools/Dockerfile
@@ -36,7 +36,7 @@ RUN echo "Host *\n\tStrictHostKeyChecking no" > ~/.ssh/config
COPY ssh-addkey.sh /usr/local/bin/ssh-addkey
COPY python-gitlab-set-private-token.sh /usr/local/bin/python-gitlab-set-private-token
COPY report_badge.py /usr/local/bin/report_badge
-COPY report_compare.py /usr/local/bin/report_compare
+COPY report_diff.py /usr/local/bin/report_diff
COPY entrypoint.sh /usr/local/bin/entrypoint
ENTRYPOINT [ "/bin/bash", "/usr/local/bin/entrypoint" ]
diff --git a/dockerfiles/ci-tools/report_compare.py b/dockerfiles/ci-tools/report_compare.py
deleted file mode 100755
index fea475b..0000000
--- a/dockerfiles/ci-tools/report_compare.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-
-from junitparser import JUnitXml, TestSuite
-
-
-def compare_reports(path_before, path_after, path_out):
- before = JUnitXml.fromfile(path_before)
- after = JUnitXml.fromfile(path_after)
-
- passed = []
- for suite in before:
- for case in suite:
- if case.is_passed:
- passed.append(case)
-
- failed = []
- for suite in after:
- for case in suite:
- if not case.is_passed and not case.is_skipped:
- failed.append(case)
-
- newly_failed = []
- for f in failed:
- for p in passed:
- if (f.classname, f.name) == (p.classname, p.name):
- newly_failed.append(f)
-
- xml = JUnitXml()
- suite = TestSuite('newly_failed_tests')
- suite.add_testcases(newly_failed)
- xml.add_testsuite(suite)
- xml.update_statistics()
- xml.write(path_out, to_console=False)
-
-
-def main():
- parser = argparse.ArgumentParser(
- description='Create a new report with testcases that passed before but fail or error now.'
- )
- parser.add_argument("before", help="Path of the initial XML report to compare.")
- parser.add_argument("after", help="Path of the re-run XML report to compare.")
- parser.add_argument("output", help='Path to write compare report to.')
- args = parser.parse_args()
- compare_reports(args.before, args.after, args.output)
-
-
-if __name__ == '__main__':
- main()
diff --git a/dockerfiles/ci-tools/report_diff.py b/dockerfiles/ci-tools/report_diff.py
new file mode 100755
index 0000000..675bb67
--- /dev/null
+++ b/dockerfiles/ci-tools/report_diff.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import argparse
+
+from junitparser import JUnitXml, TestSuite
+
+
+def case_dictionary(report):
+ result = {}
+ for suite in report:
+ for case in suite:
+ key = (case.classname, case.name)
+ result[key] = case
+ return result
+
+
+def compare_reports(path_before, path_after, path_out):
+ before = JUnitXml.fromfile(path_before)
+ after = JUnitXml.fromfile(path_after)
+ before_cases = case_dictionary(before)
+ after_cases = case_dictionary(after)
+
+ changed_cases = TestSuite('tests_with_changed_result')
+
+ for key, after_case in after_cases.items():
+ if key in before_cases:
+ before_case = before_cases[key]
+ before_results = [type(r) for r in before_case.result]
+ after_results = [type(r) for r in after_case.result]
+ if set(before_results) != set(after_results):
+ changed_cases.add_testcase(after_case)
+
+ xml = JUnitXml()
+ xml.add_testsuite(changed_cases)
+ xml.update_statistics()
+ xml.write(path_out, to_console=False)
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Create a new report with testcases present before and after but with different status.'
+ )
+ parser.add_argument("before", help="Path of the initial XML report to compare.")
+ parser.add_argument("after", help="Path of the re-run XML report to compare.")
+ parser.add_argument("output", help='Path to write compare report to.')
+ args = parser.parse_args()
+ compare_reports(args.before, args.after, args.output)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/templates/report/changes.yml b/templates/report/changes.yml
deleted file mode 100644
index 6f79059..0000000
--- a/templates/report/changes.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-report_changes:
- image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG}
- variables:
- REPORT_CHANGES_JOB: conda_test
- REPORT_CHANGES_ARTIFACT_FILE: report.xml
- REPORT_CHANGES_ARTIFACT_URL: "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_COMMIT_REF_NAME/raw/$REPORT_CHANGES_ARTIFACT_FILE?job=$REPORT_CHANGES_JOB"
- stage: test_post
- rules:
- - if: $CI_AWE_SKIP_REPORT_CHANGES
- when: never
- - if: $CI_AWE_RUN_REPORT_CHANGES
- - !reference [.primary_ref_jobs, rules]
- script:
- - |
- if ! junitparser verify "$REPORT_CHANGES_ARTIFACT_FILE"; then
- echo "Test report of this pipeline contains failures. Comparing it to the previous run."
- # Download the report of the previous pipeline.
- curl --output "$REPORT_CHANGES_FILE.old" --location --header "JOB-TOKEN: $CI_JOB_TOKEN" "$REPORT_CHANGES_ARTIFACT_URL"
- # Compare it to the report of this pipeline.
- report_compare "$REPORT_CHANGES_FILE.old" "$REPORT_CHANGES_ARTIFACT_FILE" report_changes.xml
- junitparser verify report_changes.xml
- fi
- artifacts:
- expire_in: 1 day
- when: always
- paths:
- - report_changes.xml
- reports:
- junit: report_changes.xml
diff --git a/templates/report/badge.yml b/templates/testreport/badge.yml
similarity index 58%
rename from templates/report/badge.yml
rename to templates/testreport/badge.yml
index b7c3c84..27c56ee 100644
--- a/templates/report/badge.yml
+++ b/templates/testreport/badge.yml
@@ -1,18 +1,18 @@
include:
- local: '/templates/shared/all.yml'
-report_badge:
+test_report_badge:
image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG}
stage: test_post
variables:
- REPORT_BADGE_ARTIFACT_FILE: report.xml
+ TEST_REPORT_ARTIFACT_FILE: report.xml
rules:
- - if: $CI_AWE_SKIP_REPORT_BADGE
+ - if: $CI_AWE_SKIP_TEST_REPORT_BADGE
when: never
- - if: $CI_AWE_RUN_REPORT_BADGE
+ - if: $CI_AWE_RUN_TEST_REPORT_BADGE
- !reference [.primary_ref_jobs, rules]
script:
- - report_badge "$REPORT_BADGE_ARTIFACT_FILE" report.svg
+ - report_badge "$TEST_REPORT_BADGE_ARTIFACT_FILE" report.svg
artifacts:
paths:
- report.svg
diff --git a/templates/testreport/diff.yml b/templates/testreport/diff.yml
new file mode 100644
index 0000000..25961f1
--- /dev/null
+++ b/templates/testreport/diff.yml
@@ -0,0 +1,25 @@
+test_report_diff:
+ image: ${CI_AWE_IMAGE_BASE}/ci-tools:${CI_AWE_IMAGE_TAG}
+ variables:
+ TEST_REPORT_JOB: conda_test
+ TEST_REPORT_ARTIFACT_FILE: report.xml
+ TEST_REPORT_ARTIFACT_URL: "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_COMMIT_REF_NAME/raw/$TEST_REPORT_ARTIFACT_FILE?job=$TEST_REPORT_JOB"
+ stage: test_post
+ rules:
+ - if: $CI_AWE_SKIP_TEST_REPORT_DIFF
+ when: never
+ - if: $CI_AWE_RUN_TEST_REPORT_DIFF
+ - !reference [.primary_ref_jobs, rules]
+ script:
+ # Download the report of the previous pipeline.
+ - 'curl --output "$TEST_REPORT_FILE.old" --location --header "JOB-TOKEN: $CI_JOB_TOKEN" "$TEST_REPORT_ARTIFACT_URL"'
+ # Compare it to the report of this pipeline.
+ - report_diff "$TEST_REPORT_FILE.old" "$TEST_REPORT_ARTIFACT_FILE" report_diff.xml
+ - junitparser verify report_diff.xml
+ artifacts:
+ expire_in: 1 day
+ when: always
+ paths:
+ - report_diff.xml
+ reports:
+ junit: report_diff.xml
--
GitLab