Commit de22542a authored by Hugo Buddelmeijer's avatar Hugo Buddelmeijer
Browse files

Merge branch 'tnutma/feature/black' into 'master'

Add Black CI template

See merge request omegacen/ci-templates!40
parents 9c21c94f 5b500c17
......@@ -10,6 +10,36 @@ Ready to include templates for common CI jobs.
Usage
=====
black.yml
-------------
By including this template, all Python files will be linted with `black`_.
If they are not properly formatted according to black, the CI will
fail and an assist-MR will be created to automatically fix the formatting.
To use it, follow these steps:
#. Add `CI Bot`_ as a member to your project in at least the Developer role.
#. Lastly, include the following snippet in your ``.gitlab-ci.yml`` file:
.. code-block:: yaml
include:
- project: 'omegacen/ci-templates'
ref: v3
file: '/black.yml'
The variables that can be changed are:
=================== ======== ========= ================== =================================================================
Name Required Protected Default value Purpose
=================== ======== ========= ================== =================================================================
BLACK_LINE_LENGTH No No 120 How many characters per line black allows.
BLACK_MR_LABELS No No assist,black Comma-separated list of labels to attach to the black assist MRs.
=================== ======== ========= ================== =================================================================
conda.yml
---------
......@@ -187,6 +217,7 @@ However, including a template from the ``master`` branch might lead to breaking
changes every now and then.
.. _black: https://black.readthedocs.io/
.. _conda-wise documentation: http://omegacen.pages.astro-wise.org/conda/maintainers.html
.. _CI variables: https://docs.gitlab.com/ce/ci/variables/#variables
.. _CalVer: http://calver.org/
......
stages:
- lint
- build
- test
- quality
- release
.abstract_black:
image: omegacen/black
stage: lint
variables:
BLACK_LINE_LENGTH: 120
BLACK_MR_LABELS: assist,black
BLACK_COMMENT_MARKER: black-auto-comment
GIT_DEPTH: 0
black_branch:
extends: .abstract_black
rules:
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH == 'master'
- if: $CI_COMMIT_BRANCH == 'develop'
- if: $CI_COMMIT_BRANCH =~ /^release\/.*$/
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
script:
- black --line-length ${BLACK_LINE_LENGTH} --check .
black_mr:
extends: .abstract_black
rules:
- if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/black\/.*$/
before_script:
- export BLACK_VERSION=$(black --version | cut -d ' ' -f3)
# Save configuration for python-gitlab. Somehow we can't use simple environment variables for this $%@#$@.
- |
cat << EOF > ~/.python-gitlab.cfg
[global]
default = astro-wise
[astro-wise]
url = ${CI_SERVER_URL}
private_token = ${BLACK_TOKEN}
api_version = 4
EOF
script:
- |
# Run black only on changed files.
CHANGED_FILES=$(git diff --diff-filter=d --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME...$CI_COMMIT_SHA -- | grep '\.py$' || true)
echo "Found these changed Python files:"
echo "${CHANGED_FILES}" | sed 's/^/- /g'
echo "Proceeding to lint them with black."
if echo "$CHANGED_FILES" | xargs --no-run-if-empty black --line-length ${BLACK_LINE_LENGTH} --check ; then
echo "All files are correctly formatted."
# Resolve any discussions we started.
DISCUSSION_IDS=$(gitlab --output json project-merge-request-discussion list \
--project-id ${CI_PROJECT_ID} \
--mr-iid ${CI_MERGE_REQUEST_IID} \
--all \
| jq --raw-output ".[] | select(.notes[0].author.username==\"ci-bot\") | select(.notes[0].body | strings | test(\"${BLACK_COMMENT_MARKER}\")) | select(.notes[0].resolved == false) | .id")
if [ -n "${DISCUSSION_IDS}" ]; then
COMMENT_BODY=$(echo -e "All black formatting issues have been fixed.\n\n[//]: # \"${BLACK_COMMENT_MARKER}\"\n")
while read discussion_id; do
gitlab project-merge-request-discussion-note create \
--project-id ${CI_PROJECT_ID} \
--mr-iid ${CI_MERGE_REQUEST_IID} \
--discussion-id ${discussion_id} \
--body "${COMMENT_BODY}" > /dev/null
echo "Resolving discussion ${discussion_id}."
gitlab project-merge-request-discussion update \
--project-id ${CI_PROJECT_ID} \
--mr-iid ${CI_MERGE_REQUEST_IID} \
--id ${discussion_id} \
--resolved true > /dev/null
done <<< "${DISCUSSION_IDS}"
else
echo "No dicussions to resolve."
fi
else
BLACK_BRANCH="assist/black/${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/${CI_COMMIT_SHORT_SHA}"
GIT_REPO="https://ci-bot:${BLACK_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
if git ls-remote --exit-code --heads "${GIT_REPO}" "${BLACK_BRANCH}"; then
echo "Blackened branch ${BLACK_BRANCH} already exists. Exiting."
else
echo "Blackened branch ${BLACK_BRANCH} does not yet exists, continuing."
# 1. Clone and checkout a new branch.
git config --global user.name "CI Bot"
git config --global user.email "astrowisegitlab@rug.nl"
tmpdir=$(mktemp -d)
git clone --quiet "${GIT_REPO}" "${tmpdir}"
cd "${tmpdir}"
git checkout --quiet "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
git checkout --quiet -b "${BLACK_BRANCH}"
# 2. Format files with black.
echo "Formatting files with black."
echo "$CHANGED_FILES" | xargs --no-run-if-empty black --line-length ${BLACK_LINE_LENGTH}
# 3. Commit and push.
echo "Committing and pushing."
git add .
git commit --quiet -m "Auto-format files with black"
git push --quiet origin "${BLACK_BRANCH}" > /dev/null 2>&1
# 4.1 Delete previous comments.
PREVIOUS_COMMENT_IDS=$(gitlab --output json --fields 'id,body,author' project-merge-request-note list \
--project-id ${CI_PROJECT_ID} \
--mr-iid ${CI_MERGE_REQUEST_IID} \
--all | jq ".[] | select(.author.username==\"ci-bot\") | select(.body | strings | test(\"${BLACK_COMMENT_MARKER}\")) | .id")
if [ -n "${PREVIOUS_COMMENT_IDS}" ]; then
while read comment_id; do
echo "Deleting previous black comment ${comment_id}."
gitlab project-merge-request-note delete --project-id ${CI_PROJECT_ID} --mr-iid ${CI_MERGE_REQUEST_IID} --id ${comment_id}
done <<< "${PREVIOUS_COMMENT_IDS}"
else
echo "No previous comments to delete."
fi
# 4.2. Close previous MRs.
PREVIOUS_MRS=$(gitlab --output json --fields iid,source_branch project-merge-request list \
--project-id ${CI_PROJECT_ID} \
--state opened \
--target-branch ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} \
--assignee-id ${GITLAB_USER_ID} \
--labels "${BLACK_MR_LABELS}" \
--all | jq ".[] | select(.source_branch | strings | startswith(\"assist/black\"))")
PREVIOUS_MR_IIDS=$(echo "${PREVIOUS_MRS}" | jq ".iid")
if [ -n "${PREVIOUS_MR_IIDS}" ]; then
while read mr_iid; do
echo "Closing previous black MR ${mr_iid}."
gitlab project-merge-request update --project-id ${CI_PROJECT_ID} --iid ${mr_iid} --state-event close > /dev/null
done <<< "${PREVIOUS_MR_IIDS}"
else
echo "No previous black MRs to close."
fi
# 4.3. Delete previous branches
PREVIOUS_MR_BRANCHES=$(echo "${PREVIOUS_MRS}" | jq --raw-output ".source_branch")
if [ -n "${PREVIOUS_MR_BRANCHES}" ]; then
while read previous_branch; do
echo "Deleting previous black branch ${previous_branch}."
gitlab project-branch delete --project-id ${CI_PROJECT_ID} --name "${previous_branch}"
done <<< "${PREVIOUS_MR_BRANCHES}"
else
echo "No previous black branches to delete."
fi
# 5. Create MR.
# Note: we do this via the API and not via push options, because push options do not
# support assigning the MR to a user.
ASSIST_MR_IID=$(gitlab --output json project-merge-request create \
--project-id ${CI_PROJECT_ID} \
--source-branch "${BLACK_BRANCH}" \
--target-branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" \
--remove-source-branch true \
--assignee-id ${GITLAB_USER_ID} \
--labels "${BLACK_MR_LABELS}" \
--title "Assist: Blacken ${CI_MERGE_REQUEST_TITLE}" \
--description "Auto-format files with black ${BLACK_VERSION}. Assist-MR for !${CI_MERGE_REQUEST_IID}." \
| jq '.iid')
echo "Assist-MR !${ASSIST_MR_IID} created."
# 6. Post comment on MR.
echo "Posting comment on original MR."
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r $(git rev-parse HEAD) | sed "s|^|\\\n- |g")
# Can't use heredoc here because of the indentation. This is an ugly workaround with hardcoded newlines
# and a call to `echo -e` later on (`echo -e` expands `\n` to actual line breaks).
COMMENT_BODY="# Black linting\n\nSome files are not formatted correctly according to black ${BLACK_VERSION}:\n${CHANGED_FILES}\n\nThe assist-MR \!${ASSIST_MR_IID} has been created to fix the formatting.\n\n[//]: # \"${BLACK_COMMENT_MARKER}\"\n"
gitlab project-merge-request-discussion create \
--project-id ${CI_PROJECT_ID} \
--mr-iid ${CI_MERGE_REQUEST_IID} \
--body "$(echo -e ${COMMENT_BODY} | sed 's/\\!/!/g')" \
> /dev/null
# 7. Post comment on previous MRs we've just closed.
COMMENT_BODY=$(echo -e "Closed in favor of \!${ASSIST_MR_IID}.\n\n[//]: # \"${BLACK_COMMENT_MARKER}\"\n" | sed 's/\\!/!/g')
if [ -n "${PREVIOUS_MR_IIDS}" ]; then
while read mr_iid; do
echo "Commenting on previous black MR ${mr_iid}."
gitlab project-merge-request-note create --project-id ${CI_PROJECT_ID} --mr-iid ${mr_iid} --body "${COMMENT_BODY}"
done <<< "${PREVIOUS_MR_IIDS}"
else
echo "No previous black MRs to comment on."
fi
fi
# Finally, return false so the pipeline fails.
echo "Explicitly returning a non-zero exit code to fail the CI because not all files are properly formatted according to black."
false
fi
......@@ -7,6 +7,7 @@
#
stages:
- lint
- build
- test
- quality
......@@ -29,7 +30,7 @@ variables:
- if: $CI_COMMIT_BRANCH == 'develop'
- if: $CI_COMMIT_BRANCH =~ /^release\/.*$/
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_MERGE_REQUEST_ID
- if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/black\/.*$/
- if: $CI_PIPELINE_SOURCE == 'schedule' && $CONDA_BUILD_RUN_JOB
before_script:
- if [ -z "${CONDA_DOWNLOAD_KEY}" ]; then echo "CI variable 'CONDA_DOWNLOAD_KEY' is not set, exiting."; false; fi
......
......@@ -7,6 +7,7 @@
#
stages:
- lint
- build
- test
- quality
......
......@@ -42,7 +42,7 @@ latex_pdf:
- if: $CI_COMMIT_BRANCH == 'develop'
- if: $CI_COMMIT_BRANCH =~ /^release\/.*$/
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_MERGE_REQUEST_ID
- if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/black\/.*$/
latex_pdf_diff:
# Compile a pdf with the differences highlighted.
......
stages:
- lint
- build
- test
- quality
......
stages:
- lint
- build
- test
- quality
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment