Merge branch 'tnutma/feature/autopep8' into 'master'

Add autopep8 CI template

See merge request omegacen/ci-templates!44

README.rst

@@ -10,19 +10,47 @@ Ready to include templates for common CI jobs.
 Usage
 =====
 
-black.yml
--------------
+autopep8.yml
+------------
 
 By including this template in your project, its Python files will be linted
-with `black`_ 19.10b0. The behaviour is different for the default branch
-and for Merge Requests (MRs):
+with autopep8 for each Merge Request:
+
+  * Only the changed files will be checked with black.
+  * If they are not properly formatted according to black, the CI will fail.
+  * An assist-MR will be created to automatically fix the formatting.
+  * Upon merging this assist-MR, the changed files are checked again
+    and the CI should pass.
+
+To use the autopep8 CI template, follow these steps:
+
+#. Add `CI Bot`_ as a member to your project in at least the Developer role.
+
+#. Lastly, include the following snippet in your ``.gitlab-ci.yml`` file:
+
+   .. code-block:: yaml
+
+      include:
+        - project: 'omegacen/ci-templates'
+          ref: v4
+          file: 'autopep8.yml'
 
-* For the default / master / release branch(es):
+The variables that can be changed are:
+
+=================== ======== ========= ============================================================================== =========================================
+      Name          Required Protected                                Default value                                                    Purpose
+=================== ======== ========= ============================================================================== =========================================
+AUTOPEP8_SELECT     No       No        E101,E121,E123,E125,E131,E133,E201,E202,E21,E22,E23,E24,E25,E27,E3,E502,W2,W3  The pycodestyle codes to fix.
+AUTOPEP8_IGNORE     No       No        E203,E26                                                                       The pycodestyles to explicitly not fix.
+=================== ======== ========= ============================================================================== =========================================
+
+This CI template cannot be used together with the black CI template.
 
-  * All Python files will be checked with black.
-  * If they are not all properly formatted according to black, the CI will fail.
+black.yml
+---------
 
-* For MRs:
+By including this template in your project, its Python files will be linted
+with `black`_ 19.10b0 for each Merge Request:
 
   * Only the changed files will be checked with black.
   * If they are not properly formatted according to black, the CI will fail.
@@ -30,9 +58,8 @@ and for Merge Requests (MRs):
   * Upon merging this assist-MR, the changed files are checked again
     and the CI should pass.
 
-Thus before first use of this template, all files must be manually formatted
-with black 19.10b. The default line length is 120, although this can be
-changed via the ``BLACK_LINE_LENGTH`` variable.
+The default line length is 120, although this can be changed via the
+``BLACK_LINE_LENGTH`` variable.
 
 To use the black CI template, follow these steps:
 
@@ -44,8 +71,8 @@ To use the black CI template, follow these steps:
 
       include:
         - project: 'omegacen/ci-templates'
-          ref: v3
-          file: '/black.yml'
+          ref: v4
+          file: 'black.yml'
 
 The variables that can be changed are:
 
@@ -53,9 +80,9 @@ The variables that can be changed are:
       Name          Required Protected    Default value                              Purpose
 =================== ======== ========= ================== =================================================================
 BLACK_LINE_LENGTH   No       No        120                How many characters per line black allows.
-BLACK_MR_LABELS     No       No        assist,black       Comma-separated list of labels to attach to the black assist MRs.
 =================== ======== ========= ================== =================================================================
 
+This CI template cannot be used together with the autopep8 CI template.
 
 conda.yml
 ---------
@@ -86,8 +113,8 @@ Next, include the following snippet in your ``.gitlab-ci.yml`` file:
 
    include:
      - project: 'omegacen/ci-templates'
-       ref: v3
-       file: '/conda.yml'
+       ref: v4
+       file: 'conda.yml'
 
 The complete list of variables is given below.
 
@@ -115,8 +142,8 @@ In that case, you must include the ``conda-build.yml`` template instead of
 
    include:
      - project: 'omegacen/ci-templates'
-       ref: v3
-       file: '/conda-build.yml'
+       ref: v4
+       file: 'conda-build.yml'
 
 
 monthlymerge.yml
@@ -137,8 +164,8 @@ To use it, follow these steps:
 
       include:
         - project: 'omegacen/ci-templates'
-          ref: v3
-          file: '/monthlymerge.yml'
+          ref: v4
+          file: 'monthlymerge.yml'
 
 #. Next, create a `pipeline schedule`_ in your project. Set it to the first of
    the month. This pipeline schedule should ideally be owned by `CI Bot`_.
@@ -181,8 +208,8 @@ To use it, follow these steps:
 
       include:
         - project: 'omegacen/ci-templates'
-          ref: v3
-          file: '/sonarqube.yml'
+          ref: v4
+          file: 'sonarqube.yml'
 
 #. Optionally, you can add one or more `Project badges`_. In your GitLab
    project, go to *Settings* -> *General* -> *Badges*. Then add the following:
@@ -220,8 +247,8 @@ To use it, include the following snippet in your ``.gitlab-ci.yml`` file:
 
    include:
      - project: 'omegacen/ci-templates'
-       ref: v3
-       file: '/latex.yml'
+       ref: v4
+       file: 'latex.yml'
 
 The filename of the main tex file is determined automatically. This can be
 overruled by specifying the ``FILENAME_TEX`` variable.

_autoformat.yml

+stages:
+  - lint
+  - build
+  - test
+  - quality
+  - release
+
+compare_ast:
+  stage: lint
+  image: omegacen/autoformat
+  rules:
+    - if: $AUTOFORMAT_COMPARE_AST && $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^assist\/autoformat\/.*$/
+  script:
+    - |
+      cat << EOF > ~/.python-gitlab.cfg
+      [global]
+      default = astro-wise
+
+      [astro-wise]
+      url = ${CI_SERVER_URL}
+      private_token = ${AUTOFORMAT_TOKEN}
+      api_version = 4
+      EOF
+    - |
+      AST_OUTPUT=$(python-compare-ast 2>&1) && AST_RESULT=0 || AST_RESULT=1
+      echo "$AST_OUTPUT"
+      if [ $AST_RESULT -ne 0 ]; then
+        echo "Posting comment about failed AST check on assist-MR."
+        AST_CHANGED_FILES=$( \
+          echo "$AST_OUTPUT" \
+          | grep "^warning: AST does not match for " \
+          | sed "s|^warning: AST does not match for |\\\n- |g" \
+        )
+        COMMENT_BODY="While auto-formatting, the Python AST changed for these files:\n${AST_CHANGED_FILES}\n\nPlease review the changes in this MR carefully. (Hint: did the indentation of docstrings change?)"
+        gitlab project-merge-request-note create \
+            --project-id ${CI_PROJECT_ID} \
+            --mr-iid ${CI_MERGE_REQUEST_IID} \
+            --body "$(echo -e ${COMMENT_BODY} | sed 's/\\!/!/g')" \
+            > /dev/null
+        echo "Explicitly returning a non-zero exit code to fail the CI."
+        false
+      fi
+
+.autoformat_mr:
+  stage: lint
+  image: omegacen/autoformat
+  variables:
+    AUTOFORMAT_MR_LABELS: autoformat
+    AUTOFORMAT_COMMENT_MARKER: autoformat-comment
+    AUTOFORMAT_NAME: abstractautoformatter
+  rules:
+    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/
+  script:
+    - export -f autoformat-check
+    - export -f autoformat-change
+    # Save configuration for python-gitlab. Somehow we can't use simple environment variables for this $%@#$@.
+    - |
+      cat << EOF > ~/.python-gitlab.cfg
+      [global]
+      default = astro-wise
+
+      [astro-wise]
+      url = ${CI_SERVER_URL}
+      private_token = ${AUTOFORMAT_TOKEN}
+      api_version = 4
+      EOF
+    - |
+      # Run the formatter only on changed files.
+      CHANGED_FILES=$(git diff --diff-filter=d --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME...$CI_COMMIT_SHA -- | grep '\.py$' || true)
+      echo "Found these changed Python files:"
+      echo "${CHANGED_FILES}" | sed 's/^/- /g'
+      echo "Proceeding to lint them with ${AUTOFORMAT_NAME}."
+      if echo "$CHANGED_FILES" | xargs --no-run-if-empty -I {} bash -c 'autoformat-check "{}"'; then
+
+          echo "All files are correctly formatted."
+
+          # Resolve any discussions we started.
+          DISCUSSION_IDS=$(gitlab --output json project-merge-request-discussion list \
+              --project-id ${CI_PROJECT_ID} \
+              --mr-iid ${CI_MERGE_REQUEST_IID} \
+              --all \
+              | jq --raw-output ".[] | select(.notes[0].author.username==\"ci-bot\") | select(.notes[0].body | strings | test(\"${AUTOFORMAT_COMMENT_MARKER}\")) | select(.notes[0].resolved == false) | .id")
+          if [ -n "${DISCUSSION_IDS}" ]; then
+              COMMENT_BODY=$(echo -e "All ${AUTOFORMAT_NAME} formatting issues have been fixed. If an assist-MR has been merged, do not forget to update your local \`$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\` branch via a \`git pull\` command.\n\n[//]: # \"${AUTOFORMAT_COMMENT_MARKER}\"\n")
+              while read discussion_id; do
+                  gitlab project-merge-request-discussion-note create \
+                      --project-id ${CI_PROJECT_ID} \
+                      --mr-iid ${CI_MERGE_REQUEST_IID} \
+                      --discussion-id ${discussion_id} \
+                      --body "${COMMENT_BODY}" > /dev/null
+                  echo "Resolving discussion ${discussion_id}."
+                  gitlab project-merge-request-discussion update \
+                      --project-id ${CI_PROJECT_ID} \
+                      --mr-iid ${CI_MERGE_REQUEST_IID} \
+                      --id ${discussion_id} \
+                      --resolved true > /dev/null
+              done <<< "${DISCUSSION_IDS}"
+          else
+              echo "No dicussions to resolve."
+          fi
+
+      else
+          AUTOFORMAT_BRANCH="assist/autoformat/${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/${CI_COMMIT_SHORT_SHA}"
+          GIT_REPO="https://ci-bot:${AUTOFORMAT_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
+          if git ls-remote --exit-code --heads "${GIT_REPO}" "${AUTOFORMAT_BRANCH}"; then
+              echo "Autoformat branch ${AUTOFORMAT_BRANCH} already exists. Exiting."
+          else
+              echo "Autoformat branch ${AUTOFORMAT_BRANCH} does not yet exists, continuing."
+
+              # 1. Clone and checkout a new branch.
+              git config --global user.name "CI Bot"
+              git config --global user.email "astrowisegitlab@rug.nl"
+              tmpdir=$(mktemp -d)
+              git clone --quiet "${GIT_REPO}" "${tmpdir}"
+              cd "${tmpdir}"
+              git checkout --quiet "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
+              git checkout --quiet -b "${AUTOFORMAT_BRANCH}"
+
+              # 2. Auto format files.
+              echo "Formatting files with ${AUTOFORMAT_NAME}."
+              echo "$CHANGED_FILES" | xargs --no-run-if-empty -I {} bash -c 'autoformat-change "{}"'
+
+              # 3. Commit and push.
+              echo "Committing and pushing."
+              git add .
+              git commit --quiet -m "Auto-format files with ${AUTOFORMAT_NAME}"
+              git push --quiet origin "${AUTOFORMAT_BRANCH}" > /dev/null 2>&1
+
+              # 4.1 Delete previous comments.
+              PREVIOUS_COMMENT_IDS=$(gitlab --output json --fields 'id,body,author' project-merge-request-note list \
+                  --project-id ${CI_PROJECT_ID} \
+                  --mr-iid ${CI_MERGE_REQUEST_IID} \
+                  --all | jq ".[] | select(.author.username==\"ci-bot\") | select(.body | strings | test(\"${AUTOFORMAT_COMMENT_MARKER}\")) | .id")
+              if [ -n "${PREVIOUS_COMMENT_IDS}" ]; then
+                  while read comment_id; do
+                      echo "Deleting previous ${AUTOFORMAT_NAME} comment ${comment_id}."
+                      gitlab project-merge-request-note delete --project-id ${CI_PROJECT_ID} --mr-iid ${CI_MERGE_REQUEST_IID} --id ${comment_id}
+                  done <<< "${PREVIOUS_COMMENT_IDS}"
+              else
+                  echo "No previous comments to delete."
+              fi
+
+              # 4.2. Close previous MRs.
+              PREVIOUS_MRS=$(gitlab --output json --fields iid,source_branch project-merge-request list \
+                  --project-id ${CI_PROJECT_ID} \
+                  --state opened \
+                  --target-branch ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} \
+                  --assignee-id ${GITLAB_USER_ID} \
+                  --labels "${AUTOFORMAT_MR_LABELS}" \
+                  --all | jq ".[] | select(.source_branch | strings | startswith(\"assist/autoformat\"))")
+              PREVIOUS_MR_IIDS=$(echo "${PREVIOUS_MRS}" | jq ".iid")
+              if [ -n "${PREVIOUS_MR_IIDS}" ]; then
+                  while read mr_iid; do
+                      echo "Closing previous ${AUTOFORMAT_NAME} MR ${mr_iid}."
+                      gitlab project-merge-request update --project-id ${CI_PROJECT_ID} --iid ${mr_iid} --state-event close > /dev/null
+                  done <<< "${PREVIOUS_MR_IIDS}"
+              else
+                  echo "No previous ${AUTOFORMAT_NAME} MRs to close."
+              fi
+
+              # 4.3. Delete previous branches
+              PREVIOUS_MR_BRANCHES=$(echo "${PREVIOUS_MRS}" | jq --raw-output ".source_branch")
+              if [ -n "${PREVIOUS_MR_BRANCHES}" ]; then
+                  while read previous_branch; do
+                      echo "Deleting previous ${AUTOFORMAT_NAME} branch ${previous_branch}."
+                      gitlab project-branch delete --project-id ${CI_PROJECT_ID} --name "${previous_branch}"
+                  done <<< "${PREVIOUS_MR_BRANCHES}"
+              else
+                  echo "No previous ${AUTOFORMAT_NAME} branches to delete."
+              fi
+
+              # 5. Create MR.
+              #    Note: we do this via the API and not via push options, because push options do not
+              #    support assigning the MR to a user.
+              MR_DESCRIPTION="Auto-format files with ${AUTOFORMAT_NAME}. Assist-MR for !${CI_MERGE_REQUEST_IID}.\n\n_Note_: after merging this MR, do not forget to update your local \`$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\` branch via a \`git pull\` command."
+              ASSIST_MR_IID=$(gitlab --output json project-merge-request create \
+                  --project-id ${CI_PROJECT_ID} \
+                  --source-branch "${AUTOFORMAT_BRANCH}" \
+                  --target-branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" \
+                  --remove-source-branch true \
+                  --assignee-id ${GITLAB_USER_ID} \
+                  --labels "${AUTOFORMAT_MR_LABELS}" \
+                  --title "Assist: ${AUTOFORMAT_NAME} ${CI_MERGE_REQUEST_TITLE}" \
+                  --description "$(echo -e ${MR_DESCRIPTION} | sed 's/\\!/!/g')" \
+                  | jq '.iid')
+              echo "Assist-MR !${ASSIST_MR_IID} created."
+
+              # 6. Post comment on MR.
+              echo "Posting comment on original MR."
+              CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r $(git rev-parse HEAD) | sed "s|^|\\\n- |g")
+              # Can't use heredoc here because of the indentation. This is an ugly workaround with hardcoded newlines
+              # and a call to `echo -e` later on (`echo -e` expands `\n` to actual line breaks).
+              COMMENT_BODY="# ${AUTOFORMAT_NAME} linting\n\nSome files are not formatted correctly according to ${AUTOFORMAT_NAME}:\n${CHANGED_FILES}\n\nThe assist-MR \!${ASSIST_MR_IID} has been created to fix the formatting.\n\n[//]: # \"${AUTOFORMAT_COMMENT_MARKER}\"\n"
+              gitlab project-merge-request-discussion create \
+                  --project-id ${CI_PROJECT_ID} \
+                  --mr-iid ${CI_MERGE_REQUEST_IID} \
+                  --body "$(echo -e ${COMMENT_BODY} | sed 's/\\!/!/g')" \
+                  > /dev/null
+
+              # 7. Post comment on previous MRs we've just closed.
+              COMMENT_BODY=$(echo -e "Closed in favor of \!${ASSIST_MR_IID}.\n\n[//]: # \"${AUTOFORMAT_COMMENT_MARKER}\"\n" | sed 's/\\!/!/g')
+              if [ -n "${PREVIOUS_MR_IIDS}" ]; then
+                  while read mr_iid; do
+                      echo "Commenting on previous ${AUTOFORMAT_NAME} MR ${mr_iid}."
+                      gitlab project-merge-request-note create --project-id ${CI_PROJECT_ID} --mr-iid ${mr_iid} --body "${COMMENT_BODY}" > /dev/null
+                  done <<< "${PREVIOUS_MR_IIDS}"
+              else
+                  echo "No previous ${AUTOFORMAT_NAME} MRs to comment on."
+              fi
+
+          fi
+          # Finally, return false so the pipeline fails.
+          echo "Explicitly returning a non-zero exit code to fail the CI because not all files are properly."
+          false
+      fi

autopep8.yml

+include:
+  - local: '_autoformat.yml'
+
+variables:
+  AUTOFORMAT_COMPARE_AST: 1
+
+autopep8:
+  extends: .autoformat_mr
+  variables:
+    AUTOFORMAT_MR_LABELS: autoformat,autopep8
+    AUTOFORMAT_NAME: autopep8
+    # Only enable autopep8 fixes that enforce:
+    #
+    #   * Indents to be a multiple of four, or
+    #   * Correct whitespacing around assigments and operators.
+    #
+    # See https://github.com/hhatto/autopep8#features for a full list of codes.
+    # Here's why we omitted / excluded various codes:
+    #
+    #   * The omitted E1XX codes are excluded because they try to visually improve
+    #     the indents. I (Teake) found this to be too aggressive.
+    #
+    #   * We exclude E203 because it is incompatible with Black,
+    #     see https://black.readthedocs.io/en/stable/compatible_configs.html#flake8.
+    #
+    #   * We disable E26X (fix spacing of comments) because a lot of comments
+    #     are commented-out code that have a hash directly at the start of the line
+    #     -- these would get indented to a multiple of four (which is not what we want).
+    #
+    #   * We exclude E4XX (fix imports) because they change the AST.
+    #
+    #   * The other omitted fix codes are omitted because they do not improve the
+    #     code formatting, but improve the code style (e.g. use "!=" instead of "<>").
+    #
+    AUTOPEP8_SELECT: "E101,E121,E123,E125,E131,E133,E201,E202,E21,E22,E23,E24,E25,E27,E3,E502,W2,W3"
+    AUTOPEP8_IGNORE: "E203,E26"
+  before_script:
+    - |
+      autoformat-check () {
+        autopep8 --exit-code --diff --select "${AUTOPEP8_SELECT}" --ignore "${AUTOPEP8_IGNORE}" "$@" > /dev/null
+      }
+    - |
+      autoformat-change () {
+        autopep8 --in-place --select "${AUTOPEP8_SELECT}" --ignore "${AUTOPEP8_IGNORE}"  "$@" > /dev/null
+      }

black.yml

-stages:
-  - lint
-  - build
-  - test
-  - quality
-  - release
+include:
+  - local: '_autoformat.yml'
 
-
-.abstract_black:
-  image: omegacen/black
-  stage: lint
+autopep8:
+  extends: .autoformat_mr
   variables:
+    AUTOFORMAT_MR_LABELS: autoformat,black
+    AUTOFORMAT_NAME: black
     BLACK_LINE_LENGTH: 120
-    BLACK_MR_LABELS: assist,black
-    BLACK_COMMENT_MARKER: black-auto-comment
-    GIT_DEPTH: 0
-
-black_branch:
-  extends: .abstract_black
-  rules:
-    - if: $CI_COMMIT_TAG
-    - if: $CI_COMMIT_BRANCH == 'master'
-    - if: $CI_COMMIT_BRANCH == 'develop'
-    - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-  script:
-    - black --line-length ${BLACK_LINE_LENGTH} --check .
-
-black_mr:
-  extends: .abstract_black
-  rules:
-    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/black\/.*$/
   before_script:
-    - export BLACK_VERSION=$(black --version | cut -d ' ' -f3)
-    # Save configuration for python-gitlab. Somehow we can't use simple environment variables for this $%@#$@.
     - |
-      cat << EOF > ~/.python-gitlab.cfg
-      [global]
-      default = astro-wise
-
-      [astro-wise]
-      url = ${CI_SERVER_URL}
-      private_token = ${BLACK_TOKEN}
-      api_version = 4
-      EOF
-  script:
+      autoformat-check () {
+        black --line-length ${BLACK_LINE_LENGTH} --check "$@"
+      }
     - |
-      # Run black only on changed files.
-      CHANGED_FILES=$(git diff --diff-filter=d --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME...$CI_COMMIT_SHA -- | grep '\.py$' || true)
-      echo "Found these changed Python files:"
-      echo "${CHANGED_FILES}" | sed 's/^/- /g'
-      echo "Proceeding to lint them with black."
-      if echo "$CHANGED_FILES" | xargs --no-run-if-empty black --line-length ${BLACK_LINE_LENGTH} --check ; then
-
-          echo "All files are correctly formatted."
-
-          # Resolve any discussions we started.
-          DISCUSSION_IDS=$(gitlab --output json project-merge-request-discussion list \
-              --project-id ${CI_PROJECT_ID} \
-              --mr-iid ${CI_MERGE_REQUEST_IID} \
-              --all \
-              | jq --raw-output ".[] | select(.notes[0].author.username==\"ci-bot\") | select(.notes[0].body | strings | test(\"${BLACK_COMMENT_MARKER}\")) | select(.notes[0].resolved == false) | .id")
-          if [ -n "${DISCUSSION_IDS}" ]; then
-              COMMENT_BODY=$(echo -e "All black formatting issues have been fixed.\n\n[//]: # \"${BLACK_COMMENT_MARKER}\"\n")
-              while read discussion_id; do
-                  gitlab project-merge-request-discussion-note create \
-                      --project-id ${CI_PROJECT_ID} \
-                      --mr-iid ${CI_MERGE_REQUEST_IID} \
-                      --discussion-id ${discussion_id} \
-                      --body "${COMMENT_BODY}" > /dev/null
-                  echo "Resolving discussion ${discussion_id}."
-                  gitlab project-merge-request-discussion update \
-                      --project-id ${CI_PROJECT_ID} \
-                      --mr-iid ${CI_MERGE_REQUEST_IID} \
-                      --id ${discussion_id} \
-                      --resolved true > /dev/null
-              done <<< "${DISCUSSION_IDS}"
-          else
-              echo "No dicussions to resolve."
-          fi
-
-      else
-          BLACK_BRANCH="assist/black/${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/${CI_COMMIT_SHORT_SHA}"
-          GIT_REPO="https://ci-bot:${BLACK_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
-          if git ls-remote --exit-code --heads "${GIT_REPO}" "${BLACK_BRANCH}"; then
-              echo "Blackened branch ${BLACK_BRANCH} already exists. Exiting."
-          else
-              echo "Blackened branch ${BLACK_BRANCH} does not yet exists, continuing."
-
-              # 1. Clone and checkout a new branch.
-              git config --global user.name "CI Bot"
-              git config --global user.email "astrowisegitlab@rug.nl"
-              tmpdir=$(mktemp -d)
-              git clone --quiet "${GIT_REPO}" "${tmpdir}"
-              cd "${tmpdir}"
-              git checkout --quiet "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
-              git checkout --quiet -b "${BLACK_BRANCH}"
-
-              # 2. Format files with black.
-              echo "Formatting files with black."
-              echo "$CHANGED_FILES" | xargs --no-run-if-empty black --line-length ${BLACK_LINE_LENGTH}
-
-              # 3. Commit and push.
-              echo "Committing and pushing."
-              git add .
-              git commit --quiet -m "Auto-format files with black"
-              git push --quiet origin "${BLACK_BRANCH}" > /dev/null 2>&1
-
-              # 4.1 Delete previous comments.
-              PREVIOUS_COMMENT_IDS=$(gitlab --output json --fields 'id,body,author' project-merge-request-note list \
-                  --project-id ${CI_PROJECT_ID} \
-                  --mr-iid ${CI_MERGE_REQUEST_IID} \
-                  --all | jq ".[] | select(.author.username==\"ci-bot\") | select(.body | strings | test(\"${BLACK_COMMENT_MARKER}\")) | .id")
-              if [ -n "${PREVIOUS_COMMENT_IDS}" ]; then
-                  while read comment_id; do
-                      echo "Deleting previous black comment ${comment_id}."
-                      gitlab project-merge-request-note delete --project-id ${CI_PROJECT_ID} --mr-iid ${CI_MERGE_REQUEST_IID} --id ${comment_id}
-                  done <<< "${PREVIOUS_COMMENT_IDS}"
-              else
-                  echo "No previous comments to delete."
-              fi
-
-              # 4.2. Close previous MRs.
-              PREVIOUS_MRS=$(gitlab --output json --fields iid,source_branch project-merge-request list \
-                  --project-id ${CI_PROJECT_ID} \
-                  --state opened \
-                  --target-branch ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} \
-                  --assignee-id ${GITLAB_USER_ID} \
-                  --labels "${BLACK_MR_LABELS}" \
-                  --all | jq ".[] | select(.source_branch | strings | startswith(\"assist/black\"))")
-              PREVIOUS_MR_IIDS=$(echo "${PREVIOUS_MRS}" | jq ".iid")
-              if [ -n "${PREVIOUS_MR_IIDS}" ]; then
-                  while read mr_iid; do
-                      echo "Closing previous black MR ${mr_iid}."
-                      gitlab project-merge-request update --project-id ${CI_PROJECT_ID} --iid ${mr_iid} --state-event close > /dev/null
-                  done <<< "${PREVIOUS_MR_IIDS}"
-              else
-                  echo "No previous black MRs to close."
-              fi
-
-              # 4.3. Delete previous branches
-              PREVIOUS_MR_BRANCHES=$(echo "${PREVIOUS_MRS}" | jq --raw-output ".source_branch")
-              if [ -n "${PREVIOUS_MR_BRANCHES}" ]; then
-                  while read previous_branch; do
-                      echo "Deleting previous black branch ${previous_branch}."
-                      gitlab project-branch delete --project-id ${CI_PROJECT_ID} --name "${previous_branch}"
-                  done <<< "${PREVIOUS_MR_BRANCHES}"
-              else
-                  echo "No previous black branches to delete."
-              fi
-
-              # 5. Create MR.
-              #    Note: we do this via the API and not via push options, because push options do not
-              #    support assigning the MR to a user.
-              ASSIST_MR_IID=$(gitlab --output json project-merge-request create \
-                  --project-id ${CI_PROJECT_ID} \
-                  --source-branch "${BLACK_BRANCH}" \
-                  --target-branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" \
-                  --remove-source-branch true \
-                  --assignee-id ${GITLAB_USER_ID} \
-                  --labels "${BLACK_MR_LABELS}" \
-                  --title "Assist: Blacken ${CI_MERGE_REQUEST_TITLE}" \
-                  --description "Auto-format files with black ${BLACK_VERSION}. Assist-MR for !${CI_MERGE_REQUEST_IID}." \
-                  | jq '.iid')
-              echo "Assist-MR !${ASSIST_MR_IID} created."
-
-              # 6. Post comment on MR.
-              echo "Posting comment on original MR."
-              CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r $(git rev-parse HEAD) | sed "s|^|\\\n- |g")
-              # Can't use heredoc here because of the indentation. This is an ugly workaround with hardcoded newlines
-              # and a call to `echo -e` later on (`echo -e` expands `\n` to actual line breaks).
-              COMMENT_BODY="# Black linting\n\nSome files are not formatted correctly according to black ${BLACK_VERSION}:\n${CHANGED_FILES}\n\nThe assist-MR \!${ASSIST_MR_IID} has been created to fix the formatting.\n\n[//]: # \"${BLACK_COMMENT_MARKER}\"\n"
-              gitlab project-merge-request-discussion create \
-                  --project-id ${CI_PROJECT_ID} \
-                  --mr-iid ${CI_MERGE_REQUEST_IID} \
-                  --body "$(echo -e ${COMMENT_BODY} | sed 's/\\!/!/g')" \
-                  > /dev/null
-
-              # 7. Post comment on previous MRs we've just closed.
-              COMMENT_BODY=$(echo -e "Closed in favor of \!${ASSIST_MR_IID}.\n\n[//]: # \"${BLACK_COMMENT_MARKER}\"\n" | sed 's/\\!/!/g')
-              if [ -n "${PREVIOUS_MR_IIDS}" ]; then
-                  while read mr_iid; do
-                      echo "Commenting on previous black MR ${mr_iid}."
-                      gitlab project-merge-request-note create --project-id ${CI_PROJECT_ID} --mr-iid ${mr_iid} --body "${COMMENT_BODY}"
-                  done <<< "${PREVIOUS_MR_IIDS}"
-              else
-                  echo "No previous black MRs to comment on."
-              fi
-
-          fi
-          # Finally, return false so the pipeline fails.
-          echo "Explicitly returning a non-zero exit code to fail the CI because not all files are properly formatted according to black."
-          false
-      fi
+      autoformat-change () {
+        black --line-length ${BLACK_LINE_LENGTH} "$@"
+      }

conda-build.yml

@@ -30,7 +30,7 @@ variables:
     - if: $CI_COMMIT_BRANCH == 'develop'
     - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/black\/.*$/
+    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/
     - if: $CI_PIPELINE_SOURCE == 'schedule' && $CONDA_BUILD_RUN_JOB
   before_script:
     - if [ -z "${CONDA_DOWNLOAD_KEY}" ]; then echo "CI variable 'CONDA_DOWNLOAD_KEY' is not set, exiting."; false; fi

latex.yml

@@ -42,7 +42,7 @@ latex_pdf:
     - if: $CI_COMMIT_BRANCH == 'develop'
     - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/black\/.*$/
+    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^assist\/autoformat\/.*$/
 
 latex_pdf_diff:
   # Compile a pdf with the differences highlighted.