Commit 5c4d619e authored by Teake Nutma's avatar Teake Nutma Committed by Ewout Helmich
Browse files

Dynamically fetch the Pylint rules from SonarQube

parent 27695ab9
......@@ -14,31 +14,73 @@ stages:
SONAR_HOST_URL: https://sonar.astro-wise.org
SONAR_SOURCES: '.'
SONAR_INCLUSIONS: '**/*.py,**/*.html,**/*.tmpl,**/*.js,**/*.ts,**/*.css'
PYLINT_ENABLE: >-
C0121,C0202,C0203,C0204,C0322,C0323,C0324,C0326,C0328,C0411,C0412,C0413,
E0001,E0102,E0108,E0109,E0202,E0203,E0211,E0213,E0221,E0222,E0238,E0501,
E0502,E0503,E0603,E0604,E0701,E0710,E0711,E0712,E1003,E1101,E1102,E1103,
E1111,E1120,E1121,E1122,E1123,E1124,E1201,F0004,R0401,R0904,W0102,W0104,
W0105,W0106,W0109,W0120,W0150,W0199,W0201,W0211,W0212,W0221,W0222,W0223,
W0231,W0232,W0233,W0234,W0301,W0311,W0312,W0401,W0402,W0404,W0406,W0601,
W0602,W0611,W0613,W0614,W0622,W0623,W0631,W0632,W0633,W0701,W0702,W0703,
W0704,W0711,W1111,W1501
GIT_DEPTH: 0
before_script:
- |
call-sonar-api-without-project() {
endpoint="$1"
shift
data_params="${@/#/--data }"
curl -s -u $SONAR_TOKEN: ${data_params} "${SONAR_HOST_URL}/${endpoint}"
}
call-sonar-api() {
call-sonar-api-without-project "$@" "project=$CI_PROJECT_PATH"
}
- |
count-lines () {
echo "$@" | grep -cve '^\s*$' || true
}
- |
perform-sonar-scan() {
# First, fetch the Pylint rules.
qp_json=$(call-sonar-api api/qualityprofiles/search language=py)
qp_key=$(echo "$qp_json" | jq -r ".profiles[0]?.key? // empty")
if [ -z "$qp_key" ]; then
echo "No Python Quality Profile found for project." \
"Falling back to the default profile."
qp_json=$(call-sonar-api-without-project \
api/qualityprofiles/search \
language=py defaults=true \
)
qp_key=$(echo "$qp_json" | jq -r ".profiles[0]?.key? // empty")
fi
qp_name=$(echo "$qp_json" | jq -r ".profiles[0]?.name? // empty")
echo "Fetching Pylint rules for Python Quality Profile '${qp_name}'."
# Note: the `api/rules/search` endpoint is paginated.
# I'm assuming we'll never have more than 500 Pylint rules enabled.
pylint_rules_json=$(call-sonar-api-without-project \
api/rules/search \
"qprofile=$qp_key" \
activation=true ps=500 repositories=Pylint f=internalKey \
)
pylint_rules_list=$(echo "$pylint_rules_json" | jq -r ".rules[]?.internalKey?")
pylint_rules=$(echo "$pylint_rules_list" | sort | paste -sd "," -)
echo "Number of Pylint rules found: " $(count-lines "$pylint_rules_list")
# Now determine which files to lint.
if [ -z "${CI_MERGE_REQUEST_ID}" ]; then
# Not a MR -- scan all Python files.
echo "Executing Pylint for all Python files."
PYLINT_FILES=$(find . -type f -name '*.py')
pylint_files=$(find . -type f -name '*.py')
else
# Is a MR -- scan only changed Python files.
echo "Executing Pylint only for changed Python files."
FORK_POINT=$(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA)
PYLINT_FILES=$(git diff --diff-filter=d --name-only $FORK_POINT $CI_COMMIT_SHA | grep '\.py$' || true)
echo "Executing Pylint for only the changed Python files."
fork_point=$(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA)
pylint_files=$(git diff --diff-filter=d --name-only $fork_point $CI_COMMIT_SHA | grep '\.py$' || true)
fi
echo -n "Number of Python files to pass to Pylint: "; echo "$PYLINT_FILES" | grep -cve '^\s*$' || true
echo "$PYLINT_FILES" | xargs --no-run-if-empty pylint --exit-zero --disable=all --enable="$PYLINT_ENABLE" -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" > /tmp/pylint.log
echo "Number of Python files to pass to Pylint: " $(count-lines "$pylint_files")
# Perform the Pylint scan.
echo "$pylint_files" | xargs --no-run-if-empty pylint \
--exit-zero \
--disable=all \
--enable="$pylint_rules" \
--reports=n \
--msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
> /tmp/pylint.log
# Perform the sonar scan.
sonar-scanner \
-Dsonar.projectKey=$CI_PROJECT_PATH \
-Dsonar.projectVersion=$CI_COMMIT_SHA \
......@@ -51,13 +93,6 @@ stages:
$SONAR_SCANNER_EXTRA_ARGS \
$@
}
- |
call-sonar-api() {
endpoint="$1"
shift
data_params="${@/#/--data }"
curl -u $SONAR_TOKEN: --data "project=$CI_PROJECT_PATH" ${data_params} "${SONAR_HOST_URL}/${endpoint}"
}
sonar_default_branch:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment