Commit e030f92e authored by Hugo Buddelmeijer's avatar Hugo Buddelmeijer Committed by Teake Nutma
Browse files

Merge branch 'tnutma/feature/sonarqube-8.9' into 'master'

Update sonarqube template for SonarQube 8.9

See merge request omegacen/ci-templates!48
parent 67cddb61
......@@ -186,6 +186,7 @@ SONAR_SOURCES No No .
SONAR_INCLUSIONS No No `**/*.py, **/*.html, **/*.tmpl, **/*.js, **/*.ts, **/*.css` Explicitly including files by patterns (comma separated).
SONAR_EXCLUSIONS No No Explicitly excluding files by patterns (comma separated).
SONAR_SCANNER_EXTRA_ARGS No No Adding extra parameters to Sonar Scanner command line.
SONAR_PYLINT_RULES No No `C0326:MINOR:1,C0328:MINOR:1,[...],W0703:MINOR:20` Comma-separated list of `<pylint_rule_id>:<severity>:<effort>`
========================= ======== ========= =========================================================== ==============================================================
......
......@@ -16,6 +16,19 @@ stages:
SONAR_PROJECT_KEY: $CI_PROJECT_PATH_SLUG
SONAR_SOURCES: '.'
SONAR_INCLUSIONS: '**/*.py,**/*.html,**/*.tmpl,**/*.js,**/*.ts,**/*.css'
SONAR_PYLINT_RULES: >
C0326:MINOR:1,
C0328:MINOR:1,
C0411:MAJOR:5,
C0412:MINOR:5,
C0413:MINOR:5,
E0102:MAJOR:15,
W0311:MINOR:1,
W0404:MAJOR:1,
W0406:MINOR:15,
W0611:MINOR:1,
W0613:MINOR:15,
W0703:MINOR:20
GIT_DEPTH: 0
before_script:
- |
......@@ -33,50 +46,67 @@ stages:
echo "$@" | grep -cve '^\s*$' || true
}
- |
get_pylint_rules_list() {
get_quality_profile_key () {
call-sonar-api-without-project api/qualityprofiles/search language=py "$@" \
| jq -r ".profiles[0]?.key? // empty"
}
# Determine the Quality Profile.
qp_key=$(get_quality_profile_key "project=$SONAR_PROJECT_KEY")
if [ -z "$qp_key" ]; then
# No Python Quality Profile found for project.
# Falling back to the default profile.
qp_key=$(get_quality_profile_key defaults=true)
fi
# Note: the `api/rules/search` endpoint is paginated.
# I'm assuming we'll never have more than 500 Pylint rules enabled.
call-sonar-api-without-project \
api/rules/search \
"qprofile=$qp_key" \
activation=true ps=500 repositories=Pylint f=internalKey \
| jq -r ".rules[]?.internalKey?"
colon_sep_to_json() {
local sep=':'
local columns="$1"
shift
local fields="$@"
echo "$fields" \
| cut -d "$sep" -f"$columns" \
| tr "$sep" '\n' \
| jq -n -R 'reduce inputs as $i ({}; . + { ($i): (input|(tonumber? // .)) })'
}
- |
pylint-to-sonarjson() {
# Turn comma-separated string into newline-separated string
local RULES=$(echo "$SONAR_PYLINT_RULES" | tr ',' '\n' | awk '{$1=$1;print}')
# Get only the rule identifiers to feed to Pylint.
local RULE_IDS=$(echo "$RULES" | cut -d ':' -f 1 | paste -s -d "," -)
# Make JSON for mappings to severities and minutes of effort.
local RULE_SEVERITIES=$(colon_sep_to_json '1,2' "$RULES")
local RULE_EFFORT_MIN=$(colon_sep_to_json '1,3' "$RULES")
local SONAR_JSON_FORMAT='{
engineId: "PYLINT",
ruleId: .["message-id"],
type: "CODE_SMELL",
primaryLocation: {
message: .message,
filePath: .path,
textRange: {
startLine: .line,
startColumn: .column
}
},
severity: .["message-id"],
effortMinutes: .["message-id"]
}'
pylint \
--exit-zero \
--disable=all \
--enable="$RULE_IDS" \
--reports=n \
--output-format=json "$@" \
| jq ".[] | $SONAR_JSON_FORMAT" \
| jq \
--argjson severities "$RULE_SEVERITIES" \
--argjson effort_min "$RULE_EFFORT_MIN" \
'.effortMinutes |= $effort_min[.] | .severity |= $severities[.]'
}
- |
perform-sonar-scan() {
# First, fetch the Pylint rules.
pylint_rules_list=$(get_pylint_rules_list)
pylint_rules=$(echo "$pylint_rules_list" | sort | paste -sd "," -)
echo "Number of Pylint rules found: " $(count-lines "$pylint_rules_list")
echo "Pylint rules: $pylint_rules"
# Now determine which files to lint.
# Determine which files to lint.
pylint_files=$(pylint_files_strategy)
echo "Number of Python files to pass to Pylint: " $(count-lines "$pylint_files")
# Perform the Pylint scan.
echo "$pylint_files" | xargs --no-run-if-empty pylint \
--exit-zero \
--disable=all \
--enable="$pylint_rules" \
--reports=n \
--msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
> /tmp/pylint.log
# Export the function and variables so that the subshell in xargs can find them.
export -f pylint-to-sonarjson
export -f colon_sep_to_json
export SONAR_PYLINT_RULES
echo "$pylint_files" | xargs --no-run-if-empty -I {} bash -c 'pylint-to-sonarjson "$@"' _ {} > /tmp/pylint.log
# One last pass with jq to collect all issues in proper SonarQube JSON.
jq -s "{issues: .}" /tmp/pylint.log > /tmp/pylint.json
# Perform the sonar scan.
sonar-scanner \
......@@ -87,7 +117,8 @@ stages:
-Dsonar.inclusions="$SONAR_INCLUSIONS" \
-Dsonar.exclusions="$SONAR_EXCLUSIONS" \
-Dsonar.sourceEncoding=utf-8 \
-Dsonar.python.pylint.reportPath=/tmp/pylint.log \
-Dsonar.externalIssuesReportPaths=/tmp/pylint.json \
-Dsonar.qualitygate.wait=true \
$SONAR_SCANNER_EXTRA_ARGS \
$@
}
......@@ -134,7 +165,7 @@ sonar_mr:
# Make sure the ALM provider is set to GitLab to enable MR commenting.
# Note: there has to be a global setting in SonarQube that has the same name
# as the value of $CI_SERVER_HOST and has the correct user token for the CI bot.
call-sonar-api api/alm_settings/set_gitlab_binding "almSetting=$CI_SERVER_HOST"
call-sonar-api api/alm_settings/set_gitlab_binding "almSetting=$CI_SERVER_HOST" "repository=$CI_PROJECT_ID"
# Do the actual scan.
perform-sonar-scan
else
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment