From 104e71dcf620641ef20fed23579ca47d05ba1936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 6 Oct 2018 03:14:49 +0200 Subject: [PATCH] core -- Add black,isort,flake8 to CI and pre-commit hook On CI: - black and isort in check mode must pass. - flake8 only prints errors WIP: make it also failed. On pre-commit: - black and isort will format staged files, if installed on path. - flake8 prints its output if necessary. --- .gitlab-ci.yml | 48 ++++++++++++------- .pre-commit.sh | 106 +++++++++++++++++++++++++++++++++++++++++ README.md | 10 ++++ pyproject.toml | 9 ++++ requirements-devel.txt | 5 ++ setup.cfg | 24 ++++++++++ 6 files changed, 186 insertions(+), 16 deletions(-) create mode 100755 .pre-commit.sh create mode 100644 pyproject.toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5386f031..8fcd1144 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,25 +22,41 @@ variables: # psql password authentication PGPASSWORD: $POSTGRES_PASSWORD -cache: - paths: - - vendor/ - -before_script: - - mkdir -p vendor/{pip,apt} - - apt-get update -q && apt-get -o dir::cache::archives="vendor/apt" install -yqq postgresql-client - - sed -E 's/^REDIS_HOST.*/REDIS_HOST = "redis"/' cof/settings/secret_example.py > cof/settings/secret.py - - sed -i.bak -E 's;^REDIS_PASSWD = .*$;REDIS_PASSWD = "";' cof/settings/secret.py - # Remove the old test database if it has not been done yet - - psql --username=$POSTGRES_USER --host=$DBHOST -c "DROP DATABASE IF EXISTS test_$POSTGRES_DB" - - pip install --upgrade -r requirements.txt - - pip install coverage - - python --version - test: stage: test + before_script: + - mkdir -p vendor/{pip,apt} + - apt-get update -q && apt-get -o dir::cache::archives="vendor/apt" install -yqq postgresql-client + - sed -E 's/^REDIS_HOST.*/REDIS_HOST = "redis"/' cof/settings/secret_example.py > cof/settings/secret.py + - sed -i.bak -E 's;^REDIS_PASSWD = .*$;REDIS_PASSWD = "";' cof/settings/secret.py + # Remove the old test database if it has not been done yet + - psql --username=$POSTGRES_USER --host=$DBHOST -c "DROP DATABASE IF EXISTS test_$POSTGRES_DB" + - pip install --upgrade -r requirements.txt coverage + - python --version script: - coverage run manage.py test + after_script: - coverage report - # For GitLab, keep this commented. + cache: + key: test + paths: + - vendor/ + # For GitLab CI to get coverage from build. + # Keep this disabled for now, at it may kill GitLab... # coverage: '/TOTAL.*\s(\d+\.\d+)\%$/' + +linters: + image: python:3.6 + stage: test + before_script: + - mkdir -p vendor/pip + - pip install --upgrade black isort flake8 + script: + - black --check . + - isort --recursive --check-only --diff bda cof gestioncof kfet provisioning shared utils + # Print errors only + - flake8 --exit-zero bda cof gestioncof kfet provisioning shared utils + cache: + key: linters + paths: + - vendor/ diff --git a/.pre-commit.sh b/.pre-commit.sh new file mode 100755 index 00000000..e621b126 --- /dev/null +++ b/.pre-commit.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# pre-commit hook for gestioCOF project. +# +# Run formatters first, then checkers. +# Formatters which changed a file must set the flag 'formatter_updated'. + +exit_code=0 +formatter_updated=0 +checker_dirty=0 + +# TODO(AD): We should check only staged changes. +# Working? -> Stash unstaged changes, run it, pop stash +STAGED_PYTHON_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep ".py$") + +# Formatter: black + +printf "> black ... " + +if type black &>/dev/null; then + if [ -z $STAGED_PYTHON_FILES ]; then + printf "OK\n" + else + BLACK_OUTPUT="/tmp/gc-black-output.log" + touch $BLACK_OUTPUT + black --check $STAGED_PYTHON_FILES &>$BLACK_OUTPUT + printf "OK\n" + + if [ $? -ne 0 ]; then + black $STAGED_PYTHON_FILES &>$BLACK_OUTPUT + tail -1 $BLACK_OUTPUT + formatter_updated=1 + fi + fi +else + printf "SKIP: program not found\n" + printf "HINT: Install black with 'pip3 install black' (black requires Python>=3.6)\n" +fi + +# Formatter: isort + +printf "> isort ... " + +if type isort &>/dev/null; then + if [ -z $STAGED_PYTHON_FILES ]; then + printf "OK\n" + else + ISORT_OUTPUT="/tmp/gc-isort-output.log" + touch $ISORT_OUTPUT + isort --check-only $STAGED_PYTHON_FILES &>$ISORT_OUTPUT + printf "OK\n" + + if [ $? -ne 0 ]; then + isort $STAGED_PYTHON_FILES &>$ISORT_OUTPUT + printf "Reformatted.\n" + formatter_updated=1 + fi + fi +else + printf "SKIP: program not found\n" + printf "HINT: Install isort with 'pip install isort'\n" +fi + +# Checker: flake8 + +printf "> flake8 ... " + +if type flake8 &>/dev/null; then + if [ -z $STAGED_PYTHON_FILES ]; then + printf "OK\n" + else + FLAKE8_OUTPUT="/tmp/gc-flake8-output.log" + touch $FLAKE8_OUTPUT + flake8 $STAGED_PYTHON_FILES &>$FLAKE8_OUTPUT + + if [ $? -eq 0 ]; then + printf "OK\n" + else + printf "FAIL\n" + cat $FLAKE8_OUTPUT + checker_dirty=1 + fi + fi +else + printf "SKIP: program not found\n" + printf "HINT: Install flake8 with 'pip install flake8'\n" +fi + +# End + +if [ $checker_dirty -ne 0 ] +then + printf ">>> Checker(s) detect(s) issue(s)\n" + printf " You can still commit and push :)\n" + printf " Be warned that our CI may cause you more trouble.\n" +fi + +if [ $formatter_updated -ne 0 ] +then + printf ">>> Working tree updated by formatter(s)\n" + printf " Add changes to staging area and retry.\n" + exit_code=1 +fi + +printf "\n" + +exit $exit_code diff --git a/README.md b/README.md index 524e558d..2f08f3aa 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,16 @@ pour profiter de façon transparente des mises à jour du fichier: ln -s secret_example.py cof/settings/secret.py +Nous avons un git hook de pre-commit pour formatter et vérifier que votre code +vérifie nos conventions. Pour bénéficier des mises à jour du hook, préférez +encore l'installation *via* un lien symbolique: + + ln -s ../../.pre-commit.sh .git/hooks/pre-commit + +Pour plus d'informations à ce sujet, consulter la +[page](https://git.eleves.ens.fr/cof-geek/gestioCOF/wikis/coding-style) +du wiki gestioCOF liée aux conventions. + #### Fin d'installation diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..93b26440 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.black] +# Automatically ignore files in .gitignore (opened at this time): +# https://github.com/ambv/black/issues/475 +exclude = ''' +/( + \.pyc + | venv +)/ +''' diff --git a/requirements-devel.txt b/requirements-devel.txt index 83053f76..6a5acdd7 100644 --- a/requirements-devel.txt +++ b/requirements-devel.txt @@ -2,3 +2,8 @@ django-debug-toolbar django-debug-panel ipython + +# Tools +# black # Uncomment when GC & most distros run with Python>=3.6 +flake8 +isort diff --git a/setup.cfg b/setup.cfg index 35eb701a..ec29c73c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,27 @@ branch = true [coverage:report] precision = 2 show_missing = true + +[flake8] +exclude = migrations +max-line-length = 88 +ignore = + # whitespace before ':' (not PEP8-compliant for slicing) + E203, + # lambda expression + E731, + # line break before binary operator (not PEP8-compliant) + W503 + +[isort] +# For black compat: https://github.com/ambv/black#how-black-wraps-lines +combine_as_imports = true +default_section = THIRDPARTY +force_grid_wrap = 0 +include_trailing_comma = true +known_django = django +known_first_party = bda,cof,gestioncof,kfet,shared,utils +line_length = 88 +multi_line_output = 3 +not_skip = __init__.py +sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER