From b789c92e81b839b4c1f98b8e8ed336eb52eee382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 3 May 2022 20:26:18 -0500 Subject: [PATCH 01/16] chore: drop 3.6 from test matrix, add 3.9 and 3.10 --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe3eade..4eb7fcc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,18 +8,18 @@ on: jobs: build: - + strategy: matrix: - python-version: [3.6, 3.7, 3.8] - + python-version: ["3.7", "3.8", "3.9", "3.10"] + runs-on: ubuntu-latest steps: - name: Checking out repo uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.container[1] }} + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From e477698ca75079bd4bd4d7e98c564d683161c627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 3 May 2022 20:28:27 -0500 Subject: [PATCH 02/16] chore: add pre-commit --- .github/workflows/main.yml | 3 --- .pre-commit-config.yaml | 35 +++++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 7 ++++--- 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4eb7fcc..8911bfa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,9 +29,6 @@ jobs: python -m pip install -U pip setuptools wheel python3 -m pip install -U .[dev] - - name: Check code formatting - run: find singer tests -type f -name '*.py' | xargs unify --check-only - - name: Analysing the code with pylint run: pylint singer diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b8266e9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +# See https://pre-commit.com for more information + +# See https://pre-commit.ci/ for information about the continuous +# integration service for the pre-commit framework +ci: + autofix_prs: false + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore(deps): pre-commit autoupdate' + + +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/asottile/pyupgrade + rev: v2.32.0 + hooks: + - id: pyupgrade + name: pyupgrade (python) + args: [--py37-plus] +- repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort (python) +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + name: black (python) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 076acf3..1dc7443 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,7 @@ ## Submitting a PR -1. Write tests to cover any new code or code changes. -2. Please make sure that all tests pass and that the code passes linting with `make`. -3. Open up the PR. +1. Install and use [pre-commit](https://pre-commit.com/) to keep your changes in the style of the project. +2. Write tests to cover any new code or code changes. +3. Please make sure that all tests pass and that the code passes linting with `make`. +4. Open up the PR. From 597c653f66c23b2078ae8166287d0a960efb0d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 4 May 2022 19:07:51 -0500 Subject: [PATCH 03/16] chore: use `python_requires` option in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9de29ab..2e628d7 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ setup(name="pipelinewise-singer-python", version='2.0.1', description="Singer.io utility library - PipelineWise compatible", + python_requires=">=3.7.0, <3.11", long_description=long_description, long_description_content_type="text/markdown", author="TransferWise", From ec832b75ab65f81845b74e4821323f1ec33ed57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 4 May 2022 23:05:49 -0500 Subject: [PATCH 04/16] chore: add fail-fast: true --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8911bfa..37db256 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,7 @@ jobs: build: strategy: + fail-fast: false matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] From 7f8562e44950ba0de9e396ea9ffc90cb3e4c2f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 4 May 2022 23:15:11 -0500 Subject: [PATCH 05/16] test: use pytest instead of nose `nose` is no longer maintained and last had a release on 2015. It's also not compatible with Python 3.10. `pytest` is able to run the same test suite without any changes. --- .github/workflows/main.yml | 2 +- .gitignore | 1 - pyproject.toml | 16 ++++++++++++++++ setup.py | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37db256..c4813d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,4 +34,4 @@ jobs: run: pylint singer - name: Runs tests with coverage - run: nosetests --with-doctest -v --nocapture + run: pytest diff --git a/.gitignore b/.gitignore index 03e17bb..7fbbd47 100644 --- a/.gitignore +++ b/.gitignore @@ -86,7 +86,6 @@ ENV/ # Pipenv Pipfile Pipfile.lock -pyproject.toml # Spyder project settings .spyderproject diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3dbe15a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.black] +line-length = 88 + +[tool.isort] +profile = "black" +multi_line_output = 3 +src_paths = "singer" +use_parentheses = true +known_first_party = "singer" +include_trailing_comma = true +add_imports = [ + "from __future__ import annotations", +] + +[tool.pytest.ini_options] +addopts = "-v --doctest-modules" diff --git a/setup.py b/setup.py index 2e628d7..5914bbc 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ extras_require={ 'dev': [ 'pylint==2.11.1', + 'pytest==7.1.2', 'ipython', 'ipdb', 'nose', From 091b6384318d4c269439fffc48afcd05c35ccab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 4 May 2022 23:22:27 -0500 Subject: [PATCH 06/16] test: run `coverage` tool and echo combined coverage report --- .github/workflows/main.yml | 40 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 7 +++++++ setup.py | 1 + 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4813d8..7d09dd0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,4 +34,42 @@ jobs: run: pylint singer - name: Runs tests with coverage - run: pytest + run: coverage run --parallel -m pytest + + - name: Upload coverage data + uses: actions/upload-artifact@v3 + with: + name: coverage-data + path: ".coverage.*" + + coverage: + runs-on: ubuntu-latest + needs: build + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.8" + + - name: Install and upgrade dependencies + run: | + python -m pip install -U pip setuptools wheel + python3 -m pip install -U .[dev] + + - name: Download coverage data + uses: actions/download-artifact@v3.0.0 + with: + name: coverage-data + + - name: Combine coverage data and display human readable report + run: | + coverage combine + coverage xml + coverage report + + # Optional if you want to use codecov.io + # - name: Upload coverage report + # uses: codecov/codecov-action@v3 diff --git a/pyproject.toml b/pyproject.toml index 3dbe15a..257bf6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,13 @@ [tool.black] line-length = 88 +[tool.coverage.paths] +source = ["singer", "*/site-packages"] + +[tool.coverage.run] +branch = true +source = ["singer"] + [tool.isort] profile = "black" multi_line_output = 3 diff --git a/setup.py b/setup.py index 5914bbc..26c80c1 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ 'dev': [ 'pylint==2.11.1', 'pytest==7.1.2', + 'coverage[toml]', 'ipython', 'ipdb', 'nose', From 0c3cf31a79eb5c9c57aaee107fa2b2254a37f50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 4 May 2022 23:19:11 -0500 Subject: [PATCH 07/16] chore: auto-update actions with dependabot --- .github/dependabot.yml | 4 ++++ .github/workflows/main.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b673657..960ed72 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,7 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d09dd0..2e3d8ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,10 +18,10 @@ jobs: steps: - name: Checking out repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} From e25ec0594c640f9a170c18f5e1393a741cc97882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 10 May 2022 13:38:07 -0500 Subject: [PATCH 08/16] add workflow_dispatch trigger --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e3d8ed..9272a91 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,6 +5,8 @@ on: branches: [master] pull_request: branches: [master] + workflow_dispatch: + inputs: {} jobs: build: From 350e5306bf7767d87160f52183c32bd1077e7337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 10 May 2022 14:22:47 -0500 Subject: [PATCH 09/16] remove nose --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 26c80c1..fd48a52 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ 'coverage[toml]', 'ipython', 'ipdb', - 'nose', 'unify==0.5' ] }, From dbb947ce338a6f84879147b48f135d81a8118510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 21:09:38 -0500 Subject: [PATCH 10/16] test: fail for coverage under 76% --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 257bf6d..6d77b3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,9 @@ source = ["singer", "*/site-packages"] branch = true source = ["singer"] +[tool.coverage.report] +fail_under = 76.0 + [tool.isort] profile = "black" multi_line_output = 3 From 74fb6d1efc983883dac81ee64028716d1a750acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 21:15:04 -0500 Subject: [PATCH 11/16] pinned coverage to version 6.* --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fd48a52..50ad070 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'dev': [ 'pylint==2.11.1', 'pytest==7.1.2', - 'coverage[toml]', + 'coverage[toml]~=6.3', 'ipython', 'ipdb', 'unify==0.5' From 12c51403064b2efac75ab977be3dc2982cbd8dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 21:15:56 -0500 Subject: [PATCH 12/16] set max line length to 120 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6d77b3a..5cfc99a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.black] -line-length = 88 +line-length = 120 [tool.coverage.paths] source = ["singer", "*/site-packages"] From f3b1e46f842605f6e1cecd6f74ebf5951681b32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 21:17:49 -0500 Subject: [PATCH 13/16] test: remove */site-packages/ path equivalence to singer/ --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5cfc99a..96df883 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ line-length = 120 [tool.coverage.paths] -source = ["singer", "*/site-packages"] +source = ["singer"] [tool.coverage.run] branch = true From e3d095d1cbbf12e8b05849fd971b77df8a8ad96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 21:26:19 -0500 Subject: [PATCH 14/16] chore: Bump orjson to 3.6.5 to make it compatible with MacOS on 3.10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50ad070..86dbee3 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ install_requires=[ 'pytz', 'jsonschema==3.2.0', - 'orjson==3.6.1', + 'orjson==3.6.5', 'python-dateutil>=2.6.0', 'backoff==1.11.1', 'ciso8601', From c076ba22e008c96e70b43eb58058dec896d42dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 21:56:38 -0500 Subject: [PATCH 15/16] chore: add pylint pre-commit hook --- .pre-commit-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8266e9..f083f21 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # See https://pre-commit.com for more information # See https://pre-commit.ci/ for information about the continuous -# integration service for the pre-commit framework +# integration service for the pre-commit framework ci: autofix_prs: false autoupdate_schedule: weekly @@ -33,3 +33,8 @@ repos: hooks: - id: black name: black (python) +- repo: https://github.com/PyCQA/pylint + rev: v2.13.8 + hooks: + - id: pylint + name: pylint (python) From 58044d6ae2c8b68d2cde31b61c53031e9eefdf32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 12 May 2022 22:16:58 -0500 Subject: [PATCH 16/16] Separate coverage steps --- .github/workflows/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9272a91..aa881dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,10 +66,16 @@ jobs: with: name: coverage-data - - name: Combine coverage data and display human readable report + - name: Combine coverage data run: | coverage combine + + - name: Generate XML coverage report + run: | coverage xml + + - name: Display human readable report + run: | coverage report # Optional if you want to use codecov.io