From 13520fb12ef53c49a36ef9b44d6800bc18abe9ed Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Mon, 22 Jan 2024 17:22:26 +0100 Subject: [PATCH 01/13] Add github workflows --- .github/workflows/lint.yml | 36 ++++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..4f61613 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,36 @@ +name: Lint python package + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install pip + run: python -m pip install -U pip + + - name: Install tox + run: python -m pip install tox + + - name: Run linter + run: tox -e lint + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: lint-codeclimate + path: artifacts/lint/lint-codeclimate.json + retention-days: 3 + overwrite: true + + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..89f3c72 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: Test python package + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install pip + run: python -m pip install -U pip + + - name: Install package + run: pip install -e .[dev,test] + + - name: Install tox + run: python -m install tox + + - name: Run tests + run: tox -e py From 72c7320159cc13f7ce0d6204ec05e3bd62be4b9f Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 00:18:28 +0100 Subject: [PATCH 02/13] docs: update links --- CONTRIBUTING.md | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0813c2e..f2c94b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Great see that you are interested in contributing to 'damast'. To make the process as smooth as possible for all involed parties, you will find some instructions below. In case there is some information missing, or you feel lost, do not -hesitate to ask or [open an issue](https://gitlab.com/simula-srl/damast/-/issues/new) +hesitate to ask or [open an issue](https://github.com/simula/damast/issues/new) on this project. ## Installation @@ -65,7 +65,7 @@ to run a particular test case (-k) and redirecting the output to stdout (-s). ## Merge Requests Please fork the repository and push your code change into a branch. -Then create a [new merge request](https://gitlab.com/simula-srl/damast/-/merge_requests/new) from this branch. +Then create a new merge request from this branch. Generally, read through the documentation first to understand the main ideas and some underlying assumptions of this project. If you intend to change some of the behavior, please try to validate by reading the documentation or opening an issue whether diff --git a/README.md b/README.md index 0af10e8..80ed031 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Create your venv and activate it: Clone the repo and install: ``` - git clone https://gitlab.com/simula-srl/damast + git clone https://github.com/simula/damast cd damast pip install -e ".[test,dev]" From a927384d9d55cc02e25cfe24b52c23457ce5f449 Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 00:30:14 +0100 Subject: [PATCH 03/13] github: fix pip install command --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89f3c72..3831cb1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,10 +19,10 @@ jobs: run: python -m pip install -U pip - name: Install package - run: pip install -e .[dev,test] + run: python -m pip install -e .[dev,test] - name: Install tox - run: python -m install tox + run: python -m pip install tox - name: Run tests run: tox -e py From 224be96c4fef75b9353487935bb3a4753e46636e Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 00:30:54 +0100 Subject: [PATCH 04/13] github: condition workflow trigger based on path filter --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3831cb1..abd4edf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test python package on: push: + paths: + - tests/** + - src/** pull_request: workflow_dispatch: From 9e3260492ee7875755b0c5c5dd6dfc2a2de081f7 Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 09:29:27 +0100 Subject: [PATCH 05/13] damast: name top level module Mainly to test the path filter trigger --- src/damast/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/damast/__init__.py b/src/damast/__init__.py index 5ecaef9..230e6de 100644 --- a/src/damast/__init__.py +++ b/src/damast/__init__.py @@ -1,3 +1,5 @@ +"""Main module""" + from damast import cli, core, data_handling, domains from damast.core import artifacts, describe, input, output From b0a5f35f8966c6a3b71f1d743b6a381752c46aa6 Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 11:45:34 +0100 Subject: [PATCH 06/13] github: add gh-pages workflow --- .github/workflows/gh-pages.yml | 59 ++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/gh-pages.yml diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 0000000..1b2e6d1 --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,59 @@ +name: gh-pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["2maz/workflows"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Single deploy job since we're just deploying + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install pip + run: python -m pip install -U pip + + - name: Install tox + run: python -m pip install tox + + - name: Generate page content + run: | + tox -e build_docs; + cd _build/html; + tar czf ../../github-pages.tar.gz . + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + From f29841ff6a8b92082945cf529d62f9d00a6be28b Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 11:52:09 +0100 Subject: [PATCH 07/13] Restrict astroid version to <3 Fix: autoapi/mappers/python/parser.py", line 240, in parse_module "doc": _prepare_docstring(node.doc or ""), AttributeError: 'Module' object has no attribute 'doc' --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a1c2682..e7f09c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ repository = "https://gitlab.com/simula-srl/damast" [project.optional-dependencies] dev = [ + "astroid<3", "flake8-gl-codeclimate", "isort", "jupyter-book", From da8a75ce994dc6f922af24a94faea2f6f7397f9f Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 12:01:38 +0100 Subject: [PATCH 08/13] github: allow action to package and upload the folder --- .github/workflows/gh-pages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 1b2e6d1..c24b28a 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,4 +1,4 @@ -name: gh-pages +name: Deploy github pages on: # Runs on pushes targeting the default branch @@ -44,14 +44,14 @@ jobs: - name: Generate page content run: | tox -e build_docs; - cd _build/html; - tar czf ../../github-pages.tar.gz . - name: Setup Pages uses: actions/configure-pages@v4 - name: Upload artifact uses: actions/upload-pages-artifact@v3 + with: + path: _build/html - name: Deploy to GitHub Pages id: deployment From d0187f6f3813ae4b1fce8b2910a932de92d4538e Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 12:10:55 +0100 Subject: [PATCH 09/13] github: gh-pages: update only on main branch --- .github/workflows/gh-pages.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index c24b28a..047c480 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -3,8 +3,7 @@ name: Deploy github pages on: # Runs on pushes targeting the default branch push: - branches: ["2maz/workflows"] - + branches: ["main"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 997230d6574eb540bf8e3874805ac0f48b548532 Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Tue, 23 Jan 2024 15:23:34 +0100 Subject: [PATCH 10/13] docs: Update documentation url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80ed031..e36a093 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # damast: Creation of reproducible data processing pipelines -Documentation at: https://simula-srl.gitlab.io/damast +Documentation at: https://simula.github.io/damast ## Installation and Development Setup From e7d8f245972c2c649cc977f5a57b96193737988d Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Wed, 24 Jan 2024 12:08:53 +0100 Subject: [PATCH 11/13] tests: fix missing dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7f09c7..811725d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,8 @@ dev = [ test = [ "coverage", "pytest", - "pytest-cov" + "pytest-cov", + "pydot" ] [project.scripts] From 54aca2784943278f0f3a1e273a90ad7a23b9b768 Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Wed, 24 Jan 2024 12:09:30 +0100 Subject: [PATCH 12/13] tests: add workaround for set mask issue --- src/damast/data_handling/exploration.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/damast/data_handling/exploration.py b/src/damast/data_handling/exploration.py index 760bcf0..6682aaf 100644 --- a/src/damast/data_handling/exploration.py +++ b/src/damast/data_handling/exploration.py @@ -35,8 +35,14 @@ def plot_lat_lon(*, :return: Path to the file """ - plt.scatter(x=df[longitude_name].evaluate(), - y=df[latitude_name].evaluate(), + longitude = df[longitude_name].evaluate() + latitude = df[latitude_name].evaluate() + + # .copy() to address ValueError: assignment destination is read-only + # raised in numpy/ma/core.py:3528: in mask + # self.__setmask__(value) + plt.scatter(x=longitude.copy(), + y=latitude.copy(), alpha=1) plt.xlim(-180, 180) plt.ylim(-90, 90) From e7f3e09a6b30639a49ccbd1851dbd7a6a0031eeb Mon Sep 17 00:00:00 2001 From: Thomas Roehr Date: Wed, 24 Jan 2024 12:25:04 +0100 Subject: [PATCH 13/13] github: update test workflow --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abd4edf..bfedefb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,5 +27,8 @@ jobs: - name: Install tox run: python -m pip install tox + - name: Install os dependencies + run: sudo apt install -y graphviz + - name: Run tests run: tox -e py