diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..337d891e --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,48 @@ +name: CI + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jobqueue: ["htcondor", "pbs", "sge", "slurm", "none"] + + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Setup Conda Environment + uses: goanpeca/setup-miniconda@v1 + with: + miniconda-version: "latest" + python-version: 3.6 + environment-file: ci/environment.yml + activate-environment: dask-jobqueue + auto-activate-base: false + + - name: Setup + shell: bash -l {0} + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_before_install + + - name: Install + shell: bash -l {0} + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_install + + - name: Test + shell: bash -l {0} + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_script + + - name: Cleanup + shell: bash -l {0} + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_after_script diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 33dd6f5c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -language: python -dist: xenial - -notifications: - email: false - -services: - - docker -matrix: - include: - - python: "3.6" - env: - - JOBQUEUE=sge - - python: "3.6" - env: - # JOBQUEUE=none is for tests that do not need a cluster to run - - JOBQUEUE=none - - python: "3.6" - env: - - JOBQUEUE=pbs - - python: "3.6" - env: - - JOBQUEUE=slurm - - python: "3.6" - env: - - JOBQUEUE=htcondor - -before_install: - - set -e - - pwd - # Init jobqueue environment: load init and test methods - - source ci/${JOBQUEUE}.sh - - jobqueue_before_install - - set +e -install: - - set -e - - jobqueue_install - - set +e -script: - - set -e - - jobqueue_script - - set +e -after_script: - - jobqueue_after_script - - # TODO - # - pip install --no-cache-dir coveralls - # - coveralls diff --git a/ci/environment.yml b/ci/environment.yml new file mode 100644 index 00000000..8734d9c5 --- /dev/null +++ b/ci/environment.yml @@ -0,0 +1,12 @@ +name: dask-jobqueue +channels: + - conda-forge + - defaults +dependencies: + - python=3.6 + - dask + - distributed + - flake8 + - black + - pytest + - pytest-asyncio diff --git a/ci/none.sh b/ci/none.sh index 3fc15dc4..2cb8a9f8 100644 --- a/ci/none.sh +++ b/ci/none.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash function jobqueue_before_install { - # Install miniconda - ./ci/conda_setup.sh - export PATH="$HOME/miniconda/bin:$PATH" - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION dask distributed flake8 black pytest pytest-asyncio + true # Pass } function jobqueue_install { diff --git a/ci/pbs.sh b/ci/pbs.sh index f633430d..c8dbb249 100644 --- a/ci/pbs.sh +++ b/ci/pbs.sh @@ -10,30 +10,30 @@ function jobqueue_before_install { ./start-pbs.sh cd - - docker exec -it -u pbsuser pbs_master pbsnodes -a + docker exec -u pbsuser pbs_master pbsnodes -a docker ps -a docker images } function jobqueue_install { - docker exec -it pbs_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec pbs_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec -it -u pbsuser pbs_master /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E pbs" + docker exec -u pbsuser pbs_master /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E pbs" } function jobqueue_after_script { - docker exec -it -u pbsuser pbs_master qstat -fx - docker exec -it pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*' - docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_logs/*' - docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*' - docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*' - docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*' - docker exec -it pbs_slave_1 bash -c 'cat /tmp/*.e*' - docker exec -it pbs_slave_1 bash -c 'cat /tmp/*.o*' - docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*' - docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*' - docker exec -it pbs_slave_2 bash -c 'cat /tmp/*.e*' - docker exec -it pbs_slave_2 bash -c 'cat /tmp/*.o*' + docker exec -u pbsuser pbs_master qstat -fx + docker exec pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*|| true' + docker exec pbs_master bash -c 'cat /var/spool/pbs/server_logs/*|| true' + docker exec pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*|| true' + docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*|| true' + docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*|| true' + docker exec pbs_slave_1 bash -c 'cat /tmp/*.e*|| true' + docker exec pbs_slave_1 bash -c 'cat /tmp/*.o*|| true' + docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*|| true' + docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*|| true' + docker exec pbs_slave_2 bash -c 'cat /tmp/*.e*|| true' + docker exec pbs_slave_2 bash -c 'cat /tmp/*.o*|| true' } diff --git a/ci/pbs/start-pbs.sh b/ci/pbs/start-pbs.sh index 4162f11a..edbe50bf 100755 --- a/ci/pbs/start-pbs.sh +++ b/ci/pbs/start-pbs.sh @@ -1,7 +1,7 @@ #!/bin/bash docker-compose up -d -while [ `docker exec -it -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ] +while [ `docker exec -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ] do echo "Waiting for PBS slave nodes to become available"; sleep 2 diff --git a/ci/sge.sh b/ci/sge.sh index d85c1609..c581e7e3 100644 --- a/ci/sge.sh +++ b/ci/sge.sh @@ -15,15 +15,15 @@ function jobqueue_before_install { } function jobqueue_install { - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec sge_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pytest dask_jobqueue --verbose -s -E sge" + docker exec sge_master /bin/bash -c "cd /dask-jobqueue; pytest dask_jobqueue --verbose -s -E sge" } function jobqueue_after_script { - docker exec -it sge_master bash -c 'cat /tmp/sge*' - docker exec -it slave_one bash -c 'cat /tmp/exec*' - docker exec -it slave_two bash -c 'cat /tmp/exec*' + docker exec sge_master bash -c 'cat /tmp/sge*' + docker exec slave_one bash -c 'cat /tmp/exec*' + docker exec slave_two bash -c 'cat /tmp/exec*' } diff --git a/ci/sge/start-sge.sh b/ci/sge/start-sge.sh index c5559828..e77282ab 100755 --- a/ci/sge/start-sge.sh +++ b/ci/sge/start-sge.sh @@ -1,7 +1,7 @@ #!/bin/bash docker-compose up -d -while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] +while [ `docker exec sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] do echo "Waiting for SGE slots to become available"; sleep 1 diff --git a/ci/slurm.sh b/ci/slurm.sh index 49ea1315..76320394 100644 --- a/ci/slurm.sh +++ b/ci/slurm.sh @@ -19,21 +19,21 @@ function show_network_interfaces { for c in slurmctld c1 c2; do echo '------------------------------------------------------------' echo docker container: $c - docker exec -it $c python -c 'import psutil; print(psutil.net_if_addrs().keys())' + docker exec $c python -c 'import psutil; print(psutil.net_if_addrs().keys())' echo '------------------------------------------------------------' done } function jobqueue_install { - docker exec -it slurmctld /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec slurmctld /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec -it slurmctld /bin/bash -c "pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s" + docker exec slurmctld /bin/bash -c "pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s" } function jobqueue_after_script { - docker exec -it slurmctld bash -c 'sinfo' - docker exec -it slurmctld bash -c 'squeue' - docker exec -it slurmctld bash -c 'sacct -l' + docker exec slurmctld bash -c 'sinfo' + docker exec slurmctld bash -c 'squeue' + docker exec slurmctld bash -c 'sacct -l' }