From d183f6eb389fef27163f0c43c20d77b8b7670a3b Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:14:59 +0100 Subject: [PATCH 1/7] Switch from Travis to GitHub Actions --- .github/workflows/ci.yaml | 35 ++++++++++++++++++++++++++++ .travis.yml | 48 --------------------------------------- 2 files changed, 35 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..4065387d --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,35 @@ +name: CI + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jobqueue: ["htcondor", "pbs", "sge", "slurm", "none"] + + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Setup + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_before_install + + - name: Install + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_install + + - name: Test + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_script + + - name: Cleanup + run: | + source ci/${{ matrix.jobqueue }}.sh + jobqueue_after_script diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 33dd6f5c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -language: python -dist: xenial - -notifications: - email: false - -services: - - docker -matrix: - include: - - python: "3.6" - env: - - JOBQUEUE=sge - - python: "3.6" - env: - # JOBQUEUE=none is for tests that do not need a cluster to run - - JOBQUEUE=none - - python: "3.6" - env: - - JOBQUEUE=pbs - - python: "3.6" - env: - - JOBQUEUE=slurm - - python: "3.6" - env: - - JOBQUEUE=htcondor - -before_install: - - set -e - - pwd - # Init jobqueue environment: load init and test methods - - source ci/${JOBQUEUE}.sh - - jobqueue_before_install - - set +e -install: - - set -e - - jobqueue_install - - set +e -script: - - set -e - - jobqueue_script - - set +e -after_script: - - jobqueue_after_script - - # TODO - # - pip install --no-cache-dir coveralls - # - coveralls From 285efebbfc8843729c42db97d52438dfbb22bdc8 Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:20:44 +0100 Subject: [PATCH 2/7] Set python to 3.6 --- ci/none.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/none.sh b/ci/none.sh index 3fc15dc4..77375630 100644 --- a/ci/none.sh +++ b/ci/none.sh @@ -4,7 +4,7 @@ function jobqueue_before_install { # Install miniconda ./ci/conda_setup.sh export PATH="$HOME/miniconda/bin:$PATH" - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION dask distributed flake8 black pytest pytest-asyncio + conda install --yes -c conda-forge python=3.6 dask distributed flake8 black pytest pytest-asyncio } function jobqueue_install { From c5abc47d712ed525208556e4f023d4297032462e Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:21:22 +0100 Subject: [PATCH 3/7] Remove interactive Docker execs --- ci/pbs.sh | 30 +++++++++++++++--------------- ci/pbs/start-pbs.sh | 2 +- ci/sge.sh | 10 +++++----- ci/sge/start-sge.sh | 2 +- ci/slurm.sh | 12 ++++++------ 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/ci/pbs.sh b/ci/pbs.sh index f633430d..51378396 100644 --- a/ci/pbs.sh +++ b/ci/pbs.sh @@ -10,30 +10,30 @@ function jobqueue_before_install { ./start-pbs.sh cd - - docker exec -it -u pbsuser pbs_master pbsnodes -a + docker exec -u pbsuser pbs_master pbsnodes -a docker ps -a docker images } function jobqueue_install { - docker exec -it pbs_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec pbs_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec -it -u pbsuser pbs_master /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E pbs" + docker exec -u pbsuser pbs_master /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E pbs" } function jobqueue_after_script { - docker exec -it -u pbsuser pbs_master qstat -fx - docker exec -it pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*' - docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_logs/*' - docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*' - docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*' - docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*' - docker exec -it pbs_slave_1 bash -c 'cat /tmp/*.e*' - docker exec -it pbs_slave_1 bash -c 'cat /tmp/*.o*' - docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*' - docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*' - docker exec -it pbs_slave_2 bash -c 'cat /tmp/*.e*' - docker exec -it pbs_slave_2 bash -c 'cat /tmp/*.o*' + docker exec -u pbsuser pbs_master qstat -fx + docker exec pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*' + docker exec pbs_master bash -c 'cat /var/spool/pbs/server_logs/*' + docker exec pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*' + docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*' + docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*' + docker exec pbs_slave_1 bash -c 'cat /tmp/*.e*' + docker exec pbs_slave_1 bash -c 'cat /tmp/*.o*' + docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*' + docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*' + docker exec pbs_slave_2 bash -c 'cat /tmp/*.e*' + docker exec pbs_slave_2 bash -c 'cat /tmp/*.o*' } diff --git a/ci/pbs/start-pbs.sh b/ci/pbs/start-pbs.sh index 4162f11a..edbe50bf 100755 --- a/ci/pbs/start-pbs.sh +++ b/ci/pbs/start-pbs.sh @@ -1,7 +1,7 @@ #!/bin/bash docker-compose up -d -while [ `docker exec -it -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ] +while [ `docker exec -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ] do echo "Waiting for PBS slave nodes to become available"; sleep 2 diff --git a/ci/sge.sh b/ci/sge.sh index d85c1609..c581e7e3 100644 --- a/ci/sge.sh +++ b/ci/sge.sh @@ -15,15 +15,15 @@ function jobqueue_before_install { } function jobqueue_install { - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec sge_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pytest dask_jobqueue --verbose -s -E sge" + docker exec sge_master /bin/bash -c "cd /dask-jobqueue; pytest dask_jobqueue --verbose -s -E sge" } function jobqueue_after_script { - docker exec -it sge_master bash -c 'cat /tmp/sge*' - docker exec -it slave_one bash -c 'cat /tmp/exec*' - docker exec -it slave_two bash -c 'cat /tmp/exec*' + docker exec sge_master bash -c 'cat /tmp/sge*' + docker exec slave_one bash -c 'cat /tmp/exec*' + docker exec slave_two bash -c 'cat /tmp/exec*' } diff --git a/ci/sge/start-sge.sh b/ci/sge/start-sge.sh index c5559828..e77282ab 100755 --- a/ci/sge/start-sge.sh +++ b/ci/sge/start-sge.sh @@ -1,7 +1,7 @@ #!/bin/bash docker-compose up -d -while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] +while [ `docker exec sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] do echo "Waiting for SGE slots to become available"; sleep 1 diff --git a/ci/slurm.sh b/ci/slurm.sh index 49ea1315..76320394 100644 --- a/ci/slurm.sh +++ b/ci/slurm.sh @@ -19,21 +19,21 @@ function show_network_interfaces { for c in slurmctld c1 c2; do echo '------------------------------------------------------------' echo docker container: $c - docker exec -it $c python -c 'import psutil; print(psutil.net_if_addrs().keys())' + docker exec $c python -c 'import psutil; print(psutil.net_if_addrs().keys())' echo '------------------------------------------------------------' done } function jobqueue_install { - docker exec -it slurmctld /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec slurmctld /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec -it slurmctld /bin/bash -c "pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s" + docker exec slurmctld /bin/bash -c "pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s" } function jobqueue_after_script { - docker exec -it slurmctld bash -c 'sinfo' - docker exec -it slurmctld bash -c 'squeue' - docker exec -it slurmctld bash -c 'sacct -l' + docker exec slurmctld bash -c 'sinfo' + docker exec slurmctld bash -c 'squeue' + docker exec slurmctld bash -c 'sacct -l' } From dec99a8f48278d8e9b2c61a324097ab914c135be Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:26:01 +0100 Subject: [PATCH 4/7] Install conda with action step --- .github/workflows/ci.yaml | 9 +++++++++ ci/environment.yml | 12 ++++++++++++ ci/none.sh | 4 ---- 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 ci/environment.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4065387d..0d6fef9d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,6 +14,15 @@ jobs: - name: Checkout source uses: actions/checkout@v2 + - name: Setup Conda Environment + uses: goanpeca/setup-miniconda@v1 + with: + miniconda-version: "latest" + python-version: 3.6 + environment-file: ci/environment.yml + activate-environment: dask-jobqueue + auto-activate-base: false + - name: Setup run: | source ci/${{ matrix.jobqueue }}.sh diff --git a/ci/environment.yml b/ci/environment.yml new file mode 100644 index 00000000..8734d9c5 --- /dev/null +++ b/ci/environment.yml @@ -0,0 +1,12 @@ +name: dask-jobqueue +channels: + - conda-forge + - defaults +dependencies: + - python=3.6 + - dask + - distributed + - flake8 + - black + - pytest + - pytest-asyncio diff --git a/ci/none.sh b/ci/none.sh index 77375630..03d7b31b 100644 --- a/ci/none.sh +++ b/ci/none.sh @@ -1,10 +1,6 @@ #!/usr/bin/env bash function jobqueue_before_install { - # Install miniconda - ./ci/conda_setup.sh - export PATH="$HOME/miniconda/bin:$PATH" - conda install --yes -c conda-forge python=3.6 dask distributed flake8 black pytest pytest-asyncio } function jobqueue_install { From 37a4ac452c10b89eaedf16575dbd847302a5b33a Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:30:02 +0100 Subject: [PATCH 5/7] Functions need something inside them --- ci/none.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/none.sh b/ci/none.sh index 03d7b31b..2cb8a9f8 100644 --- a/ci/none.sh +++ b/ci/none.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash function jobqueue_before_install { + true # Pass } function jobqueue_install { From a91072fb018af54e60f0c689f0d6fc0a8fdd43e4 Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:44:03 +0100 Subject: [PATCH 6/7] Ignore failed cats --- ci/pbs.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/pbs.sh b/ci/pbs.sh index 51378396..c8dbb249 100644 --- a/ci/pbs.sh +++ b/ci/pbs.sh @@ -25,15 +25,15 @@ function jobqueue_script { function jobqueue_after_script { docker exec -u pbsuser pbs_master qstat -fx - docker exec pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*' - docker exec pbs_master bash -c 'cat /var/spool/pbs/server_logs/*' - docker exec pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*' - docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*' - docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*' - docker exec pbs_slave_1 bash -c 'cat /tmp/*.e*' - docker exec pbs_slave_1 bash -c 'cat /tmp/*.o*' - docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*' - docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*' - docker exec pbs_slave_2 bash -c 'cat /tmp/*.e*' - docker exec pbs_slave_2 bash -c 'cat /tmp/*.o*' + docker exec pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*|| true' + docker exec pbs_master bash -c 'cat /var/spool/pbs/server_logs/*|| true' + docker exec pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*|| true' + docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*|| true' + docker exec pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*|| true' + docker exec pbs_slave_1 bash -c 'cat /tmp/*.e*|| true' + docker exec pbs_slave_1 bash -c 'cat /tmp/*.o*|| true' + docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*|| true' + docker exec pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*|| true' + docker exec pbs_slave_2 bash -c 'cat /tmp/*.e*|| true' + docker exec pbs_slave_2 bash -c 'cat /tmp/*.o*|| true' } From 4cc05ede14681d2e6533c4b69c6e46ab20e762b1 Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Wed, 27 May 2020 15:46:11 +0100 Subject: [PATCH 7/7] Explicitly set shell --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0d6fef9d..337d891e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,21 +24,25 @@ jobs: auto-activate-base: false - name: Setup + shell: bash -l {0} run: | source ci/${{ matrix.jobqueue }}.sh jobqueue_before_install - name: Install + shell: bash -l {0} run: | source ci/${{ matrix.jobqueue }}.sh jobqueue_install - name: Test + shell: bash -l {0} run: | source ci/${{ matrix.jobqueue }}.sh jobqueue_script - name: Cleanup + shell: bash -l {0} run: | source ci/${{ matrix.jobqueue }}.sh jobqueue_after_script