From d9582d353df392e2bdcbf39bb3b5a7083098c490 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 12 Mar 2018 10:34:54 -0700 Subject: [PATCH 1/9] initial travis setup (will require debugging) --- .travis.yml | 53 +++++++++++++++++++++++++++++++++++++++ ci/conda_setup.sh | 13 ++++++++++ ci/sge/Dockerfile-master | 29 +++++++++++++++++++++ ci/sge/Dockerfile-slave | 21 ++++++++++++++++ ci/sge/docker-compose.yml | 44 ++++++++++++++++++++++++++++++++ ci/sge/hosts.txt | 2 ++ ci/sge/queue.txt | 50 ++++++++++++++++++++++++++++++++++++ ci/sge/run_master.sh | 23 +++++++++++++++++ ci/sge/run_slave.sh | 10 ++++++++ ci/sge/scheduler.txt | 35 ++++++++++++++++++++++++++ ci/sge/setup-master.sh | 24 ++++++++++++++++++ ci/sge/setup-slave.sh | 15 +++++++++++ ci/sge/start-sge.sh | 8 ++++++ 13 files changed, 327 insertions(+) create mode 100644 .travis.yml create mode 100644 ci/conda_setup.sh create mode 100644 ci/sge/Dockerfile-master create mode 100644 ci/sge/Dockerfile-slave create mode 100644 ci/sge/docker-compose.yml create mode 100644 ci/sge/hosts.txt create mode 100644 ci/sge/queue.txt create mode 100644 ci/sge/run_master.sh create mode 100644 ci/sge/run_slave.sh create mode 100644 ci/sge/scheduler.txt create mode 100755 ci/sge/setup-master.sh create mode 100755 ci/sge/setup-slave.sh create mode 100644 ci/sge/start-sge.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..83d7d0a5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,53 @@ +language: python +dist: trusty +sudo: required + +notifications: + email: false + +services: + - docker + +matrix: + include: + - python: "2.7" + env: OS=ubuntu-14.04 + - python: "3.6" + env: OS=ubuntu-14.04 + +env: + global: + - DOCKER_COMPOSE_VERSION=1.6.0 + + +before_install: + - pwd + - docker version + - docker-compose version + + # Install miniconda + - ./ci/conda_setup.sh + - export PATH="$HOME/miniconda/bin:$PATH" + - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION distributed + + # Start containers + - ./ci/sge/start-sge.sh + - docker ps -a + - docker images + +install: + - which python + - pip install --no-deps -e . + +script: + - flake8 -j auto dask_jobqueue + - py.test dask_jobqueue --verbose # run tests (skip queue tests) + - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pip install --no-cache-dir ." + - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose" + +after_success: + - docker exec -it sge_master bash -c 'cat /tmp/sge*' + - docker exec -it slave_one bash -c 'cat /tmp/exec*' + - docker exec -it slave_two bash -c 'cat /tmp/exec*' + # - pip install --no-cache-dir coveralls + # - coveralls diff --git a/ci/conda_setup.sh b/ci/conda_setup.sh new file mode 100644 index 00000000..58131b6f --- /dev/null +++ b/ci/conda_setup.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Install miniconda +wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh +bash ~/miniconda.sh -b -p $HOME/miniconda +export PATH="$HOME/miniconda/bin:$PATH" +conda update conda --yes +conda clean -tipy +conda config --set always_yes yes --set changeps1 no +conda --version diff --git a/ci/sge/Dockerfile-master b/ci/sge/Dockerfile-master new file mode 100644 index 00000000..745eddcb --- /dev/null +++ b/ci/sge/Dockerfile-master @@ -0,0 +1,29 @@ +FROM ubuntu:14.04 + +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install curl bzip2 git gcc -y --fix-missing + +RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash miniconda.sh -f -b -p /opt/anaconda && \ + /opt/anaconda/bin/conda clean -tipy && \ + rm -f miniconda.sh +ENV PATH /opt/anaconda/bin:$PATH +RUN conda install -n root conda=4.4.11 && conda clean -tipy +RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip psutil && conda clean -tipy +RUN pip install --no-cache-dir drmaa +RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade + +COPY ./*.sh / +COPY ./*.txt / +RUN bash ./setup-master.sh + +# expose ports +EXPOSE 8000 +EXPOSE 6444 +EXPOSE 6445 +EXPOSE 6446 + +ENV DRMAA_LIBRARY_PATH /usr/lib/gridengine-drmaa/lib/libdrmaa.so +ENV SGE_ROOT /var/lib/gridengine/ +ENV SGE_CELL default diff --git a/ci/sge/Dockerfile-slave b/ci/sge/Dockerfile-slave new file mode 100644 index 00000000..8beaaf5e --- /dev/null +++ b/ci/sge/Dockerfile-slave @@ -0,0 +1,21 @@ +FROM ubuntu:14.04 + +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install curl bzip2 git gcc -y --fix-missing + +RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash miniconda.sh -f -b -p /opt/anaconda && \ + /opt/anaconda/bin/conda clean -tipy && \ + rm -f miniconda.sh +ENV PATH /opt/anaconda/bin:$PATH +RUN conda install -n root conda=4.4.11 && conda clean -tipy +RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip psutil && conda clean -tipy +RUN pip install --no-cache-dir drmaa +RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade + +COPY ./setup-slave.sh / +COPY ./run-slave.sh / +RUN bash ./setup-slave.sh + +CMD python -m SimpleHTTPServer diff --git a/ci/sge/docker-compose.yml b/ci/sge/docker-compose.yml new file mode 100644 index 00000000..dcffca11 --- /dev/null +++ b/ci/sge/docker-compose.yml @@ -0,0 +1,44 @@ +version: "2" + +services: + + master: + build: + context: . + dockerfile: Dockerfile-master + container_name: sge_master + hostname: sge_master + #network_mode: host + volumes: + - .:/dask-jobqueue + command: bash /run-master.sh + + slave-one: + build: + context: . + dockerfile: Dockerfile-slave + container_name: slave_one + hostname: slave_one + #network_mode: host + volumes: + - .:/dask-jobqueue + command: bash /run-slave.sh + links: + - "master:sge_master" + depends_on: + - master + + slave-two: + build: + context: . + dockerfile: Dockerfile-slave + container_name: slave_two + hostname: slave_two + #network_mode: host + volumes: + - .:/dask-jobqueue + command: bash /run-slave.sh + links: + - "master:sge_master" + depends_on: + - master diff --git a/ci/sge/hosts.txt b/ci/sge/hosts.txt new file mode 100644 index 00000000..5aee646d --- /dev/null +++ b/ci/sge/hosts.txt @@ -0,0 +1,2 @@ +group_name @allhosts +hostlist NONE diff --git a/ci/sge/queue.txt b/ci/sge/queue.txt new file mode 100644 index 00000000..91ee22bd --- /dev/null +++ b/ci/sge/queue.txt @@ -0,0 +1,50 @@ +qname dask.q +hostlist @allhosts +seq_no 0 +load_thresholds NONE +suspend_thresholds NONE +nsuspend 1 +suspend_interval 00:00:01 +priority 0 +min_cpu_interval 00:00:01 +processors UNDEFINED +qtype BATCH INTERACTIVE +ckpt_list NONE +pe_list make +rerun FALSE +slots 2 +tmpdir /tmp +shell /bin/csh +prolog NONE +epilog NONE +shell_start_mode posix_compliant +starter_method NONE +suspend_method NONE +resume_method NONE +terminate_method NONE +notify 00:00:01 +owner_list NONE +user_lists NONE +xuser_lists NONE +subordinate_list NONE +complex_values NONE +projects NONE +xprojects NONE +calendar NONE +initial_state default +s_rt INFINITY +h_rt INFINITY +s_cpu INFINITY +h_cpu INFINITY +s_fsize INFINITY +h_fsize INFINITY +s_data INFINITY +h_data INFINITY +s_stack INFINITY +h_stack INFINITY +s_core INFINITY +h_core INFINITY +s_rss INFINITY +h_rss INFINITY +s_vmem INFINITY +h_vmem INFINITY diff --git a/ci/sge/run_master.sh b/ci/sge/run_master.sh new file mode 100644 index 00000000..2282ca50 --- /dev/null +++ b/ci/sge/run_master.sh @@ -0,0 +1,23 @@ +#!/bin/bash + + +# start sge +sudo service gridengine-master restart + +while ! ping -c1 slave_one &>/dev/null; do :; done + +qconf -Msconf /scheduler.txt +qconf -Ahgrp /hosts.txt +qconf -Aq /queue.txt + +qconf -ah slave_one +qconf -ah slave_two +qconf -ah slave_three + +qconf -as $HOSTNAME +bash add_worker.sh dask.q slave_one 4 +bash add_worker.sh dask.q slave_two 4 + +sudo service gridengine-master restart + +python -m http.server 8888 diff --git a/ci/sge/run_slave.sh b/ci/sge/run_slave.sh new file mode 100644 index 00000000..f27121e3 --- /dev/null +++ b/ci/sge/run_slave.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# start sge +sudo service gridengine-exec restart + +sleep 4 + +sudo service gridengine-exec restart + +python -m http.server 8888 diff --git a/ci/sge/scheduler.txt b/ci/sge/scheduler.txt new file mode 100644 index 00000000..5718eba3 --- /dev/null +++ b/ci/sge/scheduler.txt @@ -0,0 +1,35 @@ +algorithm default +schedule_interval 0:0:1 +maxujobs 0 +queue_sort_method load +job_load_adjustments np_load_avg=0.50 +load_adjustment_decay_time 0:7:30 +load_formula np_load_avg +schedd_job_info true +flush_submit_sec 0 +flush_finish_sec 0 +params none +reprioritize_interval 0:0:0 +halftime 168 +usage_weight_list cpu=1.000000,mem=0.000000,io=0.000000 +compensation_factor 5.000000 +weight_user 0.250000 +weight_project 0.250000 +weight_department 0.250000 +weight_job 0.250000 +weight_tickets_functional 0 +weight_tickets_share 0 +share_override_tickets TRUE +share_functional_shares TRUE +max_functional_jobs_to_schedule 200 +report_pjob_tickets TRUE +max_pending_tasks_per_job 50 +halflife_decay_list none +policy_hierarchy OFS +weight_ticket 0.500000 +weight_waiting_time 0.278000 +weight_deadline 3600000.000000 +weight_urgency 0.500000 +weight_priority 0.000000 +max_reservation 0 +default_duration INFINITY diff --git a/ci/sge/setup-master.sh b/ci/sge/setup-master.sh new file mode 100755 index 00000000..1ea78523 --- /dev/null +++ b/ci/sge/setup-master.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Configure the master hostname for Grid Engine +echo "gridengine-master shared/gridenginemaster string $HOSTNAME" | sudo debconf-set-selections +echo "gridengine-master shared/gridenginecell string default" | sudo debconf-set-selections +echo "gridengine-master shared/gridengineconfig boolean false" | sudo debconf-set-selections +echo "gridengine-common shared/gridenginemaster string $HOSTNAME" | sudo debconf-set-selections +echo "gridengine-common shared/gridenginecell string default" | sudo debconf-set-selections +echo "gridengine-common shared/gridengineconfig boolean false" | sudo debconf-set-selections +echo "gridengine-client shared/gridenginemaster string $HOSTNAME" | sudo debconf-set-selections +echo "gridengine-client shared/gridenginecell string default" | sudo debconf-set-selections +echo "gridengine-client shared/gridengineconfig boolean false" | sudo debconf-set-selections +# Postfix mail server is also installed as a dependency +echo "postfix postfix/main_mailer_type select No configuration" | sudo debconf-set-selections + +# Install Grid Engine +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y gridengine-master gridengine-client gridengine-drmaa-dev -qq + +# Set up Grid Engine +sudo -u sgeadmin /usr/share/gridengine/scripts/init_cluster /var/lib/gridengine default /var/spool/gridengine/spooldb sgeadmin +sudo service gridengine-master restart + +# Disable Postfix +sudo service postfix stop +sudo update-rc.d postfix disable diff --git a/ci/sge/setup-slave.sh b/ci/sge/setup-slave.sh new file mode 100755 index 00000000..58d9d873 --- /dev/null +++ b/ci/sge/setup-slave.sh @@ -0,0 +1,15 @@ +#!/bin/bash +export MASTER_HOSTNAME=sge_master +echo "gridengine-common shared/gridenginemaster string $MASTER_HOSTNAME" | sudo debconf-set-selections +echo "gridengine-common shared/gridenginecell string default" | sudo debconf-set-selections +echo "gridengine-common shared/gridengineconfig boolean false" | sudo debconf-set-selections +echo "gridengine-client shared/gridenginemaster string $MASTER_HOSTNAME" | sudo debconf-set-selections +echo "gridengine-client shared/gridenginecell string default" | sudo debconf-set-selections +echo "gridengine-client shared/gridengineconfig boolean false" | sudo debconf-set-selections +echo "postfix postfix/main_mailer_type select No configuration" | sudo debconf-set-selections + +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y gridengine-exec gridengine-client gridengine-drmaa-dev -qq + +sudo service postfix stop +sudo update-rc.d postfix disable +echo $MASTER_HOSTNAME | sudo tee /var/lib/gridengine/default/common/act_qmaster diff --git a/ci/sge/start-sge.sh b/ci/sge/start-sge.sh new file mode 100644 index 00000000..b8eb3df1 --- /dev/null +++ b/ci/sge/start-sge.sh @@ -0,0 +1,8 @@ +#!/bin/bash +docker-compose up -d +while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] + do + echo "Waiting for SGE slots to become available"; + sleep 1 + done +echo "SGE properly configured" From b7fa397dfbc8fda2982aa1ac12f01d0f2a4c26f5 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 12 Mar 2018 10:39:00 -0700 Subject: [PATCH 2/9] execute privs --- ci/conda_setup.sh | 0 ci/sge/start-sge.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/conda_setup.sh mode change 100644 => 100755 ci/sge/start-sge.sh diff --git a/ci/conda_setup.sh b/ci/conda_setup.sh old mode 100644 new mode 100755 diff --git a/ci/sge/start-sge.sh b/ci/sge/start-sge.sh old mode 100644 new mode 100755 From 0de67296b32f3648c99bc91d30f9a56fb3d558e9 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 12 Mar 2018 10:48:23 -0700 Subject: [PATCH 3/9] getting the directory tree right. --- ci/sge/start-sge.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/sge/start-sge.sh b/ci/sge/start-sge.sh index b8eb3df1..e4887687 100755 --- a/ci/sge/start-sge.sh +++ b/ci/sge/start-sge.sh @@ -1,4 +1,5 @@ #!/bin/bash +cd ci/sge docker-compose up -d while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] do @@ -6,3 +7,4 @@ while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] sleep 1 done echo "SGE properly configured" +cd - From 365b3e6c3f2e92231f85e2407171f1c74ee417ee Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 12 Mar 2018 11:05:49 -0700 Subject: [PATCH 4/9] fix script copy --- ci/sge/Dockerfile-slave | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sge/Dockerfile-slave b/ci/sge/Dockerfile-slave index 8beaaf5e..509c38cc 100644 --- a/ci/sge/Dockerfile-slave +++ b/ci/sge/Dockerfile-slave @@ -15,7 +15,7 @@ RUN pip install --no-cache-dir drmaa RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade COPY ./setup-slave.sh / -COPY ./run-slave.sh / +COPY ./*.sh / RUN bash ./setup-slave.sh CMD python -m SimpleHTTPServer From 021baf8dcc3b3aeadf86d1c57a209cd468baed7a Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Fri, 23 Mar 2018 17:03:47 -0700 Subject: [PATCH 5/9] more config for sge ci --- .travis.yml | 2 +- ci/sge/Dockerfile-master | 1 + ci/sge/Dockerfile-slave | 1 + ci/sge/add_worker.sh | 26 ++++++++++++++++++++++++++ ci/sge/run-master.sh | 23 +++++++++++++++++++++++ ci/sge/run-slave.sh | 10 ++++++++++ ci/sge/run_master.sh | 2 ++ ci/sge/run_slave.sh | 0 ci/sge/start-sge.sh | 3 +-- 9 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 ci/sge/add_worker.sh create mode 100755 ci/sge/run-master.sh create mode 100755 ci/sge/run-slave.sh mode change 100644 => 100755 ci/sge/run_master.sh mode change 100644 => 100755 ci/sge/run_slave.sh diff --git a/.travis.yml b/.travis.yml index 83d7d0a5..2af3a790 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_install: - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION distributed # Start containers - - ./ci/sge/start-sge.sh + - cd ./ci/sge; ./start-sge.sh; cd - - docker ps -a - docker images diff --git a/ci/sge/Dockerfile-master b/ci/sge/Dockerfile-master index 745eddcb..163f7811 100644 --- a/ci/sge/Dockerfile-master +++ b/ci/sge/Dockerfile-master @@ -14,6 +14,7 @@ RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip p RUN pip install --no-cache-dir drmaa RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade +COPY ../../../dask-jobqueue / COPY ./*.sh / COPY ./*.txt / RUN bash ./setup-master.sh diff --git a/ci/sge/Dockerfile-slave b/ci/sge/Dockerfile-slave index 509c38cc..db64301b 100644 --- a/ci/sge/Dockerfile-slave +++ b/ci/sge/Dockerfile-slave @@ -14,6 +14,7 @@ RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip p RUN pip install --no-cache-dir drmaa RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade +COPY ../../../dask-jobqueue / COPY ./setup-slave.sh / COPY ./*.sh / RUN bash ./setup-slave.sh diff --git a/ci/sge/add_worker.sh b/ci/sge/add_worker.sh new file mode 100644 index 00000000..d48c6203 --- /dev/null +++ b/ci/sge/add_worker.sh @@ -0,0 +1,26 @@ +#`/bin/bash + +#!/bin/bash + +QUEUE=$1 +HOSTNAME=$2 +SLOTS=$3 + +# add to the execution host list +TMPFILE=/tmp/sge.hostname-$HOSTNAME +echo -e "hostname $HOSTNAME\nload_scaling NONE\ncomplex_values NONE\nuser_lists NONE\nxuser_lists NONE\nprojects NONE\nxprojects NONE\nusage_scaling NONE\nreport_variables NONE" > $TMPFILE +qconf -Ae $TMPFILE +rm $TMPFILE + +# add to the all hosts list +qconf -aattr hostgroup hostlist $HOSTNAME @allhosts + +# enable the host for the queue, in case it was disabled and not removed +qmod -e $QUEUE@$HOSTNAME + +# Add memory resource +qconf -mattr exechost complex_values h_vmem=100G $HOSTNAME + +if [ "$SLOTS" ]; then + qconf -aattr queue slots "[$HOSTNAME=$SLOTS]" $QUEUE +fi diff --git a/ci/sge/run-master.sh b/ci/sge/run-master.sh new file mode 100755 index 00000000..2282ca50 --- /dev/null +++ b/ci/sge/run-master.sh @@ -0,0 +1,23 @@ +#!/bin/bash + + +# start sge +sudo service gridengine-master restart + +while ! ping -c1 slave_one &>/dev/null; do :; done + +qconf -Msconf /scheduler.txt +qconf -Ahgrp /hosts.txt +qconf -Aq /queue.txt + +qconf -ah slave_one +qconf -ah slave_two +qconf -ah slave_three + +qconf -as $HOSTNAME +bash add_worker.sh dask.q slave_one 4 +bash add_worker.sh dask.q slave_two 4 + +sudo service gridengine-master restart + +python -m http.server 8888 diff --git a/ci/sge/run-slave.sh b/ci/sge/run-slave.sh new file mode 100755 index 00000000..f27121e3 --- /dev/null +++ b/ci/sge/run-slave.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# start sge +sudo service gridengine-exec restart + +sleep 4 + +sudo service gridengine-exec restart + +python -m http.server 8888 diff --git a/ci/sge/run_master.sh b/ci/sge/run_master.sh old mode 100644 new mode 100755 index 2282ca50..c6024621 --- a/ci/sge/run_master.sh +++ b/ci/sge/run_master.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -x +set -e # start sge sudo service gridengine-master restart diff --git a/ci/sge/run_slave.sh b/ci/sge/run_slave.sh old mode 100644 new mode 100755 diff --git a/ci/sge/start-sge.sh b/ci/sge/start-sge.sh index e4887687..c5559828 100755 --- a/ci/sge/start-sge.sh +++ b/ci/sge/start-sge.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd ci/sge + docker-compose up -d while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] do @@ -7,4 +7,3 @@ while [ `docker exec -it sge_master qhost | grep lx26-amd64 | wc -l` -ne 2 ] sleep 1 done echo "SGE properly configured" -cd - From f424d59774ea3d91cef0b6f72d0564604c5992e0 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Fri, 23 Mar 2018 17:19:11 -0700 Subject: [PATCH 6/9] remove copy --- ci/sge/Dockerfile-master | 1 - ci/sge/Dockerfile-slave | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/sge/Dockerfile-master b/ci/sge/Dockerfile-master index 163f7811..745eddcb 100644 --- a/ci/sge/Dockerfile-master +++ b/ci/sge/Dockerfile-master @@ -14,7 +14,6 @@ RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip p RUN pip install --no-cache-dir drmaa RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade -COPY ../../../dask-jobqueue / COPY ./*.sh / COPY ./*.txt / RUN bash ./setup-master.sh diff --git a/ci/sge/Dockerfile-slave b/ci/sge/Dockerfile-slave index db64301b..509c38cc 100644 --- a/ci/sge/Dockerfile-slave +++ b/ci/sge/Dockerfile-slave @@ -14,7 +14,6 @@ RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip p RUN pip install --no-cache-dir drmaa RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade -COPY ../../../dask-jobqueue / COPY ./setup-slave.sh / COPY ./*.sh / RUN bash ./setup-slave.sh From 2d8ffbcd47c45eb245094bd784b52bf09a25b73e Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Fri, 23 Mar 2018 20:30:35 -0700 Subject: [PATCH 7/9] more ci --- .travis.yml | 35 ++++++++++------------- ci/sge.sh | 28 ++++++++++++++++++ conftest.py | 22 ++++++++++++++ dask_jobqueue/tests/test_jobqueue_core.py | 4 +++ dask_jobqueue/tests/test_pbs.py | 10 ++++--- dask_jobqueue/tests/test_sge.py | 9 ++++++ dask_jobqueue/tests/test_slurm.py | 16 ++++++----- 7 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 ci/sge.sh create mode 100644 conftest.py create mode 100644 dask_jobqueue/tests/test_jobqueue_core.py create mode 100644 dask_jobqueue/tests/test_sge.py diff --git a/.travis.yml b/.travis.yml index 2af3a790..1223c8e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,47 +7,42 @@ notifications: services: - docker - matrix: include: - python: "2.7" - env: OS=ubuntu-14.04 + env: + - OS=ubuntu-14.04 + - JOBQUEUE=sge - python: "3.6" - env: OS=ubuntu-14.04 - + env: + - OS=ubuntu-14.04 + - JOBQUEUE=sge env: global: - DOCKER_COMPOSE_VERSION=1.6.0 - before_install: - pwd - docker version - docker-compose version - # Install miniconda - ./ci/conda_setup.sh - export PATH="$HOME/miniconda/bin:$PATH" - - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION distributed - - # Start containers - - cd ./ci/sge; ./start-sge.sh; cd - - - docker ps -a - - docker images - + - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION dask distributed flake8 + # Start jobqueue + - source ci/${JOBQUEUE}.travis + - jobqueue_before_install install: - which python - pip install --no-deps -e . - + - jobqueue_install script: - flake8 -j auto dask_jobqueue - py.test dask_jobqueue --verbose # run tests (skip queue tests) - - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pip install --no-cache-dir ." - - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose" - + - jobque_script after_success: - - docker exec -it sge_master bash -c 'cat /tmp/sge*' - - docker exec -it slave_one bash -c 'cat /tmp/exec*' - - docker exec -it slave_two bash -c 'cat /tmp/exec*' + - jobqueue_after_success + + # TODO # - pip install --no-cache-dir coveralls # - coveralls diff --git a/ci/sge.sh b/ci/sge.sh new file mode 100644 index 00000000..760ce248 --- /dev/null +++ b/ci/sge.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +function jobqueue_before_install { + docker version + docker-compose version + + # start sge cluster + cd ./ci/sge + ./start-sge.sh + cd - + + docker ps -a + docker images +} + +function jobqueue_install { + docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pip install --no-cache-dir ." +} + +function jobqueue_script { + docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose -E sge" +} + +function jobqueue_after_success { + docker exec -it sge_master bash -c 'cat /tmp/sge*' + docker exec -it slave_one bash -c 'cat /tmp/exec*' + docker exec -it slave_two bash -c 'cat /tmp/exec*' +} diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..3a35d9e0 --- /dev/null +++ b/conftest.py @@ -0,0 +1,22 @@ +# content of conftest.py + +import pytest + + +def pytest_addoption(parser): + parser.addoption("-E", action="store", metavar="NAME", + help="only run tests matching the environment NAME.") + + +def pytest_configure(config): + # register an additional marker + config.addinivalue_line( + "markers", "env(name): mark test to run only on named environment") + + +def pytest_runtest_setup(item): + envmarker = item.get_marker("env") + if envmarker is not None: + envname = envmarker.args[0] + if envname != item.config.getoption("-E"): + pytest.skip("test requires env %r" % envname) diff --git a/dask_jobqueue/tests/test_jobqueue_core.py b/dask_jobqueue/tests/test_jobqueue_core.py new file mode 100644 index 00000000..2f0d9a67 --- /dev/null +++ b/dask_jobqueue/tests/test_jobqueue_core.py @@ -0,0 +1,4 @@ + +def test_jq_core_placeholder(): + # to test that CI is working + pass diff --git a/dask_jobqueue/tests/test_pbs.py b/dask_jobqueue/tests/test_pbs.py index dfe65df4..a72ccad5 100644 --- a/dask_jobqueue/tests/test_pbs.py +++ b/dask_jobqueue/tests/test_pbs.py @@ -5,10 +5,12 @@ from dask.distributed import Client from distributed.utils_test import loop # noqa: F401 -from pangeo import PBSCluster +from dask_jobqueue import PBSCluster +pytestmark = pytest.mark.env("pbs") -def test_basic(loop): + +def test_basic(loop): # noqa: F811 with PBSCluster(walltime='00:02:00', threads_per_worker=2, memory='7GB', interface='ib0', loop=loop) as cluster: with Client(cluster) as client: @@ -32,7 +34,7 @@ def test_basic(loop): assert not cluster.jobs -def test_adaptive(loop): +def test_adaptive(loop): # noqa: F811 with PBSCluster(walltime='00:02:00', loop=loop) as cluster: cluster.adapt() with Client(cluster) as client: @@ -59,7 +61,7 @@ def test_adaptive(loop): assert time() < start + 10 -@pytest.mark.skipif('PBS_ACCOUNT' in os.environ, reason='PBS_ACCOUNT defined') +@pytest.mark.skipif('PBS_ACCOUNT' in os.environ, reason='PBS_ACCOUNT defined') # noqa: F811 def test_errors(loop): with pytest.raises(ValueError) as info: PBSCluster() diff --git a/dask_jobqueue/tests/test_sge.py b/dask_jobqueue/tests/test_sge.py new file mode 100644 index 00000000..15e984c8 --- /dev/null +++ b/dask_jobqueue/tests/test_sge.py @@ -0,0 +1,9 @@ + +import pytest + +pytestmark = pytest.mark.env("sge") + + +def test_sge_placeholder(): + # to test that CI is working + pass diff --git a/dask_jobqueue/tests/test_slurm.py b/dask_jobqueue/tests/test_slurm.py index 6d542782..e5f42035 100644 --- a/dask_jobqueue/tests/test_slurm.py +++ b/dask_jobqueue/tests/test_slurm.py @@ -5,12 +5,14 @@ from dask.distributed import Client from distributed.utils_test import loop # noqa: F401 -from pangeo import SLURMCluster +from dask_jobqueue import SLURMCluster +pytestmark = pytest.mark.env("pbs") -def test_basic(loop): + +def test_basic(loop): # noqa: F811 with SLURMCluster(walltime='00:02:00', threads_per_worker=2, memory='7GB', - loop=loop) as cluster: + loop=loop) as cluster: with Client(cluster) as client: workers = cluster.start_workers(2) future = client.submit(lambda x: x + 1, 10) @@ -32,7 +34,7 @@ def test_basic(loop): assert not cluster.jobs -def test_adaptive(loop): +def test_adaptive(loop): # noqa: F811 with SLURMCluster(walltime='00:02:00', loop=loop) as cluster: cluster.adapt() with Client(cluster) as client: @@ -42,8 +44,8 @@ def test_adaptive(loop): assert cluster.jobs start = time() - while len(client.scheduler_info()['workers']) \ - != cluster.config['processes']: + while (len(client.scheduler_info()['workers']) != + cluster.config['processes']): sleep(0.1) assert time() < start + 10 @@ -60,7 +62,7 @@ def test_adaptive(loop): assert time() < start + 10 -@pytest.mark.skipif('PBS_ACCOUNT' in os.environ, reason='PBS_ACCOUNT defined') +@pytest.mark.skipif('PBS_ACCOUNT' in os.environ, reason='PBS_ACCOUNT defined') # noqa: F811 def test_errors(loop): with pytest.raises(ValueError) as info: SLURMCluster() From c2158a6525c9535b1a06f297e0f3b7f0dc033b2c Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Fri, 23 Mar 2018 21:10:22 -0700 Subject: [PATCH 8/9] fix typos --- .travis.yml | 2 +- ci/sge.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1223c8e9..d9f082a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_install: - export PATH="$HOME/miniconda/bin:$PATH" - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION dask distributed flake8 # Start jobqueue - - source ci/${JOBQUEUE}.travis + - source ci/${JOBQUEUE}.sh - jobqueue_before_install install: - which python diff --git a/ci/sge.sh b/ci/sge.sh index 760ce248..de501f5d 100644 --- a/ci/sge.sh +++ b/ci/sge.sh @@ -8,7 +8,7 @@ function jobqueue_before_install { cd ./ci/sge ./start-sge.sh cd - - + docker ps -a docker images } From 988b339b97f1cba2ba55c88cbfc98d75b2f5f492 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Fri, 23 Mar 2018 22:08:30 -0700 Subject: [PATCH 9/9] setup.py install for docker on sge --- ci/sge.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/sge.sh b/ci/sge.sh index de501f5d..fa9d824b 100644 --- a/ci/sge.sh +++ b/ci/sge.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -x + function jobqueue_before_install { docker version docker-compose version @@ -14,7 +16,7 @@ function jobqueue_before_install { } function jobqueue_install { - docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; pip install --no-cache-dir ." + docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; python setup.py install" } function jobqueue_script {