Initial Travis CI Configuration

.travis.yml

@@ -0,0 +1,48 @@
+language: python
+dist: trusty
+sudo: required
+
+notifications:
+  email: false
+
+services:
+  - docker
+matrix:
+  include:
+    - python: "2.7"
+      env:
+        - OS=ubuntu-14.04
+        - JOBQUEUE=sge
+    - python: "3.6"
+      env:
+        - OS=ubuntu-14.04
+        - JOBQUEUE=sge
+env:
+  global:
+    - DOCKER_COMPOSE_VERSION=1.6.0
+
+before_install:
+  - pwd
+  - docker version
+  - docker-compose version
+  # Install miniconda
+  - ./ci/conda_setup.sh
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - conda install --yes -c conda-forge python=$TRAVIS_PYTHON_VERSION dask distributed flake8
+  # Start jobqueue
+  - source ci/${JOBQUEUE}.sh
+  - jobqueue_before_install
+install:
+  - which python
+  - pip install --no-deps -e .
+  - jobqueue_install
+script:
+  - flake8 -j auto dask_jobqueue
+  - py.test dask_jobqueue --verbose  # run tests (skip queue tests)
+  - jobque_script
+after_success:
+  - jobqueue_after_success
+
+  # TODO
+  # - pip install --no-cache-dir coveralls
+  # - coveralls

ci/conda_setup.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+# Install miniconda
+wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
+bash ~/miniconda.sh -b -p $HOME/miniconda
+export PATH="$HOME/miniconda/bin:$PATH"
+conda update conda --yes
+conda clean -tipy
+conda config --set always_yes yes --set changeps1 no
+conda --version

ci/sge.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -x
+
+function jobqueue_before_install {
+    docker version
+    docker-compose version
+
+    # start sge cluster
+    cd ./ci/sge
+    ./start-sge.sh
+    cd -
+
+    docker ps -a
+    docker images
+}
+
+function jobqueue_install {
+    docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; python setup.py install"
+}
+
+function jobqueue_script {
+    docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose -E sge"
+}
+
+function jobqueue_after_success {
+    docker exec -it sge_master bash -c 'cat /tmp/sge*'
+    docker exec -it slave_one bash -c 'cat /tmp/exec*'
+    docker exec -it slave_two bash -c 'cat /tmp/exec*'
+}

ci/sge/Dockerfile-master

@@ -0,0 +1,29 @@
+FROM ubuntu:14.04
+
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install curl bzip2 git gcc -y --fix-missing
+
+RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash miniconda.sh -f -b -p /opt/anaconda && \
+    /opt/anaconda/bin/conda clean -tipy && \
+    rm -f miniconda.sh
+ENV PATH /opt/anaconda/bin:$PATH
+RUN conda install -n root conda=4.4.11 && conda clean -tipy
+RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip psutil && conda clean -tipy
+RUN pip install --no-cache-dir drmaa
+RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade
+
+COPY ./*.sh /
+COPY ./*.txt /
+RUN bash ./setup-master.sh
+
+# expose ports
+EXPOSE 8000
+EXPOSE 6444
+EXPOSE 6445
+EXPOSE 6446
+
+ENV DRMAA_LIBRARY_PATH /usr/lib/gridengine-drmaa/lib/libdrmaa.so
+ENV SGE_ROOT /var/lib/gridengine/
+ENV SGE_CELL default

ci/sge/Dockerfile-slave

@@ -0,0 +1,21 @@
+FROM ubuntu:14.04
+
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install curl bzip2 git gcc -y --fix-missing
+
+RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash miniconda.sh -f -b -p /opt/anaconda && \
+    /opt/anaconda/bin/conda clean -tipy && \
+    rm -f miniconda.sh
+ENV PATH /opt/anaconda/bin:$PATH
+RUN conda install -n root conda=4.4.11 && conda clean -tipy
+RUN conda install -c conda-forge dask distributed blas pytest mock ipython pip psutil && conda clean -tipy
+RUN pip install --no-cache-dir drmaa
+RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade
+
+COPY ./setup-slave.sh /
+COPY ./*.sh /
+RUN bash ./setup-slave.sh
+
+CMD python -m SimpleHTTPServer

ci/sge/add_worker.sh

@@ -0,0 +1,26 @@
+#`/bin/bash
+
+#!/bin/bash
+
+QUEUE=$1
+HOSTNAME=$2
+SLOTS=$3
+
+# add to the execution host list
+TMPFILE=/tmp/sge.hostname-$HOSTNAME
+echo -e "hostname $HOSTNAME\nload_scaling NONE\ncomplex_values NONE\nuser_lists NONE\nxuser_lists NONE\nprojects NONE\nxprojects NONE\nusage_scaling NONE\nreport_variables NONE" > $TMPFILE
+qconf -Ae $TMPFILE
+rm $TMPFILE
+
+# add to the all hosts list
+qconf -aattr hostgroup hostlist $HOSTNAME @allhosts
+
+# enable the host for the queue, in case it was disabled and not removed
+qmod -e $QUEUE@$HOSTNAME
+
+# Add memory resource
+qconf -mattr exechost complex_values h_vmem=100G $HOSTNAME
+
+if [ "$SLOTS" ]; then
+    qconf -aattr queue slots "[$HOSTNAME=$SLOTS]" $QUEUE
+fi

ci/sge/docker-compose.yml

@@ -0,0 +1,44 @@
+version: "2"
+
+services:
+
+  master:
+    build:
+      context: .
+      dockerfile: Dockerfile-master
+    container_name: sge_master
+    hostname: sge_master
+    #network_mode: host
+    volumes:
+      - .:/dask-jobqueue
+    command: bash /run-master.sh
+
+  slave-one:
+    build:
+      context: .
+      dockerfile: Dockerfile-slave
+    container_name: slave_one
+    hostname: slave_one
+    #network_mode: host
+    volumes:
+      - .:/dask-jobqueue
+    command: bash /run-slave.sh
+    links:
+      - "master:sge_master"
+    depends_on:
+      - master
+
+  slave-two:
+    build:
+      context: .
+      dockerfile: Dockerfile-slave
+    container_name: slave_two
+    hostname: slave_two
+    #network_mode: host
+    volumes:
+      - .:/dask-jobqueue
+    command: bash /run-slave.sh
+    links:
+      - "master:sge_master"
+    depends_on:
+      - master

ci/sge/hosts.txt

@@ -0,0 +1,2 @@
+group_name @allhosts
+hostlist NONE

ci/sge/queue.txt

@@ -0,0 +1,50 @@
+qname                 dask.q
+hostlist              @allhosts
+seq_no                0
+load_thresholds       NONE
+suspend_thresholds    NONE
+nsuspend              1
+suspend_interval      00:00:01
+priority              0
+min_cpu_interval      00:00:01
+processors            UNDEFINED
+qtype                 BATCH INTERACTIVE
+ckpt_list             NONE
+pe_list               make
+rerun                 FALSE
+slots                 2
+tmpdir                /tmp
+shell                 /bin/csh
+prolog                NONE
+epilog                NONE
+shell_start_mode      posix_compliant
+starter_method        NONE
+suspend_method        NONE
+resume_method         NONE
+terminate_method      NONE
+notify                00:00:01
+owner_list            NONE
+user_lists            NONE
+xuser_lists           NONE
+subordinate_list      NONE
+complex_values        NONE
+projects              NONE
+xprojects             NONE
+calendar              NONE
+initial_state         default
+s_rt                  INFINITY
+h_rt                  INFINITY
+s_cpu                 INFINITY
+h_cpu                 INFINITY
+s_fsize               INFINITY
+h_fsize               INFINITY
+s_data                INFINITY
+h_data                INFINITY
+s_stack               INFINITY
+h_stack               INFINITY
+s_core                INFINITY
+h_core                INFINITY
+s_rss                 INFINITY
+h_rss                 INFINITY
+s_vmem                INFINITY
+h_vmem                INFINITY

ci/sge/run-master.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+# start sge
+sudo service gridengine-master restart
+
+while ! ping -c1 slave_one &>/dev/null; do :; done
+
+qconf -Msconf /scheduler.txt
+qconf -Ahgrp /hosts.txt
+qconf -Aq /queue.txt
+
+qconf -ah slave_one
+qconf -ah slave_two
+qconf -ah slave_three
+
+qconf -as $HOSTNAME
+bash add_worker.sh dask.q slave_one 4
+bash add_worker.sh dask.q slave_two 4
+
+sudo service gridengine-master restart
+
+python -m http.server 8888

ci/sge/run-slave.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# start sge
+sudo service gridengine-exec restart
+
+sleep 4
+
+sudo service gridengine-exec restart
+
+python -m http.server 8888

ci/sge/run_master.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -x
+set -e
+
+# start sge
+sudo service gridengine-master restart
+
+while ! ping -c1 slave_one &>/dev/null; do :; done
+
+qconf -Msconf /scheduler.txt
+qconf -Ahgrp /hosts.txt
+qconf -Aq /queue.txt
+
+qconf -ah slave_one
+qconf -ah slave_two
+qconf -ah slave_three
+
+qconf -as $HOSTNAME
+bash add_worker.sh dask.q slave_one 4
+bash add_worker.sh dask.q slave_two 4
+
+sudo service gridengine-master restart
+
+python -m http.server 8888

ci/sge/run_slave.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# start sge
+sudo service gridengine-exec restart
+
+sleep 4
+
+sudo service gridengine-exec restart
+
+python -m http.server 8888

ci/sge/scheduler.txt

@@ -0,0 +1,35 @@
+algorithm                         default
+schedule_interval                 0:0:1
+maxujobs                          0
+queue_sort_method                 load
+job_load_adjustments              np_load_avg=0.50
+load_adjustment_decay_time        0:7:30
+load_formula                      np_load_avg
+schedd_job_info                   true
+flush_submit_sec                  0
+flush_finish_sec                  0
+params                            none
+reprioritize_interval             0:0:0
+halftime                          168
+usage_weight_list                 cpu=1.000000,mem=0.000000,io=0.000000
+compensation_factor               5.000000
+weight_user                       0.250000
+weight_project                    0.250000
+weight_department                 0.250000
+weight_job                        0.250000
+weight_tickets_functional         0
+weight_tickets_share              0
+share_override_tickets            TRUE
+share_functional_shares           TRUE
+max_functional_jobs_to_schedule   200
+report_pjob_tickets               TRUE
+max_pending_tasks_per_job         50
+halflife_decay_list               none
+policy_hierarchy                  OFS
+weight_ticket                     0.500000
+weight_waiting_time               0.278000
+weight_deadline                   3600000.000000
+weight_urgency                    0.500000
+weight_priority                   0.000000
+max_reservation                   0
+default_duration                  INFINITY