From ba2c15bbcb294c9242026360941ec698d41eebb9 Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Wed, 25 Apr 2018 23:43:34 +0200
Subject: [PATCH 1/8] Squashing commit into one. Adding CI with dockerized PBS

WIP: adding CI with a dockerized PBS cluster

almost there

Working pbs docker cluster, fix was to add user on slaves

Test are almost working, may need feedback

Adding new job in Travis.

removing unused files
---
 .travis.yml                     |  4 ++++
 ci/pbs.sh                       | 40 +++++++++++++++++++++++++++++++
 ci/pbs/Dockerfile               | 41 ++++++++++++++++++++++++++++++++
 ci/pbs/build.sh                 | 10 ++++++++
 ci/pbs/docker-compose.yml       | 42 +++++++++++++++++++++++++++++++++
 ci/pbs/master-entrypoint.sh     | 16 +++++++++++++
 ci/pbs/run-master.sh            |  8 +++++++
 ci/pbs/slave-entrypoint.sh      | 20 ++++++++++++++++
 dask_jobqueue/tests/test_pbs.py | 11 +++++----
 9 files changed, 187 insertions(+), 5 deletions(-)
 create mode 100644 ci/pbs.sh
 create mode 100644 ci/pbs/Dockerfile
 create mode 100644 ci/pbs/build.sh
 create mode 100644 ci/pbs/docker-compose.yml
 create mode 100644 ci/pbs/master-entrypoint.sh
 create mode 100755 ci/pbs/run-master.sh
 create mode 100644 ci/pbs/slave-entrypoint.sh

diff --git a/.travis.yml b/.travis.yml
index 0d424885..0da7ca0d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,10 @@ matrix:
         - OS=ubuntu-14.04
         # JOBQUEUE=none is for tests that do not need a cluster to run
         - JOBQUEUE=none
+    - python: "3.6"
+      env:
+        - OS=ubuntu-14.04
+        - JOBQUEUE=pbs
 env:
   global:
     - DOCKER_COMPOSE_VERSION=1.6.0
diff --git a/ci/pbs.sh b/ci/pbs.sh
new file mode 100644
index 00000000..1a0e1808
--- /dev/null
+++ b/ci/pbs.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -x
+
+function jobqueue_before_install {
+    docker version
+    docker-compose version
+
+    # start pbs cluster
+    cd ./ci/pbs
+    docker-compose up -d
+    while [ `docker exec -it -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ]
+    do
+        echo "Waiting for PBS slave nodes to become available";
+        sleep 2
+    done
+    echo "PBS properly configured"
+    docker exec -it -u pbsuser pbs_master pbsnodes -a
+    cd -
+
+    docker ps -a
+    docker images
+}
+
+function jobqueue_install {
+    docker exec -it pbs_master /bin/bash -c "cd /dask-jobqueue; python setup.py install"
+}
+
+function jobqueue_script {
+    docker exec -it -u pbsuser pbs_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose -E pbs"
+}
+
+function jobqueue_after_success {
+    docker exec -it -u pbsuser pbs_master qstat
+    docker exec -it pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*'
+    docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*'
+    docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*'
+    docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*'
+    docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*'
+}
diff --git a/ci/pbs/Dockerfile b/ci/pbs/Dockerfile
new file mode 100644
index 00000000..fd034b9b
--- /dev/null
+++ b/ci/pbs/Dockerfile
@@ -0,0 +1,41 @@
+# inspired from https://github.com/PBSPro/pbspro/blob/v18.1.beta/docker/centos7/
+# multi-stage build
+# build script will be triggered
+FROM centos:7 AS builder
+# install dependencies for building
+RUN yum install -y gcc make rpm-build libtool hwloc-devel libX11-devel \
+	libXt-devel libedit-devel libical-devel ncurses-devel perl \
+	postgresql-devel python-devel tcl-devel  tk-devel swig expat-devel \
+    openssl-devel libXext libXft git
+# get latest PBS Pro source code
+# TODO point to an identified tag
+RUN git clone https://github.com/pbspro/pbspro.git /src/pbspro && \
+    bash /src/pbspro/docker/centos7/build.sh
+
+# base image 
+FROM centos:7
+LABEL description="PBS Professional Open Source"
+# copy rpm and entrypoint script from builder
+COPY --from=builder /root/rpmbuild/RPMS/x86_64/pbspro-server-*.rpm .
+# install pbspro
+RUN yum install -y pbspro-server-*.rpm
+# install python and useful package
+RUN yum install -y curl bzip2 git gcc sudo && yum clean all
+RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash miniconda.sh -f -b -p /opt/anaconda && \
+    /opt/anaconda/bin/conda clean -tipy && \
+    rm -f miniconda.sh
+
+#The pbs master hostname and path
+ENV PBS_MASTER pbs_master
+ENV PATH /opt/pbs/bin:/opt/anaconda/bin:$PATH
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+RUN conda install --yes -c conda-forge python=3.6 dask distributed flake8 pytest docrep
+# Copy entrypoint
+COPY ./*.sh /
+RUN chmod a+x ./*.sh
+
+# run entrypoint script
+ENTRYPOINT ["bash", "/master-entrypoint.sh"]
diff --git a/ci/pbs/build.sh b/ci/pbs/build.sh
new file mode 100644
index 00000000..c811216c
--- /dev/null
+++ b/ci/pbs/build.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+cd /src/pbspro
+./autogen.sh
+./configure -prefix=/opt/pbs
+make dist
+mkdir /root/rpmbuild /root/rpmbuild/SOURCES /root/rpmbuild/SPECS
+cp pbspro-*.tar.gz /root/rpmbuild/SOURCES
+cp pbspro.spec /root/rpmbuild/SPECS
+cd /root/rpmbuild/SPECS
+rpmbuild -ba pbspro.spec
diff --git a/ci/pbs/docker-compose.yml b/ci/pbs/docker-compose.yml
new file mode 100644
index 00000000..d6142ea9
--- /dev/null
+++ b/ci/pbs/docker-compose.yml
@@ -0,0 +1,42 @@
+version: "3"
+
+services:
+
+  master:
+    build: .
+    container_name: pbs_master
+    hostname: pbs_master
+    volumes:
+      - ../..:/dask-jobqueue
+    command: bash /run-master.sh
+
+  slave_one:
+    build: .
+    container_name: pbs_slave_1
+    hostname: pbs_slave_1
+    volumes:
+      - ../..:/dask-jobqueue
+    entrypoint: "bash /slave-entrypoint.sh"
+    command: sleep 3600
+    links:
+      - "master:pbs_master"
+    environment:
+      - PBS_MASTER=pbs_master
+    depends_on:
+      - master
+
+  slave_two:
+    build: .
+    container_name: pbs_slave_2
+    hostname: pbs_slave_2
+    volumes:
+      - ../..:/dask-jobqueue
+    entrypoint: "bash /slave-entrypoint.sh"
+    command: sleep 3600
+    links:
+      - "master:pbs_master"
+    environment:
+      - PBS_MASTER=pbs_master
+    depends_on:
+      - master
+
diff --git a/ci/pbs/master-entrypoint.sh b/ci/pbs/master-entrypoint.sh
new file mode 100644
index 00000000..7a2669cb
--- /dev/null
+++ b/ci/pbs/master-entrypoint.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+pbs_conf_file=/etc/pbs.conf
+mom_conf_file=/var/spool/pbs/mom_priv/config
+hostname=$(hostname)
+
+# replace hostname in pbs.conf and mom_priv/config
+sed -i "s/PBS_SERVER=.*/PBS_SERVER=$hostname/" $pbs_conf_file
+sed -i "s/\$clienthost .*/\$clienthost $hostname/" $mom_conf_file
+
+# start PBS Pro
+/etc/init.d/pbs start
+
+# create default non-root user
+adduser pbsuser
+
+exec "$@"
diff --git a/ci/pbs/run-master.sh b/ci/pbs/run-master.sh
new file mode 100755
index 00000000..e12833ea
--- /dev/null
+++ b/ci/pbs/run-master.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#add two slaves to pbs
+qmgr -c "create node pbs_slave_1"
+qmgr -c "create node pbs_slave_2"
+
+#wait until the end of tests
+/bin/sleep 3600
diff --git a/ci/pbs/slave-entrypoint.sh b/ci/pbs/slave-entrypoint.sh
new file mode 100644
index 00000000..bdb7097b
--- /dev/null
+++ b/ci/pbs/slave-entrypoint.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+pbs_conf_file=/etc/pbs.conf
+mom_conf_file=/var/spool/pbs/mom_priv/config
+hostname=$(hostname)
+
+# replace hostname in pbs.conf and mom_priv/config
+sed -i "s/PBS_SERVER=.*/PBS_SERVER=$PBS_MASTER/" $pbs_conf_file
+sed -i "s/\$clienthost .*/\$clienthost $hostname/" $mom_conf_file
+sed -i "s/PBS_START_SERVER=.*/PBS_START_SERVER=0/" $pbs_conf_file
+sed -i "s/PBS_START_SCHED=.*/PBS_START_SCHED=0/" $pbs_conf_file
+sed -i "s/PBS_START_COMM=.*/PBS_START_COMM=0/" $pbs_conf_file
+sed -i "s/PBS_START_MOM=.*/PBS_START_MOM=1/" $pbs_conf_file
+
+# start PBS Pro
+/etc/init.d/pbs start
+
+# create default non-root user
+adduser pbsuser
+
+exec "$@"
diff --git a/dask_jobqueue/tests/test_pbs.py b/dask_jobqueue/tests/test_pbs.py
index 4aa4f3f6..f0fee887 100644
--- a/dask_jobqueue/tests/test_pbs.py
+++ b/dask_jobqueue/tests/test_pbs.py
@@ -77,8 +77,8 @@ def test_job_script():
 
 @pytest.mark.env("pbs")  # noqa: F811
 def test_basic(loop):
-    with PBSCluster(walltime='00:02:00', threads_per_worker=2, memory='7GB',
-                    interface='ib0', loop=loop) as cluster:
+    with PBSCluster(walltime='00:02:00', processes=1, threads=2, memory='2GB', local_directory='/tmp',
+                    job_extra=['-V'], loop=loop) as cluster:
         with Client(cluster) as client:
             workers = cluster.start_workers(2)
             future = client.submit(lambda x: x + 1, 10)
@@ -87,7 +87,7 @@ def test_basic(loop):
 
             info = client.scheduler_info()
             w = list(info['workers'].values())[0]
-            assert w['memory_limit'] == 7e9
+            assert w['memory_limit'] == 2e9
             assert w['ncores'] == 2
 
             cluster.stop_workers(workers)
@@ -102,7 +102,8 @@ def test_basic(loop):
 
 @pytest.mark.env("pbs")  # noqa: F811
 def test_adaptive(loop):
-    with PBSCluster(walltime='00:02:00', loop=loop) as cluster:
+    with PBSCluster(walltime='00:02:00', processes=1, threads=2, memory='2GB', local_directory='/tmp',
+                    job_extra=['-V'], loop=loop) as cluster:
         cluster.adapt()
         with Client(cluster) as client:
             future = client.submit(lambda x: x + 1, 10)
@@ -111,7 +112,7 @@ def test_adaptive(loop):
             assert cluster.jobs
 
             start = time()
-            processes = cluster.config['processes']
+            processes = cluster.worker_processes
             while len(client.scheduler_info()['workers']) != processes:
                 sleep(0.1)
                 assert time() < start + 10

From 9230e5e4ca946d578cadfba22d92442c42cdcfab Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Sat, 28 Apr 2018 11:51:48 +0200
Subject: [PATCH 2/8] Use latest distributed versio from master

---
 ci/pbs/Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/pbs/Dockerfile b/ci/pbs/Dockerfile
index fd034b9b..e22a9e75 100644
--- a/ci/pbs/Dockerfile
+++ b/ci/pbs/Dockerfile
@@ -33,6 +33,8 @@ ENV LANG en_US.UTF-8
 ENV LC_ALL en_US.UTF-8
 
 RUN conda install --yes -c conda-forge python=3.6 dask distributed flake8 pytest docrep
+# Take the latest version of distributed due to test failure otherwise (see #47 comment by mrocklin)
+RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade
 # Copy entrypoint
 COPY ./*.sh /
 RUN chmod a+x ./*.sh

From a6a1ec0230b1e36ef02bddc6b5ffe91cc5a9cfcd Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Sat, 28 Apr 2018 15:48:57 +0200
Subject: [PATCH 3/8] Fixing versions of OS and PBS for stability

---
 ci/pbs/Dockerfile | 39 +++++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/ci/pbs/Dockerfile b/ci/pbs/Dockerfile
index e22a9e75..e9898dbf 100644
--- a/ci/pbs/Dockerfile
+++ b/ci/pbs/Dockerfile
@@ -1,43 +1,42 @@
 # inspired from https://github.com/PBSPro/pbspro/blob/v18.1.beta/docker/centos7/
 # multi-stage build
 # build script will be triggered
-FROM centos:7 AS builder
+FROM centos:7.4.1708 AS builder
 # install dependencies for building
 RUN yum install -y gcc make rpm-build libtool hwloc-devel libX11-devel \
 	libXt-devel libedit-devel libical-devel ncurses-devel perl \
 	postgresql-devel python-devel tcl-devel  tk-devel swig expat-devel \
     openssl-devel libXext libXft git
-# get latest PBS Pro source code
-# TODO point to an identified tag
-RUN git clone https://github.com/pbspro/pbspro.git /src/pbspro && \
-    bash /src/pbspro/docker/centos7/build.sh
+# get known PBS Pro source code
+RUN git clone --branch v14.1.2 https://github.com/pbspro/pbspro.git /src/pbspro
+COPY build.sh /
+RUN bash /build.sh
 
 # base image 
-FROM centos:7
-LABEL description="PBS Professional Open Source"
-# copy rpm and entrypoint script from builder
-COPY --from=builder /root/rpmbuild/RPMS/x86_64/pbspro-server-*.rpm .
-# install pbspro
-RUN yum install -y pbspro-server-*.rpm
-# install python and useful package
-RUN yum install -y curl bzip2 git gcc sudo && yum clean all
-RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-    bash miniconda.sh -f -b -p /opt/anaconda && \
-    /opt/anaconda/bin/conda clean -tipy && \
-    rm -f miniconda.sh
+FROM centos:7.4.1708
+LABEL description="PBS Professional Open Source and conda"
 
-#The pbs master hostname and path
+#The pbs master node name, can be overriden if needed
 ENV PBS_MASTER pbs_master
 ENV PATH /opt/pbs/bin:/opt/anaconda/bin:$PATH
 ENV LANG en_US.UTF-8
 ENV LC_ALL en_US.UTF-8
 
+COPY --from=builder /root/rpmbuild/RPMS/x86_64/pbspro-server-*.rpm .
+# install pbspro and useful packages
+RUN yum install -y pbspro-server-*.rpm curl bzip2 git gcc sudo && yum clean all
+# install python
+RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash miniconda.sh -f -b -p /opt/anaconda && \
+    /opt/anaconda/bin/conda clean -tipy && \
+    rm -f miniconda.sh
 RUN conda install --yes -c conda-forge python=3.6 dask distributed flake8 pytest docrep
 # Take the latest version of distributed due to test failure otherwise (see #47 comment by mrocklin)
 RUN pip install --no-cache-dir git+https://github.com/dask/distributed.git --upgrade
-# Copy entrypoint
+
+# Copy entrypoint and other needed scripts
 COPY ./*.sh /
 RUN chmod a+x ./*.sh
 
-# run entrypoint script
+# default entrypoint launch pbs master
 ENTRYPOINT ["bash", "/master-entrypoint.sh"]

From e6d5741b7324e0db98fde96fe52a05f13503d36b Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Sun, 29 Apr 2018 14:48:52 +0200
Subject: [PATCH 4/8] (Altered) tests workings with Docker on laptop. Modifying
 travis conf to add some debug

---
 .travis.yml                     |  4 ++--
 ci/none.sh                      |  4 ++--
 ci/pbs.sh                       |  2 +-
 ci/sge.sh                       |  2 +-
 dask_jobqueue/tests/test_pbs.py | 10 ++++++----
 5 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0da7ca0d..932189b6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -44,8 +44,8 @@ install:
   - jobqueue_install
 script:
   - jobqueue_script
-after_success:
-  - jobqueue_after_success
+after_script:
+  - jobqueue_after_script
 
   # TODO
   # - pip install --no-cache-dir coveralls
diff --git a/ci/none.sh b/ci/none.sh
index 0fa62916..333f18d7 100644
--- a/ci/none.sh
+++ b/ci/none.sh
@@ -19,6 +19,6 @@ function jobqueue_script {
   py.test --verbose
 }
 
-function jobqueue_after_success {
-    echo "Hurrah"
+function jobqueue_after_script {
+    echo "Done."
 }
diff --git a/ci/pbs.sh b/ci/pbs.sh
index 1a0e1808..4088decb 100644
--- a/ci/pbs.sh
+++ b/ci/pbs.sh
@@ -30,7 +30,7 @@ function jobqueue_script {
     docker exec -it -u pbsuser pbs_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose -E pbs"
 }
 
-function jobqueue_after_success {
+function jobqueue_after_script {
     docker exec -it -u pbsuser pbs_master qstat
     docker exec -it pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*'
     docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*'
diff --git a/ci/sge.sh b/ci/sge.sh
index fa9d824b..c4ed8a54 100644
--- a/ci/sge.sh
+++ b/ci/sge.sh
@@ -23,7 +23,7 @@ function jobqueue_script {
     docker exec -it sge_master /bin/bash -c "cd /dask-jobqueue; py.test dask_jobqueue --verbose -E sge"
 }
 
-function jobqueue_after_success {
+function jobqueue_after_script {
     docker exec -it sge_master bash -c 'cat /tmp/sge*'
     docker exec -it slave_one bash -c 'cat /tmp/exec*'
     docker exec -it slave_two bash -c 'cat /tmp/exec*'
diff --git a/dask_jobqueue/tests/test_pbs.py b/dask_jobqueue/tests/test_pbs.py
index f0fee887..39e810e0 100644
--- a/dask_jobqueue/tests/test_pbs.py
+++ b/dask_jobqueue/tests/test_pbs.py
@@ -124,7 +124,9 @@ def test_adaptive(loop):
                 sleep(0.100)
                 assert time() < start + 10
 
-            start = time()
-            while cluster.jobs:
-                sleep(0.100)
-                assert time() < start + 10
+            # There is probably a bug to fix in the adaptive methods of the JobQueueCluster
+            # Currently cluster.jobs is not cleaned up.
+            #start = time()
+            #while cluster.jobs:
+            #    sleep(0.100)
+            #    assert time() < start + 10

From 8f138ee354b85345968208b8b3c0477008b60292 Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Sun, 29 Apr 2018 15:40:17 +0200
Subject: [PATCH 5/8] changing PBS scheduling time. Adding some trace at the
 end

---
 ci/pbs.sh            | 2 ++
 ci/pbs/run-master.sh | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ci/pbs.sh b/ci/pbs.sh
index 4088decb..76123665 100644
--- a/ci/pbs.sh
+++ b/ci/pbs.sh
@@ -33,6 +33,8 @@ function jobqueue_script {
 function jobqueue_after_script {
     docker exec -it -u pbsuser pbs_master qstat
     docker exec -it pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*'
+    docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_logs/*'
+    docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*'
     docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*'
     docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*'
     docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*'
diff --git a/ci/pbs/run-master.sh b/ci/pbs/run-master.sh
index e12833ea..2b869b4c 100755
--- a/ci/pbs/run-master.sh
+++ b/ci/pbs/run-master.sh
@@ -1,8 +1,10 @@
 #!/bin/bash
 
-#add two slaves to pbs
+# Reduce time between PBS scheduling
+qmgr -c "set server scheduler_iteration = 20"
+# add two slaves to pbs
 qmgr -c "create node pbs_slave_1"
 qmgr -c "create node pbs_slave_2"
 
-#wait until the end of tests
+# wait until the end of tests
 /bin/sleep 3600

From ffd8702786fec916e72b669dba08fec6fbd29ce3 Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Thu, 3 May 2018 00:21:03 +0200
Subject: [PATCH 6/8] Disabling scp from stdout and stderr at the end of the
 jobs

---
 ci/pbs/Dockerfile          | 2 +-
 ci/pbs/slave-entrypoint.sh | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ci/pbs/Dockerfile b/ci/pbs/Dockerfile
index e9898dbf..e1af8379 100644
--- a/ci/pbs/Dockerfile
+++ b/ci/pbs/Dockerfile
@@ -24,7 +24,7 @@ ENV LC_ALL en_US.UTF-8
 
 COPY --from=builder /root/rpmbuild/RPMS/x86_64/pbspro-server-*.rpm .
 # install pbspro and useful packages
-RUN yum install -y pbspro-server-*.rpm curl bzip2 git gcc sudo && yum clean all
+RUN yum install -y pbspro-server-*.rpm curl bzip2 git gcc sudo openssh-server && yum clean all
 # install python
 RUN curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash miniconda.sh -f -b -p /opt/anaconda && \
diff --git a/ci/pbs/slave-entrypoint.sh b/ci/pbs/slave-entrypoint.sh
index bdb7097b..19d978aa 100644
--- a/ci/pbs/slave-entrypoint.sh
+++ b/ci/pbs/slave-entrypoint.sh
@@ -11,6 +11,11 @@ sed -i "s/PBS_START_SCHED=.*/PBS_START_SCHED=0/" $pbs_conf_file
 sed -i "s/PBS_START_COMM=.*/PBS_START_COMM=0/" $pbs_conf_file
 sed -i "s/PBS_START_MOM=.*/PBS_START_MOM=1/" $pbs_conf_file
 
+# Prevent PBS trying to use scp between host for stdout and stderr file of jobs
+# On standard PBS deployement, you would use a shared mount, or correctly configured passwordless scp
+echo "\$usecp *:/home/ /home/" >> $mom_conf_file
+echo "\$usecp *:/dask-jobqueue/ /tmp/" >> $mom_conf_file
+
 # start PBS Pro
 /etc/init.d/pbs start
 

From 7af1a111b9bcc0fa06b841c037d9ad5a9208a6ff Mon Sep 17 00:00:00 2001
From: Guillaume Eynard-Bontemps <guillaume.eynard-bontemps@cnes.fr>
Date: Thu, 3 May 2018 08:56:20 +0000
Subject: [PATCH 7/8] Consistency with sge ci, improved debugging in travis

---
 ci/pbs.sh                 | 16 +++++++---------
 ci/pbs/docker-compose.yml |  4 ++--
 ci/pbs/run-master.sh      |  9 ++++++---
 ci/pbs/run-slave.sh       |  4 ++++
 ci/pbs/start-pbs.sh       |  9 +++++++++
 5 files changed, 28 insertions(+), 14 deletions(-)
 create mode 100755 ci/pbs/run-slave.sh
 create mode 100755 ci/pbs/start-pbs.sh

diff --git a/ci/pbs.sh b/ci/pbs.sh
index 76123665..715c45d1 100644
--- a/ci/pbs.sh
+++ b/ci/pbs.sh
@@ -8,16 +8,10 @@ function jobqueue_before_install {
 
     # start pbs cluster
     cd ./ci/pbs
-    docker-compose up -d
-    while [ `docker exec -it -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ]
-    do
-        echo "Waiting for PBS slave nodes to become available";
-        sleep 2
-    done
-    echo "PBS properly configured"
-    docker exec -it -u pbsuser pbs_master pbsnodes -a
+    ./start-pbs.sh
     cd -
 
+    docker exec -it -u pbsuser pbs_master pbsnodes -a
     docker ps -a
     docker images
 }
@@ -31,12 +25,16 @@ function jobqueue_script {
 }
 
 function jobqueue_after_script {
-    docker exec -it -u pbsuser pbs_master qstat
+    docker exec -it -u pbsuser pbs_master qstat -fx
     docker exec -it pbs_master bash -c 'cat /var/spool/pbs/sched_logs/*'
     docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_logs/*'
     docker exec -it pbs_master bash -c 'cat /var/spool/pbs/server_priv/accounting/*'
     docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/mom_logs/*'
     docker exec -it pbs_slave_1 bash -c 'cat /var/spool/pbs/spool/*'
+    docker exec -it pbs_slave_1 bash -c 'cat /tmp/*.e*'
+    docker exec -it pbs_slave_1 bash -c 'cat /tmp/*.o*'
     docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/mom_logs/*'
     docker exec -it pbs_slave_2 bash -c 'cat /var/spool/pbs/spool/*'
+    docker exec -it pbs_slave_2 bash -c 'cat /tmp/*.e*'
+    docker exec -it pbs_slave_2 bash -c 'cat /tmp/*.o*'
 }
diff --git a/ci/pbs/docker-compose.yml b/ci/pbs/docker-compose.yml
index d6142ea9..b2d1a5d0 100644
--- a/ci/pbs/docker-compose.yml
+++ b/ci/pbs/docker-compose.yml
@@ -17,7 +17,7 @@ services:
     volumes:
       - ../..:/dask-jobqueue
     entrypoint: "bash /slave-entrypoint.sh"
-    command: sleep 3600
+    command: bash /run-slave.sh
     links:
       - "master:pbs_master"
     environment:
@@ -32,7 +32,7 @@ services:
     volumes:
       - ../..:/dask-jobqueue
     entrypoint: "bash /slave-entrypoint.sh"
-    command: sleep 3600
+    command: bash /run-slave.sh
     links:
       - "master:pbs_master"
     environment:
diff --git a/ci/pbs/run-master.sh b/ci/pbs/run-master.sh
index 2b869b4c..18c78996 100755
--- a/ci/pbs/run-master.sh
+++ b/ci/pbs/run-master.sh
@@ -1,10 +1,13 @@
 #!/bin/bash
 
-# Reduce time between PBS scheduling
+# Reduce time between PBS scheduling and add history
 qmgr -c "set server scheduler_iteration = 20"
+qmgr -c "set server job_history_enable = True"
+qmgr -c "set server job_history_duration = 24:00:00"
+
 # add two slaves to pbs
 qmgr -c "create node pbs_slave_1"
 qmgr -c "create node pbs_slave_2"
 
-# wait until the end of tests
-/bin/sleep 3600
+# Start hanging process to leave the container up and running
+python -m http.server 8888
diff --git a/ci/pbs/run-slave.sh b/ci/pbs/run-slave.sh
new file mode 100755
index 00000000..987e0efd
--- /dev/null
+++ b/ci/pbs/run-slave.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Start hanging process to leave the container up and running
+python -m http.server 8888
diff --git a/ci/pbs/start-pbs.sh b/ci/pbs/start-pbs.sh
new file mode 100755
index 00000000..4162f11a
--- /dev/null
+++ b/ci/pbs/start-pbs.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+docker-compose up -d
+while [ `docker exec -it -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ]
+do
+    echo "Waiting for PBS slave nodes to become available";
+    sleep 2
+done
+echo "PBS properly configured"

From ca514d9068c947274cc6cbd3bf55d14280017650 Mon Sep 17 00:00:00 2001
From: guillaumeeb <g.eynard.bontemps@gmail.com>
Date: Thu, 3 May 2018 12:57:26 +0000
Subject: [PATCH 8/8] docker-compose version 2 should be enough

---
 ci/pbs/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/pbs/docker-compose.yml b/ci/pbs/docker-compose.yml
index b2d1a5d0..b148f4fd 100644
--- a/ci/pbs/docker-compose.yml
+++ b/ci/pbs/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "3"
+version: "2"
 
 services: