- Rewrite R base image due to gdal updates. (#66)

- Bug fixes.

Author: pflickin

Containers/base-py/ai4e_api_tools/task_management/api_task.py

@@ -34,7 +34,7 @@ def UpdateTaskStatus(self, taskId, status):
 
     def AddPipelineTask(self, taskId, organization_moniker, version, api_name, body):
         next_url = organization_moniker + '/' + version + '/' + api_name
-        UpdateTaskStatus(taskId, "Pipelining is not supported in a single node deployment, but the next service is: " + next_url)
+        self.UpdateTaskStatus(taskId, "Pipelining is not supported in a single node deployment, but the next service is: " + next_url)
         return "Pipelining is not supported in a single node deployment, but the next service is: " + next_url
 
     def CompleteTask(self, taskId, status):

Containers/base-r/Dockerfile

@@ -1,93 +1,46 @@
 #mcr.microsoft.com/aiforearth/base-r:version
-FROM nvidia/cuda:9.2-runtime-ubuntu16.04
+FROM osgeo/gdal:ubuntu-full-3.0.3
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN mkdir /var/uwsgi
-
-RUN apt-get update \
-  && apt-get install -y python3-pip python3-dev \
-  && cd /usr/local/bin \
-  && ln -s /usr/bin/python3 python \
-  && pip3 install --upgrade pip
-
 RUN apt-get update --fix-missing && \
     apt-get install -y wget supervisor bzip2 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-ENV GDAL_VERSION=2.2.1 
-
-RUN apt-get update && \
-    apt-get -y install \
-        libpq-dev \
-        ogdi-bin \
-        libogdi3.2-dev \
-        libjasper-runtime \
-        libjasper-dev \
-        libjasper1 \
-        libgeos-dev \
-        libproj-dev \
-        libpoppler-dev \
-        libsqlite3-dev \
-        libspatialite-dev \
-        python3 \
-        python3-dev \
-        python3-numpy
-
-RUN wget http://download.osgeo.org/gdal/$GDAL_VERSION/gdal-${GDAL_VERSION}.tar.gz -O /tmp/gdal-${GDAL_VERSION}.tar.gz && \
-    tar -x -f /tmp/gdal-${GDAL_VERSION}.tar.gz -C /tmp
-
-RUN tar -xvf /tmp/gdal-${GDAL_VERSION}.tar.gz && cd /tmp/gdal-${GDAL_VERSION} \
-    && ./configure \
-    && make && make install && ldconfig \
-    && apt-get update -y \
-    && apt-get remove -y --purge build-essential wget \
-    && rm -Rf $ROOTDIR/src/*
-
-RUN rm /tmp/gdal-${GDAL_VERSION} -rf
-
-CMD gdalinfo --version && gdalinfo --formats && ogrinfo --formats
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		r-base \
+		r-base-dev \
+    r-recommended
 
 RUN apt-get update \
-&& apt-get install -y \
-  apt-transport-https \
-  build-essential \
-  ed \
-  gtk2.0 \
-  libcurl4-openssl-dev \
-  libgtk2.0-dev \
-  libiodbc2-dev \
-  libnlopt-dev \
-  libssh2-1-dev \
-  libssl-dev \
-  libxml2-dev \
-  software-properties-common \
-  wget \
-  xvfb \
-&& rm -rf /var/lib/apt/lists/*
+    && apt-get install -y \
+        apt-transport-https \
+        build-essential \
+        ed \
+        gtk2.0 \
+        libcurl4-openssl-dev \
+        libgtk2.0-dev \
+        libiodbc2-dev \
+        libnlopt-dev \
+        libssh2-1-dev \
+        libssl-dev \
+        libxml2-dev \
+        software-properties-common \
+        wget \
+        xvfb \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
-RUN add-apt-repository 'deb [arch=amd64,i386] https://cran.rstudio.com/bin/linux/ubuntu xenial/'
+RUN add-apt-repository 'deb [arch=amd64,i386] https://cran.rstudio.com/bin/linux/ubuntu bionic-cran35/'
 
 RUN apt-get update \
 	&& apt-get install -y --no-install-recommends \
 		r-base \
 		r-base-dev \
     r-recommended
 
-RUN apt-get update -qq && apt-get install -y \
-  git-core \
-  libssl-dev \
-  libcurl4-gnutls-dev
-
-# Install data_processing packages
-RUN apt-get update && \
-    apt-get install -y \
-    libgdal-dev \
-    libproj-dev
-
 RUN echo "r <- getOption('repos'); r['CRAN'] <- 'http://cran.r-project.org'; options(repos = r);" > ~/.Rprofile
-#RUN Rscript -e 'remove.packages(c("curl","httr"));'
 RUN Rscript -e 'install.packages(c("curl", "httr"));'
 RUN Rscript -e 'Sys.setenv(CURL_CA_BUNDLE="/utils/microsoft-r-open-3.4.3/lib64/R/lib/microsoft-r-cacert.pem");'
 RUN Rscript -e 'install.packages("sp");'
@@ -100,6 +53,12 @@ RUN Rscript -e 'install.packages("RCurl");'
 RUN Rscript -e 'install.packages("sjmisc");'
 RUN Rscript -e 'install.packages("reticulate");'
 
+RUN apt-get update \
+  && apt-get install -y python3-pip python3-dev \
+  && cd /usr/local/bin \
+  && ln -s /usr/bin/python3 python \
+  && pip3 install --upgrade pip
+  
 # Install Azure Blob SDK and application insights
 RUN pip install azure
 RUN pip install azure-storage-common

Containers/blob-py/Dockerfile

@@ -1,7 +1,8 @@
-FROM mcr.microsoft.com/aiforearth/base-py:1.4
+FROM mcr.microsoft.com/aiforearth/base-py:1.9
 #############################################################################################
 # Add support for mounting an Azure blob
 # Add the Microsoft package repository and install blobfuse
+RUN apt-get update && apt-get install wget
 RUN wget https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb && \
     dpkg -i packages-microsoft-prod.deb && \
     apt-get update && \

Containers/common/aad_blob.py

@@ -50,6 +50,12 @@ def write_blob_from_text(self, container, blob, text):
         service.create_blob_from_text(container, blob, text, 'utf-8', max_connections=100, timeout=600, progress_callback=self.callback)
         return service.make_blob_url(container, blob)
 
+    def create_blob_from_path(self, container, blob, path):
+        service = self._get_service()
+        print("{} for {}/{}".format('create_blob_from_path', container, blob))
+        service.create_blob_from_path(container, blob, path, max_connections=100, timeout=600, progress_callback=self.callback)
+        return service.make_blob_url(container, blob)
+
     def get_blob(self, container, blob):
         service = self._get_service()
         with io.BytesIO() as output_stream:
@@ -79,4 +85,8 @@ def save_local_blob(self, container, blob, local_file):
     def get_blob_uri(self, container, blob):
         service = self._get_service()
         url = service.make_blob_url(container, blob)
-        return url
+        return url
+
+    def does_blob_exist(self, container, blob):
+        service = self._get_service()
+        return service.exists(container, blob)