Add authorization support for NASA LPDAAC HTTP servers (fix #4); removing the locals() (fix #1); fix for Google Earth Engine support (fix #5); add more information about GEE service accounts (fix #3)

Author: jonas-eberle

Readme.md

@@ -28,6 +28,7 @@ Please make sure that the command-line tools are within the System path. Otherwi
 
 *Further requirements:*
 For MODIS HDF data processing, HDF4 image format has to be available for gdal command-line tools.
+Using the original data source for MODIS Land Products (LPDAAC), user login information from NASA Earthdata are required (see item 'userPwd' in task dictionary).
 
 Usage
 -----
@@ -43,7 +44,8 @@ Usage
         'format': 'HDF4Image',
         'EPSG': None,
         'resample': None,
-        'source': 'LPDAAC'
+        'source': 'LPDAAC',
+        'userPwd': 'username:password'
     })
     output = ingest.start()
 
@@ -66,17 +68,20 @@ Available datasets
 * MODIS Thermal Anomalies & Fire: MCD45A1
 
 Further information to the datasets can be found at the following sites:
-* NASA LPDAAC: http://lpdaac.usgs.gov/products/modis_products_table
+* NASA LPDAAC: https://lpdaac.usgs.gov/dataset_discovery/modis/modis_products_table
 * NSIDC DAAC: http://nsidc.org/data/modis/data_summaries/index.html
 
 Several MODIS and especially Landsat datasets are also available using Google Earth Engine.
 
 Google Earth Engine
 -------------------
 
-If you have an account for the Google Earth Engine (https://earthengine.google.org), you can also use the Python bindings to have access to specific MODIS products and Landsat datasets.
+If you have an account for the Google Earth Engine (https://earthengine.google.org), you can also use the Python bindings to have access to specific MODIS products and Landsat datasets. To install the earthengine Python API see the following documentation:
+https://developers.google.com/earth-engine/python_install
+    pip install earthengine-api
 
-Please set the variables MY_SERVICE_ACCOUNT and MY_PRIVATE_KEY_FILE in gee_init.py to your service account username and the private key file.
+Please set the variables MY_SERVICE_ACCOUNT and MY_PRIVATE_KEY_FILE in gee_init.py to your service account username and the private key file. Therefore you need a service account that needs to be whitelisted first by the Google Earth Engine team. Please see the following documentation for more information:
+https://developers.google.com/earth-engine/service_account
 
 *Examples:*
 

pyEOM/datasets/MODIS.py

@@ -79,9 +79,26 @@ def TestDatasetLPDAAC():
     dataset.getProduct('MOD13Q1')
     dataset.source.download('', ['h12v08'])
 
+class LPDAACHTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+    def http_error_302(self, req, fp, code, msg, headers):
+        return urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
 
 class LPDAAC(HTTPSource):
     url = 'http://e4ftl01.cr.usgs.gov'
+    userpwd = None
+    listDirectoriesIndex = 3
+
+    def setUserPwd(self, login):
+        self.userpwd = login
+        from base64 import b64encode
+        userAndPass = b64encode(str.encode(self.userpwd)).decode("ascii")
+        self.http_header = {'Authorization': 'Basic %s' % userAndPass}
+        cookieprocessor = urllib2.HTTPCookieProcessor()
+        opener = urllib2.build_opener(LPDAACHTTPRedirectHandler, cookieprocessor)
+        urllib2.install_opener(opener)
+
+    def __init__(self):
+        super(LPDAAC, self).__init__(self.url)
 
 
 class NSIDC(FTPSource):
@@ -119,7 +136,7 @@ def addTask(self, dataset, geom, publishPath, start=None, end=None, qualityValue
     def __init__(self, task, dataset, source):
         LOGGER.debug('LPDAAC init')
         self.dataset = dataset
-        self.source = locals()[source]()
+        self.source = source
         if 'start' in task != "":
             self.taskStart = datetime.strptime(task['start'], '%Y-%m-%d')
         if 'end' in task != "":

pyEOM/sources.py

@@ -1,7 +1,11 @@
 __author__ = 'Jonas Eberle <jonas.eberle@eberle-mail.de>'
 
 import pyEOM.gee_init as gee
-import ee
+try:
+    import ee
+    from ee import oauth
+except:
+    pass
 import sys
 import ogr
 
@@ -23,7 +27,10 @@ def __init__(self, username=None, password=None):
 
     def connect(self, username=None, password=None):
         if gee.MY_SERVICE_ACCOUNT != '' and gee.MY_PRIVATE_KEY_FILE != '':
-            ee.Initialize(ee.ServiceAccountCredentials(gee.MY_SERVICE_ACCOUNT, gee.MY_PRIVATE_KEY_FILE))
+            #ee.ServiceAccountCredentials(gee.MY_SERVICE_ACCOUNT, gee.MY_PRIVATE_KEY_FILE)
+            from oauth2client.service_account import ServiceAccountCredentials
+            authorization = ServiceAccountCredentials.from_p12_keyfile(gee.MY_SERVICE_ACCOUNT, gee.MY_PRIVATE_KEY_FILE, scopes=oauth.SCOPE)
+            ee.Initialize(authorization)
         elif username != None and password != None:
             pass
         else:
@@ -111,7 +118,7 @@ def listScenes(self, dataset, eeGeometry, start, end):
         return images
 
 
-import urllib
+import urllib, urllib2
 from bs4 import BeautifulSoup
 import re
 import os
@@ -136,6 +143,8 @@ class HTTPSource(object):
     directories = []
     filesObj = None
     files = []
+    listDirectoriesIndex = 1
+    http_header = {}
 
     def __init__(self, url):
         self.url = url
@@ -147,17 +156,17 @@ def listDirectories(self, directory):
         self.path = directory
         LOGGER.info('Retrieving url: '+self.url+directory)
         html = urllib.urlopen(self.url+directory).read()
-        self.directoryObj = BeautifulSoup(html)
+        self.directoryObj = BeautifulSoup(html, "lxml")
         directories = self.directoryObj.findAll(href=re.compile('/'))
-        directories = directories[1:]
+        directories = directories[self.listDirectoriesIndex:]
         for dir in directories:
             self.directories.append(dir.string[0:-1])
         return self.directories
 
     def listFiles(self, directory):
         self.path = directory
         html = urllib.urlopen(self.url+directory).read()
-        self.filesObj = BeautifulSoup(html)
+        self.filesObj = BeautifulSoup(html, "lxml")
 
     def filterFiles(self, regexp, urls=True):
         self.files = self.filesObj.findAll(text=re.compile(regexp))
@@ -169,8 +178,13 @@ def downloadFile(self, url, path):
         LOGGER.info('Download: '+url)
         filepath = path+'/'+url.split('/')[-1]
         if not os.path.isfile(filepath):
-            download = urllib.urlretrieve(url, filepath)
-            return download[0]
+            #download = urllib.urlretrieve(url, filepath)
+            filSave = open(filepath, "wb")
+            req = urllib2.Request(url, headers=self.http_header)
+            http = urllib2.urlopen(req)
+            filSave.write(http.read())
+            filSave.close()
+            return filepath
         else:
             return filepath
 

pyEOM/tasks.py

@@ -5,6 +5,7 @@
 from pyEOM import log
 import os
 import sys
+import importlib
 import ConfigParser
 
 LOGGER = log.LOGGER
@@ -102,11 +103,12 @@ def start(self):
         LOGGER.debug('Dataset: '+dataset[0])
         LOGGER.debug('Product: '+dataset[1])
         if "MODIS" == dataset[0]:
-            from datasets.predefined.MODIS import *
-            if dataset[1] not in locals():
-                raise Exception('Dataset Class is not available!')
+            try:
+                dataset = getattr(importlib.import_module('pyEOM.datasets.predefined.MODIS.'+dataset[1]), 'Dataset')
+            except Exception as e:
+                raise Exception('Dataset Class is not available: '+str(e))
                 sys.exit(1)
-            self.processing['dataset'] = locals()[dataset[1]].Dataset()
+            self.processing['dataset'] = dataset()
 
             if len(self.processing['dataset'].sources) == 0:
                 raise Exception('No sources available!')
@@ -127,7 +129,10 @@ def start(self):
             try:
                 LOGGER.debug('Try to get source class: '+source)
                 if source == 'LPDAAC' or source == 'NSIDC':
-                    self.processing['source'] = getattr(MODIS, 'MODISHDF')(self.task, self.processing['dataset'], source)
+                    sourceObj = getattr(MODIS, source)()
+                    if source == 'LPDAAC':
+                        sourceObj.setUserPwd(self.task['userPwd'])
+                    self.processing['source'] = getattr(MODIS, 'MODISHDF')(self.task, self.processing['dataset'], sourceObj)
                 else:
                     self.processing['source'] = getattr(MODIS, source)(self.task, self.processing['dataset'])
             except NameError, e: