diff --git a/GeoHealthCheck/config_main.py b/GeoHealthCheck/config_main.py index 522e33e7..b0761efe 100644 --- a/GeoHealthCheck/config_main.py +++ b/GeoHealthCheck/config_main.py @@ -154,7 +154,7 @@ 'probe_class': 'GeoHealthCheck.plugins.probe.sta.StaCaps' }, 'OGC:WFS3': { - 'probe_class': 'GeoHealthCheck.plugins.probe.wfs3.WFS3Drilldown' + 'probe_class': 'GeoHealthCheck.plugins.probe.wfs3.WFS3Caps' }, 'ESRI:FS': { 'probe_class': 'GeoHealthCheck.plugins.probe.esrifs.ESRIFSDrilldown' diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 1d7e7bfb..90c8368c 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -238,7 +238,7 @@ def sniff_test_resource(config, resource_type, url): if resource_type == 'OGC:STA': title = 'OGC STA' elif resource_type == 'OGC:WFS3': - title = 'OGC WFS3' + title = 'OGC WFS3 (OAPIF)' elif resource_type == 'ESRI:FS': title = 'ESRI ArcGIS FS' else: @@ -279,8 +279,8 @@ def geonode_get_ows(base_url): base_name = 'GeoNode {}: {{}}'.format(url.hostname) status_code = r.getcode() if status_code != 200: - msg = "Errorous response from GeoNode at {}: {}".format(base_url, - r.text) + msg = "Error response from GeoNode at {}: {}".format( + base_url, r.text) raise ValueError(msg) try: diff --git a/GeoHealthCheck/plugins/probe/tms.py b/GeoHealthCheck/plugins/probe/tms.py index c912b37b..c5a4c36b 100644 --- a/GeoHealthCheck/plugins/probe/tms.py +++ b/GeoHealthCheck/plugins/probe/tms.py @@ -110,7 +110,8 @@ def get_metadata(self, resource, version='1.0.0'): :param version: :return: Metadata object """ - return TileMapService(resource.url, version=version) + return TileMapService(resource.url, version=version, + headers=self.get_request_headers()) # Overridden: expand param-ranges from WMS metadata def expand_params(self, resource): diff --git a/GeoHealthCheck/plugins/probe/wfs.py b/GeoHealthCheck/plugins/probe/wfs.py index ea03ae7a..bdb3b120 100644 --- a/GeoHealthCheck/plugins/probe/wfs.py +++ b/GeoHealthCheck/plugins/probe/wfs.py @@ -127,7 +127,9 @@ def get_metadata(self, resource, version='1.1.0'): :param version: :return: Metadata object """ - return WebFeatureService(resource.url, version=version) + return WebFeatureService(resource.url, + version=version, + headers=self.get_request_headers()) # Overridden: expand param-ranges from WFS metadata def expand_params(self, resource): diff --git a/GeoHealthCheck/plugins/probe/wfs3.py b/GeoHealthCheck/plugins/probe/wfs3.py index 26080ab2..747968ec 100644 --- a/GeoHealthCheck/plugins/probe/wfs3.py +++ b/GeoHealthCheck/plugins/probe/wfs3.py @@ -1,25 +1,64 @@ -import requests -from owslib.wfs import WebFeatureService +from owslib.ogcapi.features import Features from openapi_spec_validator import openapi_v3_spec_validator from GeoHealthCheck.probe import Probe from GeoHealthCheck.result import Result, push_result +class WFS3Caps(Probe): + """Probe for OGC WFS3 API (OAFeat) main endpoint url""" + + NAME = 'OGC WFS3 (OAFeat) API Capabilities' + DESCRIPTION = 'Perform OGC WFS3 (OAFeat) API Capabilities ' \ + 'Operation and check validity' + RESOURCE_TYPE = 'OGC:WFS3' + + REQUEST_METHOD = 'GET' + REQUEST_HEADERS = {'Accept': 'application/json'} + + # e.g. https://demo.pygeoapi.io/master + REQUEST_TEMPLATE = '' + + def __init__(self): + Probe.__init__(self) + + CHECKS_AVAIL = { + 'GeoHealthCheck.plugins.check.checks.HttpStatusNoError': { + 'default': True + }, + 'GeoHealthCheck.plugins.check.checks.JsonParse': { + 'default': True + }, + 'GeoHealthCheck.plugins.check.checks.ContainsStrings': { + 'set_params': { + 'strings': { + 'name': 'Contains required strings', + 'value': ['/conformance', '/collections', + 'service', 'links'] + } + }, + 'default': True + }, + } + """Check for OGC WFS3 API (OGC API Features) service endpoint + availability""" + + class WFS3Drilldown(Probe): """ - Probe for WFS3 endpoint "drilldown": starting + Probe for WFS3 (OpenAPI Features or OAFeat) endpoint "drilldown": starting with top endpoint: get Collections and do - GetItems on them etc. Using OWSLib.WebFeatureService. + GetItems on them etc. Using OWSLib owslib.ogcapi package. - TODO: needs finalization. + TODO: needs renaming: WFS3 is now OAFeat. """ - NAME = 'WFS3 Drilldown' - DESCRIPTION = 'Traverses a OGC WFS3 (REST) API endpoint by drilling down' + NAME = 'WFS3 (OAFeat) Drilldown' + DESCRIPTION = 'Traverses a OGC WFS3 (OAFeat) API endpoint by drilling down' RESOURCE_TYPE = 'OGC:WFS3' REQUEST_METHOD = 'GET' + REQUEST_HEADERS = {'Accept': 'application/json'} PARAM_DEFS = { 'drilldown_level': { @@ -43,14 +82,15 @@ def perform_request(self): See https://github.com/geopython/OWSLib/blob/ master/tests/doctests/wfs3_GeoServerCapabilities.txt """ - wfs3 = None + oa_feat = None collections = None # 1.1 Test Landing Page result = Result(True, 'Test Landing Page') result.start() try: - wfs3 = WebFeatureService(self._resource.url, version='3.0') + oa_feat = Features(self._resource.url, + headers=self.get_request_headers()) except Exception as err: result.set(False, '%s:%s' % (result.message, str(err))) @@ -61,7 +101,7 @@ def perform_request(self): result = Result(True, 'conformance endpoint exists') result.start() try: - wfs3.conformance() + oa_feat.conformance() except Exception as err: result.set(False, str(err)) @@ -72,7 +112,7 @@ def perform_request(self): result = Result(True, 'Get collections') result.start() try: - collections = wfs3.collections() + collections = oa_feat.collections()['collections'] except Exception as err: result.set(False, '%s:%s' % (result.message, str(err))) @@ -84,11 +124,10 @@ def perform_request(self): result.start() try: - # TODO: OWSLib 0.17.1 has no call to '/api yet, upgrade when GHC - # supports Py3 to 0.19+. - api = wfs3_api_doc(wfs3) + # OWSLib 0.20.0+ has call to '/api now. + api_doc = oa_feat.api() for attr in ['components', 'paths', 'openapi']: - val = api.get(attr, None) + val = api_doc.get(attr, None) if val is None: msg = 'missing attr: %s' % attr result = push_result( @@ -117,7 +156,7 @@ def perform_request(self): coll_id = coll_id try: - coll = wfs3.collection(coll_id) + coll = oa_feat.collection(coll_id) # TODO: Maybe also add crs for attr in ['id', 'links']: @@ -136,7 +175,7 @@ def perform_request(self): continue try: - items = wfs3.collection_items(coll_id, limit=1) + items = oa_feat.collection_items(coll_id, limit=1) except Exception as e: msg = 'GetItems %s: OWSLib err: %s ' % (str(e), coll_id) result = push_result( @@ -162,7 +201,7 @@ def perform_request(self): fid = items['features'][0]['id'] try: - item = wfs3.collection_item(coll_id, fid) + item = oa_feat.collection_item(coll_id, fid) except Exception as e: msg = 'GetItem %s: OWSLib err: %s' \ % (str(e), coll_id) @@ -199,13 +238,13 @@ def perform_request(self): class WFS3OpenAPIValidator(Probe): """ - Probe for WFS3 OpenAPI Spec Validation (/api endpoint). + Probe for WFS3 (OAFeat) OpenAPI Spec Validation (/api endpoint). Uses https://pypi.org/project/openapi-spec-validator/. """ - NAME = 'WFS3 OpenAPI Validator' - DESCRIPTION = 'Validates WFS3 /api endpoint for OpenAPI compliance' + NAME = 'WFS3 (OAFeat) OpenAPI Validator' + DESCRIPTION = 'Validates WFS3 (OAFeat) api endpoint for OpenAPI compliance' RESOURCE_TYPE = 'OGC:WFS3' REQUEST_METHOD = 'GET' @@ -227,16 +266,17 @@ def perform_request(self): result.start() api_doc = None try: - wfs3 = WebFeatureService(self._resource.url, version='3.0') + oa_feat = Features(self._resource.url, + headers=self.get_request_headers()) - # TODO: OWSLib 0.17.1 has no call to '/api yet. - api_doc = wfs3_api_doc(wfs3) + # OWSLib 0.20.0 has call to OpenAPI doc now. + api_doc = oa_feat.api() # Basic sanity check for attr in ['components', 'paths', 'openapi']: val = api_doc.get(attr, None) if val is None: - msg = '/api: missing attr: %s' % attr + msg = 'OpenAPI doc: missing attr: %s' % attr result.set(False, msg) break except Exception as err: @@ -249,7 +289,7 @@ def perform_request(self): if api_doc is None or result.success is False: return - # ASSERTION: /api exists, next OpenAPI Validation + # ASSERTION: OpenAPI doc exists, next OpenAPI Validation # Step 2 detailed OpenAPI Compliance test result = Result(True, 'Validate OpenAPI Compliance') @@ -270,64 +310,8 @@ def perform_request(self): # Add to overall Probe result self.result.add_result(result) - -def wfs3_api_doc(wfs3): - """ - implements Requirement 3 (/req/core/api-definition-op) - @returns: OpenAPI definition object - """ - - api_url = None - - for link in wfs3.links: - if link['rel'] == 'service-desc': - api_url = link['href'] - - if not api_url: - raise RuntimeError('Did not find service-desc link in landing page') - - return requests.get(api_url).json() - - -# class WFS3Caps(Probe): -# """Probe for OGC WFS3 API main endpoint url""" -# -# NAME = 'OGC WFS3 API Capabilities' -# DESCRIPTION = 'Perform OGC WFS3 API Capabilities -# Operation and check validity' -# RESOURCE_TYPE = 'OGC:WFS3' -# -# REQUEST_METHOD = 'GET' -# -# # e.g. https://demo.pygeoapi.io/master/collections?f=json -# REQUEST_TEMPLATE = '/{endpoint}?f=json' -# -# def __init__(self): -# Probe.__init__(self) -# -# PARAM_DEFS = { -# 'endpoint': { -# 'type': 'string', -# 'description': 'The OGC WFS3 API service endpoint type', -# 'default': '/collections', -# 'required': True, -# 'range': ['collections', 'conformance', 'api'] -# } -# } -# """Param defs""" -# -# CHECKS_AVAIL = { -# 'GeoHealthCheck.plugins.check.checks.HttpStatusNoError': { -# 'default': True -# }, -# 'GeoHealthCheck.plugins.check.checks.JsonParse': { -# 'default': True -# } -# } -# """Check for OGC WFS3 API OGC WFS3 API service endpoint -# availability""" -# -# +# TODO implement: similar to single WMS layer, +# expand params with collection names etc. # class WFS3Collection(Probe): # """Probe for OGC WFS3 API main endpoint url""" # diff --git a/GeoHealthCheck/util.py b/GeoHealthCheck/util.py index 9d323248..42362506 100644 --- a/GeoHealthCheck/util.py +++ b/GeoHealthCheck/util.py @@ -283,3 +283,16 @@ def create_requests_retry_session( session.mount('http://', adapter) session.mount('https://', adapter) return session + + +# Optionally expand a URL with query clause like 'f=json' +def expand_url(url, query_clause): + if query_clause in url: + # Already present + return url + + # Determine concatenator char + concat = '?' + if concat in url: + concat = '&' + return '{}{}{}'.format(url, concat, query_clause) diff --git a/requirements.txt b/requirements.txt index afd54543..d097e25e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ Flask-Script==2.0.6 SQLAlchemy==1.3.8 Flask-SQLAlchemy==2.4.0 itsdangerous==1.1.0 -pyproj==1.9.6 -OWSLib==0.17.1 # update to new version when solution to OWSLIB #614 is released +pyproj==2.4.1 +OWSLib==0.20.0 jsonschema==3.0.2 # downgrade from 3.2.0 on sept 29, 2020, issue 331, consider better fix openapi-spec-validator==0.2.8 Sphinx==2.2.0 diff --git a/tests/data/fixtures.json b/tests/data/fixtures.json index 4695ff2a..5e0ad26b 100644 --- a/tests/data/fixtures.json +++ b/tests/data/fixtures.json @@ -92,7 +92,7 @@ "resource_type": "OSGeo:WFS3", "active": true, "title": "pygeoapi - master", - "url": "https://demo.pygeoapi.io/stable", + "url": "https://demo.pygeoapi.io/master", "tags": [ "ogc" ]