diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de37862c6..3368f985c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,7 +109,7 @@ jobs: env: POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }} run: | - pytest tests/test_api.py + pytest tests/api pytest tests/test_api_ogr_provider.py pytest tests/test_config.py pytest tests/test_csv__formatter.py diff --git a/docs/source/development.rst b/docs/source/development.rst index 561f08349..59941d780 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -20,7 +20,7 @@ Tests can be run locally as part of development workflow. They are also run on `GitHub Actions setup`_ against all commits and pull requests to the code repository. To run all tests, simply run ``pytest`` in the repository. To run a specific test file, -run ``pytest tests/test_api.py``, for example. +run ``pytest tests/api/test_itemtypes.py``, for example. CQL extension lifecycle diff --git a/pygeoapi/api.py b/pygeoapi/api.py deleted file mode 100644 index 87d672cce..000000000 --- a/pygeoapi/api.py +++ /dev/null @@ -1,4319 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# Francesco Bartoli -# Sander Schaminee -# John A Stevenson -# Colin Blackburn -# Ricardo Garcia Silva -# -# Copyright (c) 2024 Tom Kralidis -# Copyright (c) 2022 Francesco Bartoli -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# Copyright (c) 2023 Ricardo Garcia Silva -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= -""" Root level code of pygeoapi, parsing content provided by web framework. -Returns content from plugins and sets responses. -""" - -import asyncio -from collections import OrderedDict -from copy import deepcopy -from datetime import datetime, timezone -from functools import partial -from gzip import compress -from http import HTTPStatus -import json -import logging -import re -from typing import Any, Tuple, Union, Optional -import urllib.parse - -from dateutil.parser import parse as dateparse -from pygeofilter.parsers.ecql import parse as parse_ecql_text -from pygeofilter.parsers.cql_json import parse as parse_cql_json -from pyproj.exceptions import CRSError -import pytz -from shapely.errors import WKTReadingError -from shapely.wkt import loads as shapely_loads - -from pygeoapi import __version__, l10n -from pygeoapi.formatter.base import FormatterSerializationError -from pygeoapi.linked_data import (geojson2jsonld, jsonldify, - jsonldify_collection) -from pygeoapi.log import setup_logger -from pygeoapi.process.base import ( - JobNotFoundError, - JobResultNotFoundError, - ProcessorExecuteError, -) -from pygeoapi.process.manager.base import get_manager -from pygeoapi.plugin import load_plugin, PLUGINS -from pygeoapi.provider.base import ( - ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, - ProviderTypeError) -from pygeoapi.models.provider.base import (TilesMetadataFormat, - TileMatrixSetEnum) - -from pygeoapi.models.cql import CQLModel -from pygeoapi.util import (dategetter, RequestedProcessExecutionMode, - DATETIME_FORMAT, UrlPrefetcher, - filter_dict_by_key_value, get_provider_by_type, - get_provider_default, get_typed_value, JobStatus, - json_serial, render_j2_template, str2bool, - TEMPLATES, to_json, get_api_rules, get_base_url, - get_crs_from_uri, get_supported_crs_list, - modify_pygeofilter, CrsTransformSpec, - transform_bbox, Subscriber) - -LOGGER = logging.getLogger(__name__) - -#: Return headers for requests (e.g:X-Powered-By) -HEADERS = { - 'Content-Type': 'application/json', - 'X-Powered-By': f'pygeoapi {__version__}' -} - -CHARSET = ['utf-8'] -F_JSON = 'json' -F_HTML = 'html' -F_JSONLD = 'jsonld' -F_GZIP = 'gzip' -F_PNG = 'png' -F_JPEG = 'jpeg' -F_MVT = 'mvt' -F_NETCDF = 'NetCDF' - -#: Formats allowed for ?f= requests (order matters for complex MIME types) -FORMAT_TYPES = OrderedDict(( - (F_HTML, 'text/html'), - (F_JSONLD, 'application/ld+json'), - (F_JSON, 'application/json'), - (F_PNG, 'image/png'), - (F_JPEG, 'image/jpeg'), - (F_MVT, 'application/vnd.mapbox-vector-tile'), - (F_NETCDF, 'application/x-netcdf'), -)) - -#: Locale used for system responses (e.g. exceptions) -SYSTEM_LOCALE = l10n.Locale('en', 'US') - -CONFORMANCE = { - 'common': [ - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' - ], - 'feature': [ - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', - 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa - 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa - 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', - 'http://www.opengis.net/spec/ogcapi-features-5/1.0/req/core-roles-features' # noqa - ], - 'coverage': [ - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' # noqa - ], - 'map': [ - 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' - ], - 'tile': [ - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' - ], - 'record': [ - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' - ], - 'process': [ - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30' - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback', - ], - 'edr': [ - 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' - ] -} - -OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' - -DEFAULT_CRS_LIST = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', -] - -DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' -DEFAULT_STORAGE_CRS = DEFAULT_CRS - - -def pre_process(func): - """ - Decorator that transforms an incoming Request instance specific to the - web framework (i.e. Flask, Starlette or Django) into a generic - :class:`APIRequest` instance. - - :param func: decorated function - - :returns: `func` - """ - - def inner(*args): - cls, req_in = args[:2] - req_out = APIRequest.with_data(req_in, getattr(cls, 'locales', set())) - if len(args) > 2: - return func(cls, req_out, *args[2:]) - else: - return func(cls, req_out) - - return inner - - -def gzip(func): - """ - Decorator that compresses the content of an outgoing API result - instance if the Content-Encoding response header was set to gzip. - - :param func: decorated function - - :returns: `func` - """ - - def inner(*args, **kwargs): - headers, status, content = func(*args, **kwargs) - charset = CHARSET[0] - if F_GZIP in headers.get('Content-Encoding', []): - try: - if isinstance(content, bytes): - # bytes means Content-Type needs to be set upstream - content = compress(content) - else: - headers['Content-Type'] = \ - f"{headers['Content-Type']}; charset={charset}" - content = compress(content.encode(charset)) - except TypeError as err: - headers.pop('Content-Encoding') - LOGGER.error(f'Error in compression: {err}') - - return headers, status, content - - return inner - - -class APIRequest: - """ - Transforms an incoming server-specific Request into an object - with some generic helper methods and properties. - - .. note:: Typically, this instance is created automatically by the - :func:`pre_process` decorator. **Every** API method that has - been routed to a REST endpoint should be decorated by the - :func:`pre_process` function. - Therefore, **all** routed API methods should at least have 1 - argument that holds the (transformed) request. - - The following example API method will: - - - transform the incoming Flask/Starlette/Django `Request` into an - `APIRequest`using the :func:`pre_process` decorator; - - call :meth:`is_valid` to check if the incoming request was valid, i.e. - that the user requested a valid output format or no format at all - (which means the default format); - - call :meth:`API.get_format_exception` if the requested format was - invalid; - - create a `dict` with the appropriate `Content-Type` header for the - requested format and a `Content-Language` header if any specific language - was requested. - - .. code-block:: python - - @pre_process - def example_method(self, request: Union[APIRequest, Any], custom_arg): - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers() - - # generate response_body here - - return headers, HTTPStatus.OK, response_body - - - The following example API method is similar as the one above, but will also - allow the user to request a non-standard format (e.g. ``f=xml``). - If `xml` was requested, we set the `Content-Type` ourselves. For the - standard formats, the `APIRequest` object sets the `Content-Type`. - - .. code-block:: python - - @pre_process - def example_method(self, request: Union[APIRequest, Any], custom_arg): - if not request.is_valid(['xml']): - return self.get_format_exception(request) - - content_type = 'application/xml' if request.format == 'xml' else None - headers = request.get_response_headers(content_type) - - # generate response_body here - - return headers, HTTPStatus.OK, response_body - - Note that you don't *have* to call :meth:`is_valid`, but that you can also - perform a custom check on the requested output format by looking at the - :attr:`format` property. - Other query parameters are available through the :attr:`params` property as - a `dict`. The request body is available through the :attr:`data` property. - - .. note:: If the request data (body) is important, **always** create a - new `APIRequest` instance using the :meth:`with_data` factory - method. - The :func:`pre_process` decorator will use this automatically. - - :param request: The web platform specific Request instance. - :param supported_locales: List or set of supported Locale instances. - """ - def __init__(self, request, supported_locales): - # Set default request data - self._data = b'' - - # Copy request query parameters - self._args = self._get_params(request) - - # Get path info - if hasattr(request, 'scope'): - self._path_info = request.scope['path'].strip('/') - elif hasattr(request.headers, 'environ'): - self._path_info = request.headers.environ['PATH_INFO'].strip('/') - elif hasattr(request, 'path_info'): - self._path_info = request.path_info - - # Extract locale from params or headers - self._raw_locale, self._locale = self._get_locale(request.headers, - supported_locales) - - # Determine format - self._format = self._get_format(request.headers) - - # Get received headers - self._headers = self.get_request_headers(request.headers) - - @classmethod - def with_data(cls, request, supported_locales) -> 'APIRequest': - """ - Factory class method to create an `APIRequest` instance with data. - - If the request body is required, an `APIRequest` should always be - instantiated using this class method. The reason for this is, that the - Starlette request body needs to be awaited (async), which cannot be - achieved in the :meth:`__init__` method of the `APIRequest`. - However, `APIRequest` can still be initialized using :meth:`__init__`, - but then the :attr:`data` property value will always be empty. - - :param request: The web platform specific Request instance. - :param supported_locales: List or set of supported Locale instances. - :returns: An `APIRequest` instance with data. - """ - - api_req = cls(request, supported_locales) - if hasattr(request, 'data'): - # Set data from Flask request - api_req._data = request.data - elif hasattr(request, 'body'): - if 'django' in str(request.__class__): - # Set data from Django request - api_req._data = request.body - else: - # Set data from Starlette request after async - # coroutine completion - # TODO: - # this now blocks, but once Flask v2 with async support - # has been implemented, with_data() can become async too - loop = asyncio.get_event_loop() - api_req._data = asyncio.run_coroutine_threadsafe( - request.body(), loop).result(1) - return api_req - - @staticmethod - def _get_params(request): - """ - Extracts the query parameters from the `Request` object. - - :param request: A Flask or Starlette Request instance - :returns: `ImmutableMultiDict` or empty `dict` - """ - - if hasattr(request, 'args'): - # Return ImmutableMultiDict from Flask request - return request.args - elif hasattr(request, 'query_params'): - # Return ImmutableMultiDict from Starlette request - return request.query_params - elif hasattr(request, 'GET'): - # Return QueryDict from Django GET request - return request.GET - elif hasattr(request, 'POST'): - # Return QueryDict from Django GET request - return request.POST - LOGGER.debug('No query parameters found') - return {} - - def _get_locale(self, headers, supported_locales): - """ - Detects locale from "lang=" param or `Accept-Language` - header. Returns a tuple of (raw, locale) if found in params or headers. - Returns a tuple of (raw default, default locale) if not found. - - :param headers: A dict with Request headers - :param supported_locales: List or set of supported Locale instances - :returns: A tuple of (str, Locale) - """ - - raw = None - try: - default_locale = l10n.str2locale(supported_locales[0]) - except (TypeError, IndexError, l10n.LocaleError) as err: - # This should normally not happen, since the API class already - # loads the supported languages from the config, which raises - # a LocaleError if any of these languages are invalid. - LOGGER.error(err) - raise ValueError(f"{self.__class__.__name__} must be initialized" - f"with a list of valid supported locales") - - for func, mapping in ((l10n.locale_from_params, self._args), - (l10n.locale_from_headers, headers)): - loc_str = func(mapping) - if loc_str: - if not raw: - # This is the first-found locale string: set as raw - raw = loc_str - # Check if locale string is a good match for the UI - loc = l10n.best_match(loc_str, supported_locales) - is_override = func is l10n.locale_from_params - if loc != default_locale or is_override: - return raw, loc - - return raw, default_locale - - def _get_format(self, headers) -> Union[str, None]: - """ - Get `Request` format type from query parameters or headers. - - :param headers: Dict of Request headers - :returns: format value or None if not found/specified - """ - - # Optional f=html or f=json query param - # Overrides Accept header and might differ from FORMAT_TYPES - format_ = (self._args.get('f') or '').strip() - if format_: - return format_ - - # Format not specified: get from Accept headers (MIME types) - # e.g. format_ = 'text/html' - h = headers.get('accept', headers.get('Accept', '')).strip() # noqa - (fmts, mimes) = zip(*FORMAT_TYPES.items()) - # basic support for complex types (i.e. with "q=0.x") - for type_ in (t.split(';')[0].strip() for t in h.split(',') if t): - if type_ in mimes: - idx_ = mimes.index(type_) - format_ = fmts[idx_] - break - - return format_ or None - - @property - def data(self) -> bytes: - """Returns the additional data send with the Request (bytes)""" - return self._data - - @property - def params(self) -> dict: - """Returns the Request query parameters dict""" - return self._args - - @property - def path_info(self) -> str: - """Returns the web server request path info part""" - return self._path_info - - @property - def locale(self) -> l10n.Locale: - """ - Returns the user-defined locale from the request object. - If no locale has been defined or if it is invalid, - the default server locale is returned. - - .. note:: The locale here determines the language in which pygeoapi - should return its responses. This may not be the language - that the user requested. It may also not be the language - that is supported by a collection provider, for example. - For this reason, you should pass the `raw_locale` property - to the :func:`l10n.get_plugin_locale` function, so that - the best match for the provider can be determined. - - :returns: babel.core.Locale - """ - - return self._locale - - @property - def raw_locale(self) -> Union[str, None]: - """ - Returns the raw locale string from the `Request` object. - If no "lang" query parameter or `Accept-Language` header was found, - `None` is returned. - Pass this value to the :func:`l10n.get_plugin_locale` function to let - the provider determine a best match for the locale, which may be - different from the locale used by pygeoapi's UI. - - :returns: a locale string or None - """ - - return self._raw_locale - - @property - def format(self) -> Union[str, None]: - """ - Returns the content type format from the - request query parameters or headers. - - :returns: Format name or None - """ - - return self._format - - @property - def headers(self) -> dict: - """ - Returns the dictionary of the headers from - the request. - - :returns: Request headers dictionary - """ - - return self._headers - - def get_linkrel(self, format_: str) -> str: - """ - Returns the hyperlink relationship (rel) attribute value for - the given API format string. - - The string is compared against the request format and if it matches, - the value 'self' is returned. Otherwise, 'alternate' is returned. - However, if `format_` is 'json' and *no* request format was found, - the relationship 'self' is returned as well (JSON is the default). - - :param format_: The format to compare the request format against. - :returns: A string 'self' or 'alternate'. - """ - - fmt = format_.lower() - if fmt == self._format or (fmt == F_JSON and not self._format): - return 'self' - return 'alternate' - - def is_valid(self, additional_formats=None) -> bool: - """ - Returns True if: - - the format is not set (None) - - the requested format is supported - - the requested format exists in a list if additional formats - - .. note:: Format names are matched in a case-insensitive manner. - - :param additional_formats: Optional additional supported formats list - - :returns: bool - """ - - if not self._format: - return True - if self._format in FORMAT_TYPES.keys(): - return True - if self._format in (f.lower() for f in (additional_formats or ())): - return True - return False - - def get_response_headers(self, force_lang: l10n.Locale = None, - force_type: str = None, - force_encoding: str = None, - **custom_headers) -> dict: - """ - Prepares and returns a dictionary with Response object headers. - - This method always adds a 'Content-Language' header, where the value - is determined by the 'lang' query parameter or 'Accept-Language' - header from the request. - If no language was requested, the default pygeoapi language is used, - unless a `force_lang` override was specified (see notes below). - - A 'Content-Type' header is also always added to the response. - If the user does not specify `force_type`, the header is based on - the `format` APIRequest property. If that is invalid, the default MIME - type `application/json` is used. - - ..note:: If a `force_lang` override is applied, that language - is always set as the 'Content-Language', regardless of - a 'lang' query parameter or 'Accept-Language' header. - If an API response always needs to be in the same - language, 'force_lang' should be set to that language. - - :param force_lang: An optional Content-Language header override. - :param force_type: An optional Content-Type header override. - :param force_encoding: An optional Content-Encoding header override. - :returns: A header dict - """ - - headers = HEADERS.copy() - headers.update(**custom_headers) - l10n.set_response_language(headers, force_lang or self._locale) - if force_type: - # Set custom MIME type if specified - headers['Content-Type'] = force_type - elif self.is_valid() and self._format: - # Set MIME type for valid formats - headers['Content-Type'] = FORMAT_TYPES[self._format] - - if F_GZIP in FORMAT_TYPES: - if force_encoding: - headers['Content-Encoding'] = force_encoding - elif F_GZIP in self._headers.get('Accept-Encoding', ''): - headers['Content-Encoding'] = F_GZIP - - return headers - - def get_request_headers(self, headers) -> dict: - """ - Obtains and returns a dictionary with Request object headers. - - This method adds the headers of the original request and - makes them available to the API object. - - :returns: A header dict - """ - - headers_ = {item[0]: item[1] for item in headers.items()} - return headers_ - - -class API: - """API object""" - - def __init__(self, config, openapi): - """ - constructor - - :param config: configuration dict - :param openapi: openapi dict - - :returns: `pygeoapi.API` instance - """ - - self.config = config - self.openapi = openapi - self.api_headers = get_api_rules(self.config).response_headers - self.base_url = get_base_url(self.config) - self.prefetcher = UrlPrefetcher() - - CHARSET[0] = config['server'].get('encoding', 'utf-8') - if config['server'].get('gzip'): - FORMAT_TYPES[F_GZIP] = 'application/gzip' - FORMAT_TYPES.move_to_end(F_JSON) - - # Process language settings (first locale is default!) - self.locales = l10n.get_locales(config) - self.default_locale = self.locales[0] - - if 'templates' not in self.config['server']: - self.config['server']['templates'] = {'path': TEMPLATES} - - if 'pretty_print' not in self.config['server']: - self.config['server']['pretty_print'] = False - - self.pretty_print = self.config['server']['pretty_print'] - - setup_logger(self.config['logging']) - - # Create config clone for HTML templating with modified base URL - self.tpl_config = deepcopy(self.config) - self.tpl_config['server']['url'] = self.base_url - - self.manager = get_manager(self.config) - LOGGER.info('Process manager plugin loaded') - - @gzip - @pre_process - @jsonldify - def landing_page(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide API landing page - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - fcm = { - 'links': [], - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': - l10n.translate( - self.config['metadata']['identification']['description'], - request.locale) - } - - LOGGER.debug('Creating links') - # TODO: put title text in config or translatable files? - fcm['links'] = [{ - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'This document as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f"{self.base_url}?f={F_JSONLD}" - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f"{self.base_url}?f={F_HTML}", - 'hreflang': self.default_locale - }, { - 'rel': 'service-desc', - 'type': 'application/vnd.oai.openapi+json;version=3.0', - 'title': 'The OpenAPI definition as JSON', - 'href': f"{self.base_url}/openapi" - }, { - 'rel': 'service-doc', - 'type': FORMAT_TYPES[F_HTML], - 'title': 'The OpenAPI definition as HTML', - 'href': f"{self.base_url}/openapi?f={F_HTML}", - 'hreflang': self.default_locale - }, { - 'rel': 'conformance', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Conformance', - 'href': f"{self.base_url}/conformance" - }, { - 'rel': 'data', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Collections', - 'href': self.get_collections_url() - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Processes', - 'href': f"{self.base_url}/processes" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs', - 'href': f"{self.base_url}/jobs" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The list of supported tiling schemes (as JSON)', - 'href': f"{self.base_url}/TileMatrixSets?f=json" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', - 'type': FORMAT_TYPES[F_HTML], - 'title': 'The list of supported tiling schemes (as HTML)', - 'href': f"{self.base_url}/TileMatrixSets?f=html" - }] - - headers = request.get_response_headers(**self.api_headers) - if request.format == F_HTML: # render - - fcm['processes'] = False - fcm['stac'] = False - fcm['collection'] = False - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'process'): - fcm['processes'] = True - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection'): - fcm['stac'] = True - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'collection'): - fcm['collection'] = True - - content = render_j2_template(self.tpl_config, 'landing_page.html', - fcm, request.locale) - return headers, HTTPStatus.OK, content - - if request.format == F_JSONLD: - return headers, HTTPStatus.OK, to_json( - self.fcmld, self.pretty_print) - - return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - - @gzip - @pre_process - def openapi_(self, request: Union[APIRequest, Any]) -> Tuple[ - dict, int, str]: - """ - Provide OpenAPI document - - :param request: A request object - :param openapi: dict of OpenAPI definition - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - if request.format == F_HTML: - template = 'openapi/swagger.html' - if request._args.get('ui') == 'redoc': - template = 'openapi/redoc.html' - - path = f'{self.base_url}/openapi' - data = { - 'openapi-document-path': path - } - content = render_j2_template(self.tpl_config, template, data, - request.locale) - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/vnd.oai.openapi+json;version=3.0' # noqa - - if isinstance(self.openapi, dict): - return headers, HTTPStatus.OK, to_json(self.openapi, - self.pretty_print) - else: - return headers, HTTPStatus.OK, self.openapi - - @gzip - @pre_process - def conformance(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide conformance definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - conformance_list = CONFORMANCE['common'] - - for key, value in self.config['resources'].items(): - if value['type'] == 'process': - conformance_list.extend(CONFORMANCE[value['type']]) - else: - for provider in value['providers']: - if provider['type'] in CONFORMANCE: - conformance_list.extend(CONFORMANCE[provider['type']]) - - conformance = { - 'conformsTo': list(set(conformance_list)) - } - - headers = request.get_response_headers(**self.api_headers) - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, 'conformance.html', - conformance, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) - - @gzip - @pre_process - def tilematrixsets(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, - str]: - """ - Provide tileMatrixSets definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve available TileMatrixSets - enums = [e.value for e in TileMatrixSetEnum] - - tms = {"tileMatrixSets": []} - - for e in enums: - tms['tileMatrixSets'].append({ - "title": e.title, - "id": e.tileMatrixSet, - "uri": e.tileMatrixSetURI, - "links": [ - { - "rel": "self", - "type": "text/html", - "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa - }, - { - "rel": "self", - "type": "application/json", - "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa - } - ] - }) - - tms['links'] = [{ - "rel": "alternate", - "type": "text/html", - "title": "This document as HTML", - "href": f"{self.base_url}/tileMatrixSets?f=html" - }, { - "rel": "self", - "type": "application/json", - "title": "This document", - "href": f"{self.base_url}/tileMatrixSets?f=json" - }] - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/index.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - def tilematrixset(self, - request: Union[APIRequest, Any], - tileMatrixSetId) -> Tuple[dict, - int, str]: - """ - Provide tile matrix definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve relevant TileMatrixSet - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - try: - for e in enums: - if tileMatrixSetId == e.tileMatrixSet: - enum = e - if not enum: - raise ValueError('could not find this tilematrixset') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - tms = { - "title": enum.tileMatrixSet, - "crs": enum.crs, - "id": enum.tileMatrixSet, - "uri": enum.tileMatrixSetURI, - "orderedAxes": enum.orderedAxes, - "wellKnownScaleSet": enum.wellKnownScaleSet, - "tileMatrices": enum.tileMatrices - } - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/tilematrixset.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def describe_collections(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection metadata - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - fcm = { - 'collections': [], - 'links': [] - } - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if all([dataset is not None, dataset not in collections.keys()]): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - if dataset is not None: - collections_dict = { - k: v for k, v in collections.items() if k == dataset - } - else: - collections_dict = collections - - LOGGER.debug('Creating collections') - for k, v in collections_dict.items(): - if v.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - collection_data = get_provider_default(v['providers']) - collection_data_type = collection_data['type'] - - collection_data_format = None - - if 'format' in collection_data: - collection_data_format = collection_data['format'] - - is_vector_tile = (collection_data_type == 'tile' and - collection_data_format['name'] not - in [F_PNG, F_JPEG]) - - collection = { - 'id': k, - 'title': l10n.translate(v['title'], request.locale), - 'description': l10n.translate(v['description'], request.locale), # noqa - 'keywords': l10n.translate(v['keywords'], request.locale), - 'links': [] - } - - bbox = v['extents']['spatial']['bbox'] - # The output should be an array of bbox, so if the user only - # provided a single bbox, wrap it in a array. - if not isinstance(bbox[0], list): - bbox = [bbox] - collection['extent'] = { - 'spatial': { - 'bbox': bbox - } - } - if 'crs' in v['extents']['spatial']: - collection['extent']['spatial']['crs'] = \ - v['extents']['spatial']['crs'] - - t_ext = v.get('extents', {}).get('temporal', {}) - if t_ext: - begins = dategetter('begin', t_ext) - ends = dategetter('end', t_ext) - collection['extent']['temporal'] = { - 'interval': [[begins, ends]] - } - if 'trs' in t_ext: - collection['extent']['temporal']['trs'] = t_ext['trs'] - - LOGGER.debug('Processing configured collection links') - for link in l10n.translate(v.get('links', []), request.locale): - lnk = { - 'type': link['type'], - 'rel': link['rel'], - 'title': l10n.translate(link['title'], request.locale), - 'href': l10n.translate(link['href'], request.locale), - } - if 'hreflang' in link: - lnk['hreflang'] = l10n.translate( - link['hreflang'], request.locale) - content_length = link.get('length', 0) - - if lnk['rel'] == 'enclosure' and content_length == 0: - # Issue HEAD request for enclosure links without length - lnk_headers = self.prefetcher.get_headers(lnk['href']) - content_length = int(lnk_headers.get('content-length', 0)) - content_type = lnk_headers.get('content-type', lnk['type']) - if content_length == 0: - # Skip this (broken) link - LOGGER.debug(f"Enclosure {lnk['href']} is invalid") - continue - if content_type != lnk['type']: - # Update content type if different from specified - lnk['type'] = content_type - LOGGER.debug( - f"Fixed media type for enclosure {lnk['href']}") - - if content_length > 0: - lnk['length'] = content_length - - collection['links'].append(lnk) - - # TODO: provide translations - LOGGER.debug('Adding JSON and HTML link relations') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSON}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSONLD}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{k}?f={F_HTML}' - }) - - if collection_data_type in ['feature', 'coverage', 'record']: - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'{OGC_RELTYPES_BASE}/schema', - 'title': 'Schema of collection in JSON', - 'href': f'{self.get_collections_url()}/{k}/schema?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'{OGC_RELTYPES_BASE}/schema', - 'title': 'Schema of collection in HTML', - 'href': f'{self.get_collections_url()}/{k}/schema?f={F_HTML}' # noqa - }) - - if is_vector_tile or collection_data_type in ['feature', 'record']: - # TODO: translate - collection['itemType'] = collection_data_type - LOGGER.debug('Adding feature/record based links') - collection['links'].append({ - 'type': 'application/schema+json', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', - 'title': 'Queryables for this collection as JSON', - 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', - 'title': 'Queryables for this collection as HTML', - 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_HTML}' # noqa - }) - collection['links'].append({ - 'type': 'application/geo+json', - 'rel': 'items', - 'title': 'items as GeoJSON', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': 'items', - 'title': 'items as RDF (GeoJSON-LD)', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSONLD}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'items', - 'title': 'Items as HTML', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_HTML}' # noqa - }) - - # OAPIF Part 2 - list supported CRSs and StorageCRS - if collection_data_type == 'feature': - collection['crs'] = get_supported_crs_list(collection_data, DEFAULT_CRS_LIST) # noqa - collection['storageCRS'] = collection_data.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - if 'storage_crs_coordinate_epoch' in collection_data: - collection['storageCrsCoordinateEpoch'] = collection_data.get('storage_crs_coordinate_epoch') # noqa - - elif collection_data_type == 'coverage': - # TODO: translate - LOGGER.debug('Adding coverage based links') - collection['links'].append({ - 'type': 'application/prs.coverage+json', - 'rel': f'{OGC_RELTYPES_BASE}/coverage', - 'title': 'Coverage data', - 'href': f'{self.get_collections_url()}/{k}/coverage?f={F_JSON}' # noqa - }) - if collection_data_format is not None: - collection['links'].append({ - 'type': collection_data_format['mimetype'], - 'rel': f'{OGC_RELTYPES_BASE}/coverage', - 'title': f"Coverage data as {collection_data_format['name']}", # noqa - 'href': f"{self.get_collections_url()}/{k}/coverage?f={collection_data_format['name']}" # noqa - }) - if dataset is not None: - LOGGER.debug('Creating extended coverage metadata') - try: - provider_def = get_provider_by_type( - self.config['resources'][k]['providers'], - 'coverage') - p = load_plugin('provider', provider_def) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, request.format, - 'NoApplicableCode', msg) - except ProviderTypeError: - pass - else: - collection['extent']['spatial']['grid'] = [{ - 'cellsCount': p._coverage_properties['width'], - 'resolution': p._coverage_properties['resx'] - }, { - 'cellsCount': p._coverage_properties['height'], - 'resolution': p._coverage_properties['resy'] - }] - - try: - tile = get_provider_by_type(v['providers'], 'tile') - p = load_plugin('provider', tile) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, request.format, - 'NoApplicableCode', msg) - except ProviderTypeError: - tile = None - - if tile: - # TODO: translate - - LOGGER.debug('Adding tile links') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa - 'title': 'Tiles as JSON', - 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa - 'title': 'Tiles as HTML', - 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_HTML}' # noqa - }) - - try: - map_ = get_provider_by_type(v['providers'], 'map') - except ProviderTypeError: - map_ = None - - if map_: - LOGGER.debug('Adding map links') - - map_mimetype = map_['format']['mimetype'] - map_format = map_['format']['name'] - - collection['links'].append({ - 'type': map_mimetype, - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', - 'title': f'Map as {map_format}', - 'href': f"{self.get_collections_url()}/{k}/map?f={map_format}" # noqa - }) - - try: - edr = get_provider_by_type(v['providers'], 'edr') - p = load_plugin('provider', edr) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - except ProviderTypeError: - edr = None - - if edr: - # TODO: translate - LOGGER.debug('Adding EDR links') - parameters = p.get_fields() - if parameters: - collection['parameter_names'] = {} - for key, value in parameters.items(): - collection['parameter_names'][key] = { - 'id': key, - 'type': 'Parameter', - 'name': value['title'], - 'unit': { - 'label': { - 'en': value['title'] - }, - 'symbol': { - 'value': value['x-ogc-unit'], - 'type': 'http://www.opengis.net/def/uom/UCUM/' # noqa - } - } - } - - for qt in p.get_query_types(): - collection['links'].append({ - 'type': 'application/json', - 'rel': 'data', - 'title': f'{qt} query for this collection as JSON', - 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'data', - 'title': f'{qt} query for this collection as HTML', - 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_HTML}' # noqa - }) - - if dataset is not None and k == dataset: - fcm = collection - break - - fcm['collections'].append(collection) - - if dataset is None: - # TODO: translate - fcm['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}?f={F_JSON}' - }) - fcm['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}?f={F_JSONLD}' - }) - fcm['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}?f={F_HTML}' - }) - - if request.format == F_HTML: # render - fcm['collections_path'] = self.get_collections_url() - if dataset is not None: - content = render_j2_template(self.tpl_config, - 'collections/collection.html', - fcm, request.locale) - else: - content = render_j2_template(self.tpl_config, - 'collections/index.html', fcm, - request.locale) - - return headers, HTTPStatus.OK, content - - if request.format == F_JSONLD: - jsonld = self.fcmld.copy() - if dataset is not None: - jsonld['dataset'] = jsonldify_collection(self, fcm, - request.locale) - else: - jsonld['dataset'] = [ - jsonldify_collection(self, c, request.locale) - for c in fcm.get('collections', []) - ] - return headers, HTTPStatus.OK, to_json(jsonld, self.pretty_print) - - return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_collection_schema( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a collection schema - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection schema') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - schema = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/schema' - } - - if p.type != 'coverage': - schema['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - schema['properties'][k] = v - - if k == p.id_field: - schema['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - schema['properties'][k]['x-ogc-role'] = 'primary-instant' - - if request.format == F_HTML: # render - schema['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - schema['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/schema.html', - schema, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(schema, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_collection_queryables(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection queryables - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection queryables') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - queryables = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/queryables' - } - - if p.fields: - queryables['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - show_field = False - if p.properties: - if k in p.properties: - show_field = True - else: - show_field = True - - if show_field: - queryables['properties'][k] = { - 'title': k, - 'type': v['type'] - } - if 'values' in v: - queryables['properties'][k]['enum'] = v['values'] - - if k == p.id_field: - queryables['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa - - if request.format == F_HTML: # render - queryables['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - queryables['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/queryables.html', - queryables, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(queryables, self.pretty_print) - - @gzip - @pre_process - def get_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', - 'offset', 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter', 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - q = request.params.get('q') or None - - LOGGER.debug('Loading provider') - - provider_def = None - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - LOGGER.debug('Processing bbox-crs parameter') - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - # Validate bbox-crs parameter - if len(bbox) == 0: - msg = 'bbox-crs specified without bbox parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - if len(bbox_crs) == 0: - msg = 'bbox-crs specified but is empty' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa - if bbox_crs not in supported_crs_list: - msg = f'bbox-crs {bbox_crs} not supported for this collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - elif len(bbox) > 0: - # bbox but no bbox-crs parm: assume bbox is in default CRS - bbox_crs = DEFAULT_CRS - - # Transform bbox to storageCRS - # when bbox-crs different from storageCRS. - if len(bbox) > 0: - try: - # Get a pyproj CRS instance for the Collection's Storage CRS - storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - - # Do the (optional) Transform to the Storage CRS - bbox = transform_bbox(bbox, bbox_crs, storage_crs) - except CRSError as e: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', str(e)) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k in list(p.fields.keys()): - LOGGER.debug(f'Adding property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('processing filter parameter') - cql_text = request.params.get('filter') - if cql_text is not None: - try: - filter_ = parse_ecql_text(cql_text) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs_uri, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field'), - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {cql_text}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - filter_ = None - - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - # Currently only cql-text is handled, but it is optional - if filter_lang not in [None, 'cql-text']: - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - # Get provider locale (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - if provider_type == 'feature': - LOGGER.debug(f'crs: {query_crs_uri}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {properties}') - LOGGER.debug(f'select properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'language: {prv_locale}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'cql_text: {cql_text}') - LOGGER.debug(f'filter_: {filter_}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs_uri}') - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, skip_geometry=skip_geometry, - select_properties=select_properties, - crs_transform_spec=crs_transform_spec, - q=q, language=prv_locale, filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - serialized_query_params = '' - for k, v in request.params.items(): - if k not in ('f', 'offset'): - serialized_query_params += '&' - serialized_query_params += urllib.parse.quote(k, safe='') - serialized_query_params += '=' - serialized_query_params += urllib.parse.quote(str(v), safe=',') - - # TODO: translate titles - uri = f'{self.get_collections_url()}/{dataset}/items' - content['links'] = [{ - 'type': 'application/geo+json', - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}{serialized_query_params}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}{serialized_query_params}' - }] - - if offset > 0: - prev = max(0, offset - limit) - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'prev', - 'title': 'items (prev)', - 'href': f'{uri}?offset={prev}{serialized_query_params}' - }) - - if 'numberMatched' in content: - if content['numberMatched'] > (limit + offset): - next_ = offset + limit - next_href = f'{uri}?offset={next_}{serialized_query_params}' - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'next', - 'title': 'items (next)', - 'href': next_href - }) - - content['links'].append( - { - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate( - collections[dataset]['title'], request.locale), - 'rel': 'collection', - 'href': uri - }) - - content['timeStamp'] = datetime.utcnow().strftime( - '%Y-%m-%dT%H:%M:%S.%fZ') - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - # For constructing proper URIs to items - - content['items_path'] = uri - content['dataset_path'] = '/'.join(uri.split('/')[:-1]) - content['collections_path'] = self.get_collections_url() - - content['offset'] = offset - - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - # If title exists, use it as id in html templates - content['id_field'] = content['title_field'] - content = render_j2_template(self.tpl_config, - 'collections/items/index.html', - content, request.locale) - return headers, HTTPStatus.OK, content - elif request.format == 'csv': # render - formatter = load_plugin('formatter', - {'name': 'CSV', 'geom': True}) - - try: - content = formatter.write( - data=content, - options={ - 'provider_def': get_provider_by_type( - collections[dataset]['providers'], - 'feature') - } - ) - except FormatterSerializationError as err: - LOGGER.error(err) - msg = 'Error serializing output' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) - - headers['Content-Type'] = formatter.mimetype - - if p.filename is None: - filename = f'{dataset}.csv' - else: - filename = f'{p.filename}' - - cd = f'attachment; filename="{filename}"' - headers['Content-Disposition'] = cd - - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, id_field=(p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def post_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection or filter an item - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - request_headers = request.headers - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', - 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Invalid collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - val = request.params.get('q') - - q = None - if val is not None: - q = val - - LOGGER.debug('Loading provider') - - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - try: - p = load_plugin('provider', provider_def) - except ProviderGenericError as err: - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k not in p.fields.keys(): - msg = f'unknown query parameter: {k}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif k not in reserved_fieldnames and k in p.fields.keys(): - LOGGER.debug(f'Add property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - if filter_lang != 'cql-json': # @TODO add check from the configuration - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs}') - - LOGGER.debug('Processing headers') - - LOGGER.debug('Processing request content-type header') - if (request_headers.get( - 'Content-Type') or request_headers.get( - 'content-type')) != 'application/query-cql-json': - msg = ('Invalid body content-type') - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidHeaderValue', msg) - - LOGGER.debug('Processing body') - - if not request.data: - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - filter_ = None - try: - # Parse bytes data, if applicable - data = request.data.decode() - LOGGER.debug(data) - except UnicodeDecodeError as err: - LOGGER.error(err) - msg = 'Unicode error in data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - # FIXME: remove testing backend in use once CQL support is normalized - if p.name == 'PostgreSQL': - LOGGER.debug('processing PostgreSQL CQL_JSON data') - try: - filter_ = parse_cql_json(data) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field') - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - LOGGER.debug('processing Elasticsearch CQL_JSON data') - try: - filter_ = CQLModel.parse_raw(data) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, - select_properties=select_properties, - skip_geometry=skip_geometry, - q=q, - filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def manage_collection_item( - self, request: Union[APIRequest, Any], - action, dataset, identifier=None) -> Tuple[dict, int, str]: - """ - Adds an item to a collection - - :param request: A request object - :param action: an action among 'create', 'update', 'delete', 'options' - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'options': - headers['Allow'] = 'HEAD, GET' - if p.editable: - if identifier is None: - headers['Allow'] += ', POST' - else: - headers['Allow'] += ', PUT, DELETE' - return headers, HTTPStatus.OK, '' - - if not p.editable: - msg = 'Collection is not editable' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action in ['create', 'update'] and not request.data: - msg = 'No data found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'create': - LOGGER.debug('Creating item') - try: - identifier = p.create(request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - headers['Location'] = f'{self.get_collections_url()}/{dataset}/items/{identifier}' # noqa - - return headers, HTTPStatus.CREATED, '' - - if action == 'update': - LOGGER.debug('Updating item') - try: - _ = p.update(identifier, request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.NO_CONTENT, '' - - if action == 'delete': - LOGGER.debug('Deleting item') - try: - _ = p.delete(identifier) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, '' - - @gzip - @pre_process - def get_collection_item(self, request: Union[APIRequest, Any], - dataset, identifier) -> Tuple[dict, int, str]: - """ - Get a single collection item - - :param request: A request object - :param dataset: dataset name - :param identifier: item identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Processing query parameters') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - try: - LOGGER.debug(f'Fetching id {identifier}') - content = p.get( - identifier, - language=prv_locale, - crs_transform_spec=crs_transform_spec, - ) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if content is None: - msg = 'identifier not found' - return self.get_exception(HTTPStatus.BAD_REQUEST, headers, - request.format, 'NotFound', msg) - - uri = content['properties'].get(p.uri_field) if p.uri_field else \ - f'{self.get_collections_url()}/{dataset}/items/{identifier}' - - if 'links' not in content: - content['links'] = [] - - content['links'].extend([{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }, { - 'rel': request.get_linkrel(F_JSON), - 'type': 'application/geo+json', - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}' - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}' - }, { - 'rel': 'collection', - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate(collections[dataset]['title'], - request.locale), - 'href': f'{self.get_collections_url()}/{dataset}' - }]) - - link_request_format = ( - request.format if request.format is not None else F_JSON - ) - if 'prev' in content: - content['links'].append({ - 'rel': 'prev', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa - }) - if 'next' in content: - content['links'].append({ - 'rel': 'next', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa - }) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - content['title'] = l10n.translate(collections[dataset]['title'], - request.locale) - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - content['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/items/item.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, uri, (p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @pre_process - @jsonldify - def get_collection_coverage(self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection coverage - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - query_args = {} - format_ = request.format or F_JSON - - # Force response content type and language (en-US only) headers - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage') - - p = load_plugin('provider', collection_def) - except KeyError: - msg = 'collection does not exist' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, - 'InvalidParameterValue', msg) - - query_args['bbox'] = bbox - - LOGGER.debug('Processing bbox-crs parameter') - - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - query_args['bbox_crs'] = bbox_crs - - LOGGER.debug('Processing datetime parameter') - - datetime_ = request.params.get('datetime') - - try: - datetime_ = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['datetime_'] = datetime_ - query_args['format_'] = format_ - - properties = request.params.get('properties') - if properties: - LOGGER.debug('Processing properties parameter') - query_args['properties'] = [rs for - rs in properties.split(',') if rs] - LOGGER.debug(f"Fields: {query_args['properties']}") - - for a in query_args['properties']: - if a not in p.fields: - msg = 'Invalid field specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if 'subset' in request.params: - LOGGER.debug('Processing subset parameter') - try: - subsets = validate_subset(request.params['subset'] or '') - except (AttributeError, ValueError) as err: - msg = f'Invalid subset: {err}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if not set(subsets.keys()).issubset(p.axes): - msg = 'Invalid axis name' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['subsets'] = subsets - LOGGER.debug(f"Subsets: {query_args['subsets']}") - - LOGGER.debug('Querying coverage') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - if format_ == mt: # native format - if p.filename is not None: - cd = f'attachment; filename="{p.filename}"' - headers['Content-Disposition'] = cd - - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ == F_JSON: - headers['Content-Type'] = 'application/prs.coverage+json' - return headers, HTTPStatus.OK, to_json(data, self.pretty_print) - else: - return self.get_format_exception(request) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection tiles - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except (KeyError, ProviderTypeError): - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - tiles = { - 'links': [], - 'tilesets': [] - } - - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSON}' - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_HTML}' - }) - - tile_services = p.get_tiles_service( - baseurl=self.base_url, - servicepath=f'{self.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa - ) - - for service in tile_services['links']: - tiles['links'].append(service) - - tiling_schemes = p.get_tiling_schemes() - - for matrix in tiling_schemes: - tile_matrix = { - 'title': dataset, - 'tileMatrixSetURI': matrix.tileMatrixSetURI, - 'crs': matrix.crs, - 'dataType': 'vector', - 'links': [] - } - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', - 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa - 'href': f'{self.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa - }) - - tiles['tilesets'].append(tile_matrix) - - if request.format == F_HTML: # render - tiles['id'] = dataset - tiles['title'] = l10n.translate( - self.config['resources'][dataset]['title'], SYSTEM_LOCALE) - tiles['tilesets'] = [ - scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] - tiles['bounds'] = \ - self.config['resources'][dataset]['extents']['spatial']['bbox'] - tiles['minzoom'] = p.options['zoom']['min'] - tiles['maxzoom'] = p.options['zoom']['max'] - tiles['collections_path'] = self.get_collections_url() - tiles['tile_type'] = p.tile_type - - content = render_j2_template(self.tpl_config, - 'collections/tiles/index.html', tiles, - request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tiles, self.pretty_print) - - @pre_process - def get_collection_tiles_data( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None, - z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - :param z_idx: z index - :param y_idx: y index - :param x_idx: x index - - :returns: tuple of headers, status code, content - """ - - format_ = request.format - if not format_: - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - LOGGER.debug('Processing tiles') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading tile provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - - format_ = p.format_type - headers['Content-Type'] = format_ - - LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa - content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, - z=z_idx, y=y_idx, x=x_idx, format_=format_) - if content is None: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) - else: - return headers, HTTPStatus.OK, content - - # @TODO: figure out if the spec requires to return json errors - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles_metadata( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid([TilesMetadataFormat.TILEJSON]): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(t, request.raw_locale) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - tiles_metadata = p.get_metadata( - dataset=dataset, server_url=self.base_url, - layer=p.get_layer(), tileset=matrix_id, - metadata_format=request._format, title=l10n.translate( - self.config['resources'][dataset]['title'], - request.locale), - description=l10n.translate( - self.config['resources'][dataset]['description'], - request.locale), - language=prv_locale) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/tiles/metadata.html', - tiles_metadata, request.locale) - - return headers, HTTPStatus.OK, content - else: - return headers, HTTPStatus.OK, tiles_metadata - - @gzip - @pre_process - def get_collection_map(self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - query_args = { - 'crs': 'CRS84' - } - - format_ = request.format or 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - query_args['format_'] = request.params.get('f', 'png') - query_args['style'] = style - query_args['crs'] = request.params.get('bbox-crs', 4326) - query_args['transparent'] = request.params.get('transparent', True) - - try: - query_args['width'] = int(request.params.get('width', 500)) - query_args['height'] = int(request.params.get('height', 300)) - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid width/height' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing bbox parameter') - try: - bbox = request.params.get('bbox').split(',') - if len(bbox) != 4: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values should be minx,miny,maxx,maxy' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - except AttributeError: - bbox = self.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa - try: - query_args['bbox'] = [float(c) for c in bbox] - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values must be numbers' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - query_args['datetime_'] = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Generating map') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ in [None, 'html']: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - @gzip - def get_collection_map_legend( - self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map legend - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - format_ = 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Generating legend') - try: - data = p.get_legend(style, request.params.get('f', 'png')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def describe_processes(self, request: Union[APIRequest, Any], - process=None) -> Tuple[dict, int, str]: - """ - Provide processes metadata - - :param request: A request object - :param process: process identifier, defaults to None to obtain - information about all processes - - :returns: tuple of headers, status code, content - """ - - processes = [] - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if process is not None: - if process not in self.manager.processes.keys(): - msg = 'Identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - if len(self.manager.processes) > 0: - if process is not None: - relevant_processes = [process] - else: - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - relevant_processes = list(self.manager.processes)[:limit] - except TypeError: - LOGGER.debug('returning all processes') - relevant_processes = self.manager.processes.keys() - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - for key in relevant_processes: - p = self.manager.get_processor(key) - p2 = l10n.translate_struct(deepcopy(p.metadata), - request.locale) - p2['id'] = key - - if process is None: - p2.pop('inputs') - p2.pop('outputs') - p2.pop('example', None) - - p2['jobControlOptions'] = ['sync-execute'] - if self.manager.is_async: - p2['jobControlOptions'].append('async-execute') - - p2['outputTransmission'] = ['value'] - p2['links'] = p2.get('links', []) - - jobs_url = f"{self.base_url}/jobs" - process_url = f"{self.base_url}/processes/{key}" - - # TODO translation support - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'href': f'{process_url}?f={F_JSON}', - 'title': 'Process description as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'href': f'{process_url}?f={F_HTML}', - 'title': 'Process description as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_HTML}', - 'title': 'jobs for this process as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_JSON}', - 'title': 'jobs for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', - 'href': f'{process_url}/execution?f={F_JSON}', - 'title': 'Execution for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - processes.append(p2) - - if process is not None: - response = processes[0] - else: - process_url = f"{self.base_url}/processes" - response = { - 'processes': processes, - 'links': [{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{process_url}?f={F_JSON}' - }, { - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{process_url}?f={F_JSONLD}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{process_url}?f={F_HTML}' - }] - } - - if request.format == F_HTML: # render - if process is not None: - response = render_j2_template(self.tpl_config, - 'processes/process.html', - response, request.locale) - else: - response = render_j2_template(self.tpl_config, - 'processes/index.html', response, - request.locale) - - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(response, self.pretty_print) - - @gzip - @pre_process - def get_jobs(self, request: Union[APIRequest, Any], - job_id=None) -> Tuple[dict, int, str]: - """ - Get process jobs - - :param request: A request object - :param job_id: id of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if job_id is None: - jobs = sorted(self.manager.get_jobs(), - key=lambda k: k['job_start_datetime'], - reverse=True) - else: - try: - jobs = [self.manager.get_job(job_id)] - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, - 'InvalidParameterValue', job_id) - - serialized_jobs = { - 'jobs': [], - 'links': [{ - 'href': f"{self.base_url}/jobs?f={F_HTML}", - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'Jobs list as HTML' - }, { - 'href': f"{self.base_url}/jobs?f={F_JSON}", - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs list as JSON' - }] - } - for job_ in jobs: - job2 = { - 'type': 'process', - 'processID': job_['process_id'], - 'jobID': job_['identifier'], - 'status': job_['status'], - 'message': job_['message'], - 'progress': job_['progress'], - 'parameters': job_.get('parameters'), - 'job_start_datetime': job_['job_start_datetime'], - 'job_end_datetime': job_['job_end_datetime'] - } - - # TODO: translate - if JobStatus[job_['status']] in ( - JobStatus.successful, JobStatus.running, JobStatus.accepted): - - job_result_url = f"{self.base_url}/jobs/{job_['identifier']}/results" # noqa - - job2['links'] = [{ - 'href': f'{job_result_url}?f={F_HTML}', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', - 'type': FORMAT_TYPES[F_HTML], - 'title': f'results of job {job_id} as HTML' - }, { - 'href': f'{job_result_url}?f={F_JSON}', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', - 'type': FORMAT_TYPES[F_JSON], - 'title': f'results of job {job_id} as JSON' - }] - - if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], - FORMAT_TYPES[F_HTML]): - - job2['links'].append({ - 'href': job_result_url, - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa - 'type': job_['mimetype'], - 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa - }) - - serialized_jobs['jobs'].append(job2) - - if job_id is None: - j2_template = 'jobs/index.html' - else: - serialized_jobs = serialized_jobs['jobs'][0] - j2_template = 'jobs/job.html' - - if request.format == F_HTML: - data = { - 'jobs': serialized_jobs, - 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) - } - response = render_j2_template(self.tpl_config, j2_template, data, - request.locale) - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(serialized_jobs, - self.pretty_print) - - @gzip - @pre_process - def execute_process(self, request: Union[APIRequest, Any], - process_id) -> Tuple[dict, int, str]: - """ - Execute process - - :param request: A request object - :param process_id: id of process - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Responses are always in US English only - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if process_id not in self.manager.processes: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - data = request.data - if not data: - # TODO not all processes require input, e.g. time-dependent or - # random value generators - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - try: - # Parse bytes data, if applicable - data = data.decode() - LOGGER.debug(data) - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input does not appear to be valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - data_dict = data.get('inputs', {}) - LOGGER.debug(data_dict) - - subscriber = None - subscriber_dict = data.get('subscriber') - if subscriber_dict: - try: - success_uri = subscriber_dict['successUri'] - except KeyError: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', 'Missing successUri') - else: - subscriber = Subscriber( - # NOTE: successUri is mandatory according to the standard - success_uri=success_uri, - in_progress_uri=subscriber_dict.get('inProgressUri'), - failed_uri=subscriber_dict.get('failedUri'), - ) - - try: - execution_mode = RequestedProcessExecutionMode( - request.headers.get('Prefer', request.headers.get('prefer')) - ) - except ValueError: - execution_mode = None - try: - LOGGER.debug('Executing process') - result = self.manager.execute_process( - process_id, - data_dict, - execution_mode=execution_mode, - subscriber=subscriber, - ) - job_id, mime_type, outputs, status, additional_headers = result - headers.update(additional_headers or {}) - headers['Location'] = f'{self.base_url}/jobs/{job_id}' - except ProcessorExecuteError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, - request.format, err.ogc_exception_code, err.message) - - response = {} - if status == JobStatus.failed: - response = outputs - - if data.get('response', 'raw') == 'raw': - headers['Content-Type'] = mime_type - response = outputs - elif status not in (JobStatus.failed, JobStatus.accepted): - response['outputs'] = [outputs] - - if status == JobStatus.accepted: - http_status = HTTPStatus.CREATED - elif status == JobStatus.failed: - http_status = HTTPStatus.BAD_REQUEST - else: - http_status = HTTPStatus.OK - - if mime_type == 'application/json': - response2 = to_json(response, self.pretty_print) - else: - response2 = response - - return headers, http_status, response2 - - @gzip - @pre_process - def get_job_result(self, request: Union[APIRequest, Any], - job_id) -> Tuple[dict, int, str]: - """ - Get result of job (instance of a process) - - :param request: A request object - :param job_id: ID of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - try: - job = self.manager.get_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchJob', job_id - ) - - status = JobStatus[job['status']] - - if status == JobStatus.running: - msg = 'job still running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.accepted: - # NOTE: this case is not mentioned in the specification - msg = 'job accepted but not yet running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.failed: - msg = 'job failed' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - mimetype, job_output = self.manager.get_job_result(job_id) - except JobResultNotFoundError: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'JobResultNotFound', job_id - ) - - if mimetype not in (None, FORMAT_TYPES[F_JSON]): - headers['Content-Type'] = mimetype - content = job_output - else: - if request.format == F_JSON: - content = json.dumps(job_output, sort_keys=True, indent=4, - default=json_serial) - else: - # HTML - headers['Content-Type'] = "text/html" - data = { - 'job': {'id': job_id}, - 'result': job_output - } - content = render_j2_template( - self.config, 'jobs/results/index.html', - data, request.locale) - - return headers, HTTPStatus.OK, content - - @pre_process - def delete_job( - self, request: Union[APIRequest, Any], job_id - ) -> Tuple[dict, int, str]: - """ - Delete a process job - - :param job_id: job identifier - - :returns: tuple of headers, status code, content - """ - response_headers = request.get_response_headers( - SYSTEM_LOCALE, **self.api_headers) - try: - success = self.manager.delete_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, response_headers, request.format, - 'NoSuchJob', job_id - ) - else: - if success: - http_status = HTTPStatus.OK - jobs_url = f"{self.base_url}/jobs" - - response = { - 'jobID': job_id, - 'status': JobStatus.dismissed.value, - 'message': 'Job dismissed', - 'progress': 100, - 'links': [{ - 'href': jobs_url, - 'rel': 'up', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The job list for the current process' - }] - } - else: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, - request.format, 'InternalError', job_id - ) - LOGGER.info(response) - # TODO: this response does not have any headers - return {}, http_status, response - - @gzip - @pre_process - def get_collection_edr_query( - self, request: Union[APIRequest, Any], - dataset, instance, query_type, - location_id=None) -> Tuple[dict, int, str]: - """ - Queries collection EDR - - :param request: APIRequest instance with query params - :param dataset: dataset name - :param instance: instance name - :param query_type: EDR query type - :param location_id: location id of a /location/ query - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - headers = request.get_response_headers(self.default_locale, - **self.api_headers) - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing parameter_names parameter') - parameternames = request.params.get('parameter_names') or [] - if isinstance(parameternames, str): - parameternames = parameternames.split(',') - - bbox = None - if query_type in ['cube', 'locations']: - LOGGER.debug('Processing cube bbox') - try: - bbox = validate_bbox(request.params.get('bbox')) - if not bbox and query_type == 'cube': - raise ValueError('bbox parameter required by cube queries') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - LOGGER.debug('Processing coords parameter') - wkt = request.params.get('coords') - - if wkt: - try: - wkt = shapely_loads(wkt) - except WKTReadingError: - msg = 'invalid coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif query_type not in ['cube', 'locations']: - msg = 'missing coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - within = within_units = None - if query_type == 'radius': - LOGGER.debug('Processing within / within-units parameters') - within = request.params.get('within') - within_units = request.params.get('within-units') - - LOGGER.debug('Processing z parameter') - z = request.params.get('z') - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - collections[dataset]['providers'], 'edr')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if instance is not None and not p.get_instance(instance): - msg = 'Invalid instance identifier' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - if query_type not in p.get_query_types(): - msg = 'Unsupported query type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if parameternames and not any((fld in parameternames) - for fld in p.get_fields().keys()): - msg = 'Invalid parameter_names' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - query_args = dict( - query_type=query_type, - instance=instance, - format_=request.format, - datetime_=datetime_, - select_properties=parameternames, - wkt=wkt, - z=z, - bbox=bbox, - within=within, - within_units=within_units, - limit=int(self.config['server']['limit']), - location_id=location_id, - ) - - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/edr/query.html', data, - self.default_locale) - else: - content = to_json(data, self.pretty_print) - - return headers, HTTPStatus.OK, content - - @gzip - @pre_process - @jsonldify - def get_stac_root( - self, request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide STAC root page - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - id_ = 'pygeoapi-stac' - stac_version = '1.0.0-rc.2' - stac_url = f'{self.base_url}/stac' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': l10n.translate( - self.config['metadata']['identification']['description'], - request.locale), - 'links': [] - } - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - for key, value in stac_collections.items(): - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}?f={F_JSON}', - 'type': FORMAT_TYPES[F_JSON] - }) - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}', - 'type': FORMAT_TYPES[F_HTML] - }) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'stac/collection.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_stac_path(self, request: Union[APIRequest, Any], - path) -> Tuple[dict, int, str]: - """ - Provide STAC resource path - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - dataset = None - LOGGER.debug(f'Path: {path}') - dir_tokens = path.split('/') - if dir_tokens: - dataset = dir_tokens[0] - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - if dataset not in stac_collections: - msg = 'Collection not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - stac_collections[dataset]['providers'], 'stac')) - except ProviderConnectionError as err: - LOGGER.error(err) - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - id_ = f'{dataset}-stac' - stac_version = '1.0.0-rc.2' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'description': l10n.translate( - stac_collections[dataset]['description'], request.locale), - 'links': [] - } - try: - stac_data = p.get_data_path( - f'{self.base_url}/stac', - path, - path.replace(dataset, '', 1) - ) - except ProviderNotFoundError as err: - LOGGER.error(err) - msg = 'resource not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - except Exception as err: - LOGGER.error(err) - msg = 'data query error' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - if isinstance(stac_data, dict): - content.update(stac_data) - content['links'].extend( - stac_collections[dataset].get('links', [])) - - if request.format == F_HTML: # render - content['path'] = path - if 'assets' in content: # item view - if content['type'] == 'Collection': - content = render_j2_template( - self.tpl_config, - 'stac/collection_base.html', - content, - request.locale - ) - elif content['type'] == 'Feature': - content = render_j2_template( - self.tpl_config, - 'stac/item.html', - content, - request.locale - ) - else: - msg = f'Unknown STAC type {content.type}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, - request.format, - 'NoApplicableCode', - msg) - else: - content = render_j2_template(self.tpl_config, - 'stac/catalog.html', - content, request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - else: # send back file - headers.pop('Content-Type', None) - return headers, HTTPStatus.OK, stac_data - - def get_exception(self, status, headers, format_, code, - description) -> Tuple[dict, int, str]: - """ - Exception handler - - :param status: HTTP status code - :param headers: dict of HTTP response headers - :param format_: format string - :param code: OGC API exception code - :param description: OGC API exception code - - :returns: tuple of headers, status, and message - """ - - LOGGER.error(description) - exception = { - 'code': code, - 'type': code, - 'description': description - } - - if format_ == F_HTML: - headers['Content-Type'] = FORMAT_TYPES[F_HTML] - content = render_j2_template( - self.config, 'exception.html', exception, SYSTEM_LOCALE) - else: - content = to_json(exception, self.pretty_print) - - return headers, status, content - - def get_format_exception(self, request) -> Tuple[dict, int, str]: - """ - Returns a format exception. - - :param request: An APIRequest instance. - - :returns: tuple of (headers, status, message) - """ - - # Content-Language is in the system locale (ignore language settings) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - msg = f'Invalid format: {request.format}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - def get_collections_url(self): - return f"{self.base_url}/collections" - - @staticmethod - def _create_crs_transform_spec( - config: dict, - query_crs_uri: Optional[str] = None, - ) -> Union[None, CrsTransformSpec]: - """Create a `CrsTransformSpec` instance based on provider config and - *crs* query parameter. - - :param config: Provider config dictionary. - :type config: dict - :param query_crs_uri: Uniform resource identifier of the coordinate - reference system (CRS) specified in query parameter (if specified). - :type query_crs_uri: str, optional - - :raises ValueError: Error raised if the CRS specified in the query - parameter is not in the list of supported CRSs of the provider. - :raises `CRSError`: Error raised if no CRS could be identified from the - query *crs* parameter (URI). - - :returns: `CrsTransformSpec` instance if the CRS specified in query - parameter differs from the storage CRS, else `None`. - :rtype: Union[None, CrsTransformSpec] - """ - # Get storage/default CRS for Collection. - storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) - - if not query_crs_uri: - if storage_crs_uri in DEFAULT_CRS_LIST: - # Could be that storageCRS is - # http://www.opengis.net/def/crs/OGC/1.3/CRS84h - query_crs_uri = storage_crs_uri - else: - query_crs_uri = DEFAULT_CRS - LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') - - supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) - # Check that the crs specified by the query parameter is supported. - if query_crs_uri not in supported_crs_list: - raise ValueError( - f'CRS {query_crs_uri!r} not supported for this ' - 'collection. List of supported CRSs: ' - f'{", ".join(supported_crs_list)}.' - ) - crs_out = get_crs_from_uri(query_crs_uri) - - storage_crs = get_crs_from_uri(storage_crs_uri) - # Check if the crs specified in query parameter differs from the - # storage crs. - if str(storage_crs) != str(crs_out): - LOGGER.debug( - f'CRS transformation: {storage_crs} -> {crs_out}' - ) - return CrsTransformSpec( - source_crs_uri=storage_crs_uri, - source_crs_wkt=storage_crs.to_wkt(), - target_crs_uri=query_crs_uri, - target_crs_wkt=crs_out.to_wkt(), - ) - else: - LOGGER.debug('No CRS transformation') - return None - - @staticmethod - def _set_content_crs_header( - headers: dict, - config: dict, - query_crs_uri: Optional[str] = None, - ): - """Set the *Content-Crs* header in responses from providers of Feature - type. - - :param headers: Response headers dictionary. - :type headers: dict - :param config: Provider config dictionary. - :type config: dict - :param query_crs_uri: Uniform resource identifier of the coordinate - reference system specified in query parameter (if specified). - :type query_crs_uri: str, optional - """ - if query_crs_uri: - content_crs_uri = query_crs_uri - else: - # If empty use default CRS - storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) - if storage_crs_uri in DEFAULT_CRS_LIST: - # Could be that storageCRS is one of the defaults like - # http://www.opengis.net/def/crs/OGC/1.3/CRS84h - content_crs_uri = storage_crs_uri - else: - content_crs_uri = DEFAULT_CRS - - headers['Content-Crs'] = f'<{content_crs_uri}>' - - -def validate_bbox(value=None) -> list: - """ - Helper function to validate bbox parameter - - :param value: `list` of minx, miny, maxx, maxy - - :returns: bbox as `list` of `float` values - """ - - if value is None: - LOGGER.debug('bbox is empty') - return [] - - bbox = value.split(',') - - if len(bbox) not in [4, 6]: - msg = 'bbox should be either 4 values (minx,miny,maxx,maxy) ' \ - 'or 6 values (minx,miny,minz,maxx,maxy,maxz)' - LOGGER.debug(msg) - raise ValueError(msg) - - try: - bbox = [float(c) for c in bbox] - except ValueError as err: - msg = 'bbox values must be numbers' - err.args = (msg,) - LOGGER.debug(msg) - raise - - if (len(bbox) == 4 and bbox[1] > bbox[3]) \ - or (len(bbox) == 6 and bbox[1] > bbox[4]): - msg = 'miny should be less than maxy' - LOGGER.debug(msg) - raise ValueError(msg) - - if (len(bbox) == 4 and bbox[0] > bbox[2]) \ - or (len(bbox) == 6 and bbox[0] > bbox[3]): - msg = 'minx is greater than maxx (possibly antimeridian bbox)' - LOGGER.debug(msg) - - if len(bbox) == 6 and bbox[2] > bbox[5]: - msg = 'minz should be less than maxz' - LOGGER.debug(msg) - raise ValueError(msg) - - return bbox - - -def validate_datetime(resource_def, datetime_=None) -> str: - """ - Helper function to validate temporal parameter - - :param resource_def: `dict` of configuration resource definition - :param datetime_: `str` of datetime parameter - - :returns: `str` of datetime input, if valid - """ - - # TODO: pass datetime to query as a `datetime` object - # we would need to ensure partial dates work accordingly - # as well as setting '..' values to `None` so that underlying - # providers can just assume a `datetime.datetime` object - # - # NOTE: needs testing when passing partials from API to backend - - datetime_invalid = False - - if datetime_ is not None and 'temporal' in resource_def: - - dateparse_begin = partial(dateparse, default=datetime.min) - dateparse_end = partial(dateparse, default=datetime.max) - unix_epoch = datetime(1970, 1, 1, 0, 0, 0) - dateparse_ = partial(dateparse, default=unix_epoch) - - te = resource_def['temporal'] - - try: - if te['begin'] is not None and te['begin'].tzinfo is None: - te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) - if te['end'] is not None and te['end'].tzinfo is None: - te['end'] = te['end'].replace(tzinfo=pytz.UTC) - except AttributeError: - msg = 'Configured times should be RFC3339' - LOGGER.error(msg) - raise ValueError(msg) - - if '/' in datetime_: # envelope - LOGGER.debug('detected time range') - LOGGER.debug('Validating time windows') - - # normalize "" to ".." (actually changes datetime_) - datetime_ = re.sub(r'^/', '../', datetime_) - datetime_ = re.sub(r'/$', '/..', datetime_) - - datetime_begin, datetime_end = datetime_.split('/') - if datetime_begin != '..': - datetime_begin = dateparse_begin(datetime_begin) - if datetime_begin.tzinfo is None: - datetime_begin = datetime_begin.replace( - tzinfo=pytz.UTC) - - if datetime_end != '..': - datetime_end = dateparse_end(datetime_end) - if datetime_end.tzinfo is None: - datetime_end = datetime_end.replace(tzinfo=pytz.UTC) - - datetime_invalid = any([ - (te['end'] is not None and datetime_begin != '..' and - datetime_begin > te['end']), - (te['begin'] is not None and datetime_end != '..' and - datetime_end < te['begin']) - ]) - - else: # time instant - LOGGER.debug('detected time instant') - datetime__ = dateparse_(datetime_) - if datetime__ != '..': - if datetime__.tzinfo is None: - datetime__ = datetime__.replace(tzinfo=pytz.UTC) - datetime_invalid = any([ - (te['begin'] is not None and datetime__ != '..' and - datetime__ < te['begin']), - (te['end'] is not None and datetime__ != '..' and - datetime__ > te['end']) - ]) - - if datetime_invalid: - msg = 'datetime parameter out of range' - LOGGER.debug(msg) - raise ValueError(msg) - - return datetime_ - - -def validate_subset(value: str) -> dict: - """ - Helper function to validate subset parameter - - :param value: `subset` parameter - - :returns: dict of axis/values - """ - - subsets = {} - - for s in value.split(','): - LOGGER.debug(f'Processing subset {s}') - m = re.search(r'(.*)\((.*)\)', s) - subset_name, values = m.group(1, 2) - - if '"' in values: - LOGGER.debug('Values are strings') - if values.count('"') % 2 != 0: - msg = 'Invalid format: subset should be like axis("min"[:"max"])' # noqa - LOGGER.error(msg) - raise ValueError(msg) - try: - LOGGER.debug('Value is an interval') - m = re.search(r'"(\S+)":"(\S+)"', values) - values = list(m.group(1, 2)) - except AttributeError: - LOGGER.debug('Value is point') - m = re.search(r'"(.*)"', values) - values = [m.group(1)] - else: - LOGGER.debug('Values are numbers') - try: - LOGGER.debug('Value is an interval') - m = re.search(r'(\S+):(\S+)', values) - values = list(m.group(1, 2)) - except AttributeError: - LOGGER.debug('Value is point') - values = [values] - - subsets[subset_name] = list(map(get_typed_value, values)) - - return subsets diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py new file mode 100644 index 000000000..0d11c3550 --- /dev/null +++ b/pygeoapi/api/__init__.py @@ -0,0 +1,1687 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +""" +Root level code of pygeoapi, parsing content provided by web framework. +Returns content from plugins and sets responses. +""" + +import asyncio +from collections import OrderedDict +from copy import deepcopy +from datetime import datetime +from functools import partial +from gzip import compress +from http import HTTPStatus +import logging +import re +from typing import Any, Tuple, Union, Optional + +from dateutil.parser import parse as dateparse +import pytz + +from pygeoapi import __version__, l10n +from pygeoapi.linked_data import jsonldify, jsonldify_collection +from pygeoapi.log import setup_logger +from pygeoapi.plugin import load_plugin +from pygeoapi.process.manager.base import get_manager +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderGenericError, ProviderTypeError) + +from pygeoapi.util import ( + CrsTransformSpec, TEMPLATES, UrlPrefetcher, dategetter, + filter_dict_by_key_value, get_api_rules, get_base_url, + get_provider_by_type, get_provider_default, get_typed_value, + get_crs_from_uri, get_supported_crs_list, render_j2_template, to_json +) + +LOGGER = logging.getLogger(__name__) + +#: Return headers for requests (e.g:X-Powered-By) +HEADERS = { + 'Content-Type': 'application/json', + 'X-Powered-By': f'pygeoapi {__version__}' +} + +CHARSET = ['utf-8'] +F_JSON = 'json' +F_HTML = 'html' +F_JSONLD = 'jsonld' +F_GZIP = 'gzip' +F_PNG = 'png' +F_JPEG = 'jpeg' +F_MVT = 'mvt' +F_NETCDF = 'NetCDF' + +#: Formats allowed for ?f= requests (order matters for complex MIME types) +FORMAT_TYPES = OrderedDict(( + (F_HTML, 'text/html'), + (F_JSONLD, 'application/ld+json'), + (F_JSON, 'application/json'), + (F_PNG, 'image/png'), + (F_JPEG, 'image/jpeg'), + (F_MVT, 'application/vnd.mapbox-vector-tile'), + (F_NETCDF, 'application/x-netcdf'), +)) + +#: Locale used for system responses (e.g. exceptions) +SYSTEM_LOCALE = l10n.Locale('en', 'US') + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' +] + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + + +def all_apis() -> dict: + """ + Return all supported API modules + + NOTE: this is a function and not a constant to avoid import loops + + :returns: `dict` of API provider type, API module + """ + + from . import (coverages, environmental_data_retrieval, itemtypes, maps, + processes, tiles, stac) + + return { + 'coverage': coverages, + 'edr': environmental_data_retrieval, + 'itemtypes': itemtypes, + 'map': maps, + 'process': processes, + 'tile': tiles, + 'stac': stac + } + + +def pre_process(func): + """ + Decorator that transforms an incoming Request instance specific to the + web framework (i.e. Flask, Starlette or Django) into a generic + :class:`APIRequest` instance. + + :param func: decorated function + + :returns: `func` + """ + + def inner(*args): + cls, req_in = args[:2] + req_out = APIRequest.with_data(req_in, getattr(cls, 'locales', set())) + if len(args) > 2: + return func(cls, req_out, *args[2:]) + else: + return func(cls, req_out) + + return inner + + +# TODO: remove this when all functions have been refactored +def gzip(func): + """ + Decorator that compresses the content of an outgoing API result + instance if the Content-Encoding response header was set to gzip. + + :param func: decorated function + + :returns: `func` + """ + + def inner(*args, **kwargs): + headers, status, content = func(*args, **kwargs) + charset = CHARSET[0] + if F_GZIP in headers.get('Content-Encoding', []): + try: + if isinstance(content, bytes): + # bytes means Content-Type needs to be set upstream + content = compress(content) + else: + headers['Content-Type'] = \ + f"{headers['Content-Type']}; charset={charset}" + content = compress(content.encode(charset)) + except TypeError as err: + headers.pop('Content-Encoding') + LOGGER.error(f'Error in compression: {err}') + + return headers, status, content + + return inner + + +def apply_gzip(headers: dict, content: str | bytes) -> str | bytes: + """ + Compress content if requested in header. + """ + charset = CHARSET[0] + if F_GZIP in headers.get('Content-Encoding', []): + try: + if isinstance(content, bytes): + # bytes means Content-Type needs to be set upstream + content = compress(content) + else: + headers['Content-Type'] = \ + f"{headers['Content-Type']}; charset={charset}" + content = compress(content.encode(charset)) + except TypeError as err: + headers.pop('Content-Encoding') + LOGGER.error(f'Error in compression: {err}') + return content + + +class APIRequest: + """ + Transforms an incoming server-specific Request into an object + with some generic helper methods and properties. + + .. note:: Typically, this instance is created automatically by the + :func:`pre_process` decorator. **Every** API method that has + been routed to a REST endpoint should be decorated by the + :func:`pre_process` function. + Therefore, **all** routed API methods should at least have 1 + argument that holds the (transformed) request. + + The following example API method will: + + - transform the incoming Flask/Starlette/Django `Request` into an + `APIRequest`using the :func:`pre_process` decorator; + - call :meth:`is_valid` to check if the incoming request was valid, i.e. + that the user requested a valid output format or no format at all + (which means the default format); + - call :meth:`API.get_format_exception` if the requested format was + invalid; + - create a `dict` with the appropriate `Content-Type` header for the + requested format and a `Content-Language` header if any specific language + was requested. + + .. code-block:: python + + @pre_process + def example_method(self, request: Union[APIRequest, Any], custom_arg): + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers() + + # generate response_body here + + return headers, HTTPStatus.OK, response_body + + + The following example API method is similar as the one above, but will also + allow the user to request a non-standard format (e.g. ``f=xml``). + If `xml` was requested, we set the `Content-Type` ourselves. For the + standard formats, the `APIRequest` object sets the `Content-Type`. + + .. code-block:: python + + @pre_process + def example_method(self, request: Union[APIRequest, Any], custom_arg): + if not request.is_valid(['xml']): + return self.get_format_exception(request) + + content_type = 'application/xml' if request.format == 'xml' else None + headers = request.get_response_headers(content_type) + + # generate response_body here + + return headers, HTTPStatus.OK, response_body + + Note that you don't *have* to call :meth:`is_valid`, but that you can also + perform a custom check on the requested output format by looking at the + :attr:`format` property. + Other query parameters are available through the :attr:`params` property as + a `dict`. The request body is available through the :attr:`data` property. + + .. note:: If the request data (body) is important, **always** create a + new `APIRequest` instance using the :meth:`with_data` factory + method. + The :func:`pre_process` decorator will use this automatically. + + :param request: The web platform specific Request instance. + :param supported_locales: List or set of supported Locale instances. + """ + def __init__(self, request, supported_locales): + # Set default request data + self._data = b'' + + # Copy request query parameters + self._args = self._get_params(request) + + # Get path info + if hasattr(request, 'scope'): + self._path_info = request.scope['path'].strip('/') + elif hasattr(request.headers, 'environ'): + self._path_info = request.headers.environ['PATH_INFO'].strip('/') + elif hasattr(request, 'path_info'): + self._path_info = request.path_info + + # Extract locale from params or headers + self._raw_locale, self._locale = self._get_locale(request.headers, + supported_locales) + + # Determine format + self._format = self._get_format(request.headers) + + # Get received headers + self._headers = self.get_request_headers(request.headers) + + # TODO: remove this after all views have been refactored (only used + # in pre_process) + @classmethod + def with_data(cls, request, supported_locales) -> 'APIRequest': + """ + Factory class method to create an `APIRequest` instance with data. + + If the request body is required, an `APIRequest` should always be + instantiated using this class method. The reason for this is, that the + Starlette request body needs to be awaited (async), which cannot be + achieved in the :meth:`__init__` method of the `APIRequest`. + However, `APIRequest` can still be initialized using :meth:`__init__`, + but then the :attr:`data` property value will always be empty. + + :param request: The web platform specific Request instance. + :param supported_locales: List or set of supported Locale instances. + :returns: An `APIRequest` instance with data. + """ + + api_req = cls(request, supported_locales) + if hasattr(request, 'data'): + # Set data from Flask request + api_req._data = request.data + elif hasattr(request, 'body'): + if 'django' in str(request.__class__): + # Set data from Django request + api_req._data = request.body + else: + # Set data from Starlette request after async + # coroutine completion + # TODO: + # this now blocks, but once Flask v2 with async support + # has been implemented, with_data() can become async too + loop = asyncio.get_event_loop() + api_req._data = asyncio.run_coroutine_threadsafe( + request.body(), loop).result(1) + return api_req + + @classmethod + def from_flask(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for flask requests""" + api_req = cls(request, supported_locales) + api_req._data = request.data + return api_req + + @classmethod + async def from_starlette(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for starlette requests + """ + api_req = cls(request, supported_locales) + api_req._data = await request.body() + return api_req + + @classmethod + def from_django(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for django requests""" + api_req = cls(request, supported_locales) + api_req._data = request.body + return api_req + + @staticmethod + def _get_params(request): + """ + Extracts the query parameters from the `Request` object. + + :param request: A Flask or Starlette Request instance + :returns: `ImmutableMultiDict` or empty `dict` + """ + + if hasattr(request, 'args'): + # Return ImmutableMultiDict from Flask request + return request.args + elif hasattr(request, 'query_params'): + # Return ImmutableMultiDict from Starlette request + return request.query_params + elif hasattr(request, 'GET'): + # Return QueryDict from Django GET request + return request.GET + elif hasattr(request, 'POST'): + # Return QueryDict from Django GET request + return request.POST + LOGGER.debug('No query parameters found') + return {} + + def _get_locale(self, headers, supported_locales): + """ + Detects locale from "lang=" param or `Accept-Language` + header. Returns a tuple of (raw, locale) if found in params or headers. + Returns a tuple of (raw default, default locale) if not found. + + :param headers: A dict with Request headers + :param supported_locales: List or set of supported Locale instances + :returns: A tuple of (str, Locale) + """ + + raw = None + try: + default_locale = l10n.str2locale(supported_locales[0]) + except (TypeError, IndexError, l10n.LocaleError) as err: + # This should normally not happen, since the API class already + # loads the supported languages from the config, which raises + # a LocaleError if any of these languages are invalid. + LOGGER.error(err) + raise ValueError(f"{self.__class__.__name__} must be initialized" + f"with a list of valid supported locales") + + for func, mapping in ((l10n.locale_from_params, self._args), + (l10n.locale_from_headers, headers)): + loc_str = func(mapping) + if loc_str: + if not raw: + # This is the first-found locale string: set as raw + raw = loc_str + # Check if locale string is a good match for the UI + loc = l10n.best_match(loc_str, supported_locales) + is_override = func is l10n.locale_from_params + if loc != default_locale or is_override: + return raw, loc + + return raw, default_locale + + def _get_format(self, headers) -> Union[str, None]: + """ + Get `Request` format type from query parameters or headers. + + :param headers: Dict of Request headers + :returns: format value or None if not found/specified + """ + + # Optional f=html or f=json query param + # Overrides Accept header and might differ from FORMAT_TYPES + format_ = (self._args.get('f') or '').strip() + if format_: + return format_ + + # Format not specified: get from Accept headers (MIME types) + # e.g. format_ = 'text/html' + h = headers.get('accept', headers.get('Accept', '')).strip() # noqa + (fmts, mimes) = zip(*FORMAT_TYPES.items()) + # basic support for complex types (i.e. with "q=0.x") + for type_ in (t.split(';')[0].strip() for t in h.split(',') if t): + if type_ in mimes: + idx_ = mimes.index(type_) + format_ = fmts[idx_] + break + + return format_ or None + + @property + def data(self) -> bytes: + """Returns the additional data send with the Request (bytes)""" + return self._data + + @property + def params(self) -> dict: + """Returns the Request query parameters dict""" + return self._args + + @property + def path_info(self) -> str: + """Returns the web server request path info part""" + return self._path_info + + @property + def locale(self) -> l10n.Locale: + """ + Returns the user-defined locale from the request object. + If no locale has been defined or if it is invalid, + the default server locale is returned. + + .. note:: The locale here determines the language in which pygeoapi + should return its responses. This may not be the language + that the user requested. It may also not be the language + that is supported by a collection provider, for example. + For this reason, you should pass the `raw_locale` property + to the :func:`l10n.get_plugin_locale` function, so that + the best match for the provider can be determined. + + :returns: babel.core.Locale + """ + + return self._locale + + @property + def raw_locale(self) -> Union[str, None]: + """ + Returns the raw locale string from the `Request` object. + If no "lang" query parameter or `Accept-Language` header was found, + `None` is returned. + Pass this value to the :func:`l10n.get_plugin_locale` function to let + the provider determine a best match for the locale, which may be + different from the locale used by pygeoapi's UI. + + :returns: a locale string or None + """ + + return self._raw_locale + + @property + def format(self) -> Union[str, None]: + """ + Returns the content type format from the + request query parameters or headers. + + :returns: Format name or None + """ + + return self._format + + @property + def headers(self) -> dict: + """ + Returns the dictionary of the headers from + the request. + + :returns: Request headers dictionary + """ + + return self._headers + + def get_linkrel(self, format_: str) -> str: + """ + Returns the hyperlink relationship (rel) attribute value for + the given API format string. + + The string is compared against the request format and if it matches, + the value 'self' is returned. Otherwise, 'alternate' is returned. + However, if `format_` is 'json' and *no* request format was found, + the relationship 'self' is returned as well (JSON is the default). + + :param format_: The format to compare the request format against. + :returns: A string 'self' or 'alternate'. + """ + + fmt = format_.lower() + if fmt == self._format or (fmt == F_JSON and not self._format): + return 'self' + return 'alternate' + + def is_valid(self, additional_formats=None) -> bool: + """ + Returns True if: + - the format is not set (None) + - the requested format is supported + - the requested format exists in a list if additional formats + + .. note:: Format names are matched in a case-insensitive manner. + + :param additional_formats: Optional additional supported formats list + + :returns: bool + """ + + if not self._format: + return True + if self._format in FORMAT_TYPES.keys(): + return True + if self._format in (f.lower() for f in (additional_formats or ())): + return True + return False + + def get_response_headers(self, force_lang: l10n.Locale = None, + force_type: str = None, + force_encoding: str = None, + **custom_headers) -> dict: + """ + Prepares and returns a dictionary with Response object headers. + + This method always adds a 'Content-Language' header, where the value + is determined by the 'lang' query parameter or 'Accept-Language' + header from the request. + If no language was requested, the default pygeoapi language is used, + unless a `force_lang` override was specified (see notes below). + + A 'Content-Type' header is also always added to the response. + If the user does not specify `force_type`, the header is based on + the `format` APIRequest property. If that is invalid, the default MIME + type `application/json` is used. + + ..note:: If a `force_lang` override is applied, that language + is always set as the 'Content-Language', regardless of + a 'lang' query parameter or 'Accept-Language' header. + If an API response always needs to be in the same + language, 'force_lang' should be set to that language. + + :param force_lang: An optional Content-Language header override. + :param force_type: An optional Content-Type header override. + :param force_encoding: An optional Content-Encoding header override. + :returns: A header dict + """ + + headers = HEADERS.copy() + headers.update(**custom_headers) + l10n.set_response_language(headers, force_lang or self._locale) + if force_type: + # Set custom MIME type if specified + headers['Content-Type'] = force_type + elif self.is_valid() and self._format: + # Set MIME type for valid formats + headers['Content-Type'] = FORMAT_TYPES[self._format] + + if F_GZIP in FORMAT_TYPES: + if force_encoding: + headers['Content-Encoding'] = force_encoding + elif F_GZIP in self._headers.get('Accept-Encoding', ''): + headers['Content-Encoding'] = F_GZIP + + return headers + + def get_request_headers(self, headers) -> dict: + """ + Obtains and returns a dictionary with Request object headers. + + This method adds the headers of the original request and + makes them available to the API object. + + :returns: A header dict + """ + + headers_ = {item[0]: item[1] for item in headers.items()} + return headers_ + + +class API: + """API object""" + + def __init__(self, config, openapi): + """ + constructor + + :param config: configuration dict + :param openapi: openapi dict + + :returns: `pygeoapi.API` instance + """ + + self.config = config + self.openapi = openapi + self.api_headers = get_api_rules(self.config).response_headers + self.base_url = get_base_url(self.config) + self.prefetcher = UrlPrefetcher() + + CHARSET[0] = config['server'].get('encoding', 'utf-8') + if config['server'].get('gzip'): + FORMAT_TYPES[F_GZIP] = 'application/gzip' + FORMAT_TYPES.move_to_end(F_JSON) + + # Process language settings (first locale is default!) + self.locales = l10n.get_locales(config) + self.default_locale = self.locales[0] + + if 'templates' not in self.config['server']: + self.config['server']['templates'] = {'path': TEMPLATES} + + if 'pretty_print' not in self.config['server']: + self.config['server']['pretty_print'] = False + + self.pretty_print = self.config['server']['pretty_print'] + + setup_logger(self.config['logging']) + + # Create config clone for HTML templating with modified base URL + self.tpl_config = deepcopy(self.config) + self.tpl_config['server']['url'] = self.base_url + + self.manager = get_manager(self.config) + LOGGER.info('Process manager plugin loaded') + + @gzip + @pre_process + @jsonldify + def landing_page(self, + request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: + """ + Provide API landing page + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + fcm = { + 'links': [], + 'title': l10n.translate( + self.config['metadata']['identification']['title'], + request.locale), + 'description': + l10n.translate( + self.config['metadata']['identification']['description'], + request.locale) + } + + LOGGER.debug('Creating links') + # TODO: put title text in config or translatable files? + fcm['links'] = [{ + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': 'This document as JSON', + 'href': f"{self.base_url}?f={F_JSON}" + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f"{self.base_url}?f={F_JSONLD}" + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'This document as HTML', + 'href': f"{self.base_url}?f={F_HTML}", + 'hreflang': self.default_locale + }, { + 'rel': 'service-desc', + 'type': 'application/vnd.oai.openapi+json;version=3.0', + 'title': 'The OpenAPI definition as JSON', + 'href': f"{self.base_url}/openapi" + }, { + 'rel': 'service-doc', + 'type': FORMAT_TYPES[F_HTML], + 'title': 'The OpenAPI definition as HTML', + 'href': f"{self.base_url}/openapi?f={F_HTML}", + 'hreflang': self.default_locale + }, { + 'rel': 'conformance', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Conformance', + 'href': f"{self.base_url}/conformance" + }, { + 'rel': 'data', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Collections', + 'href': self.get_collections_url() + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Processes', + 'href': f"{self.base_url}/processes" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Jobs', + 'href': f"{self.base_url}/jobs" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'The list of supported tiling schemes (as JSON)', + 'href': f"{self.base_url}/TileMatrixSets?f=json" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', + 'type': FORMAT_TYPES[F_HTML], + 'title': 'The list of supported tiling schemes (as HTML)', + 'href': f"{self.base_url}/TileMatrixSets?f=html" + }] + + headers = request.get_response_headers(**self.api_headers) + if request.format == F_HTML: # render + + fcm['processes'] = False + fcm['stac'] = False + fcm['collection'] = False + + if filter_dict_by_key_value(self.config['resources'], + 'type', 'process'): + fcm['processes'] = True + + if filter_dict_by_key_value(self.config['resources'], + 'type', 'stac-collection'): + fcm['stac'] = True + + if filter_dict_by_key_value(self.config['resources'], + 'type', 'collection'): + fcm['collection'] = True + + content = render_j2_template(self.tpl_config, 'landing_page.html', + fcm, request.locale) + return headers, HTTPStatus.OK, content + + if request.format == F_JSONLD: + return headers, HTTPStatus.OK, to_json( + self.fcmld, self.pretty_print) + + return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) + + @gzip + @pre_process + def openapi_(self, request: Union[APIRequest, Any]) -> Tuple[ + dict, int, str]: + """ + Provide OpenAPI document + + :param request: A request object + :param openapi: dict of OpenAPI definition + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers(**self.api_headers) + + if request.format == F_HTML: + template = 'openapi/swagger.html' + if request._args.get('ui') == 'redoc': + template = 'openapi/redoc.html' + + path = f'{self.base_url}/openapi' + data = { + 'openapi-document-path': path + } + content = render_j2_template(self.tpl_config, template, data, + request.locale) + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/vnd.oai.openapi+json;version=3.0' # noqa + + if isinstance(self.openapi, dict): + return headers, HTTPStatus.OK, to_json(self.openapi, + self.pretty_print) + else: + return headers, HTTPStatus.OK, self.openapi + + @gzip + @pre_process + def conformance(self, + request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: + """ + Provide conformance definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + apis_dict = all_apis() + + if not request.is_valid(): + return self.get_format_exception(request) + + conformance_list = CONFORMANCE_CLASSES + + for key, value in self.config['resources'].items(): + if value['type'] == 'process': + conformance_list.extend( + apis_dict['process'].CONFORMANCE_CLASSES) + else: + for provider in value['providers']: + if provider['type'] in apis_dict: + conformance_list.extend( + apis_dict[provider['type']].CONFORMANCE_CLASSES) + if provider['type'] == 'feature': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_FEATURES) # noqa + if provider['type'] == 'record': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_RECORDS) + + conformance = { + 'conformsTo': sorted(list(set(conformance_list))) + } + + headers = request.get_response_headers(**self.api_headers) + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, 'conformance.html', + conformance, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) + + @gzip + @pre_process + @jsonldify + def describe_collections(self, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection metadata + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(**self.api_headers) + + fcm = { + 'collections': [], + 'links': [] + } + + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + if all([dataset is not None, dataset not in collections.keys()]): + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + if dataset is not None: + collections_dict = { + k: v for k, v in collections.items() if k == dataset + } + else: + collections_dict = collections + + LOGGER.debug('Creating collections') + for k, v in collections_dict.items(): + if v.get('visibility', 'default') == 'hidden': + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + collection_data = get_provider_default(v['providers']) + collection_data_type = collection_data['type'] + + collection_data_format = None + + if 'format' in collection_data: + collection_data_format = collection_data['format'] + + is_vector_tile = (collection_data_type == 'tile' and + collection_data_format['name'] not + in [F_PNG, F_JPEG]) + + collection = { + 'id': k, + 'title': l10n.translate(v['title'], request.locale), + 'description': l10n.translate(v['description'], request.locale), # noqa + 'keywords': l10n.translate(v['keywords'], request.locale), + 'links': [] + } + + bbox = v['extents']['spatial']['bbox'] + # The output should be an array of bbox, so if the user only + # provided a single bbox, wrap it in a array. + if not isinstance(bbox[0], list): + bbox = [bbox] + collection['extent'] = { + 'spatial': { + 'bbox': bbox + } + } + if 'crs' in v['extents']['spatial']: + collection['extent']['spatial']['crs'] = \ + v['extents']['spatial']['crs'] + + t_ext = v.get('extents', {}).get('temporal', {}) + if t_ext: + begins = dategetter('begin', t_ext) + ends = dategetter('end', t_ext) + collection['extent']['temporal'] = { + 'interval': [[begins, ends]] + } + if 'trs' in t_ext: + collection['extent']['temporal']['trs'] = t_ext['trs'] + + LOGGER.debug('Processing configured collection links') + for link in l10n.translate(v.get('links', []), request.locale): + lnk = { + 'type': link['type'], + 'rel': link['rel'], + 'title': l10n.translate(link['title'], request.locale), + 'href': l10n.translate(link['href'], request.locale), + } + if 'hreflang' in link: + lnk['hreflang'] = l10n.translate( + link['hreflang'], request.locale) + content_length = link.get('length', 0) + + if lnk['rel'] == 'enclosure' and content_length == 0: + # Issue HEAD request for enclosure links without length + lnk_headers = self.prefetcher.get_headers(lnk['href']) + content_length = int(lnk_headers.get('content-length', 0)) + content_type = lnk_headers.get('content-type', lnk['type']) + if content_length == 0: + # Skip this (broken) link + LOGGER.debug(f"Enclosure {lnk['href']} is invalid") + continue + if content_type != lnk['type']: + # Update content type if different from specified + lnk['type'] = content_type + LOGGER.debug( + f"Fixed media type for enclosure {lnk['href']}") + + if content_length > 0: + lnk['length'] = content_length + + collection['links'].append(lnk) + + # TODO: provide translations + LOGGER.debug('Adding JSON and HTML link relations') + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{self.base_url}?f={F_JSON}" + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{self.base_url}?f={F_HTML}" + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{self.get_collections_url()}/{k}?f={F_JSON}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{self.get_collections_url()}/{k}?f={F_JSONLD}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{self.get_collections_url()}/{k}?f={F_HTML}' + }) + + if collection_data_type in ['feature', 'coverage', 'record']: + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': f'{OGC_RELTYPES_BASE}/schema', + 'title': 'Schema of collection in JSON', + 'href': f'{self.get_collections_url()}/{k}/schema?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': f'{OGC_RELTYPES_BASE}/schema', + 'title': 'Schema of collection in HTML', + 'href': f'{self.get_collections_url()}/{k}/schema?f={F_HTML}' # noqa + }) + + if is_vector_tile or collection_data_type in ['feature', 'record']: + # TODO: translate + collection['itemType'] = collection_data_type + LOGGER.debug('Adding feature/record based links') + collection['links'].append({ + 'type': 'application/schema+json', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', + 'title': 'Queryables for this collection as JSON', + 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', + 'title': 'Queryables for this collection as HTML', + 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_HTML}' # noqa + }) + collection['links'].append({ + 'type': 'application/geo+json', + 'rel': 'items', + 'title': 'items as GeoJSON', + 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': 'items', + 'title': 'items as RDF (GeoJSON-LD)', + 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSONLD}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'items', + 'title': 'Items as HTML', + 'href': f'{self.get_collections_url()}/{k}/items?f={F_HTML}' # noqa + }) + + # OAPIF Part 2 - list supported CRSs and StorageCRS + if collection_data_type == 'feature': + collection['crs'] = get_supported_crs_list(collection_data, DEFAULT_CRS_LIST) # noqa + collection['storageCRS'] = collection_data.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + if 'storage_crs_coordinate_epoch' in collection_data: + collection['storageCrsCoordinateEpoch'] = collection_data.get('storage_crs_coordinate_epoch') # noqa + + elif collection_data_type == 'coverage': + # TODO: translate + LOGGER.debug('Adding coverage based links') + collection['links'].append({ + 'type': 'application/prs.coverage+json', + 'rel': f'{OGC_RELTYPES_BASE}/coverage', + 'title': 'Coverage data', + 'href': f'{self.get_collections_url()}/{k}/coverage?f={F_JSON}' # noqa + }) + if collection_data_format is not None: + collection['links'].append({ + 'type': collection_data_format['mimetype'], + 'rel': f'{OGC_RELTYPES_BASE}/coverage', + 'title': f"Coverage data as {collection_data_format['name']}", # noqa + 'href': f"{self.get_collections_url()}/{k}/coverage?f={collection_data_format['name']}" # noqa + }) + if dataset is not None: + LOGGER.debug('Creating extended coverage metadata') + try: + provider_def = get_provider_by_type( + self.config['resources'][k]['providers'], + 'coverage') + p = load_plugin('provider', provider_def) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, request.format, + 'NoApplicableCode', msg) + except ProviderTypeError: + pass + else: + collection['extent']['spatial']['grid'] = [{ + 'cellsCount': p._coverage_properties['width'], + 'resolution': p._coverage_properties['resx'] + }, { + 'cellsCount': p._coverage_properties['height'], + 'resolution': p._coverage_properties['resy'] + }] + + try: + tile = get_provider_by_type(v['providers'], 'tile') + p = load_plugin('provider', tile) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, request.format, + 'NoApplicableCode', msg) + except ProviderTypeError: + tile = None + + if tile: + # TODO: translate + + LOGGER.debug('Adding tile links') + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa + 'title': 'Tiles as JSON', + 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa + 'title': 'Tiles as HTML', + 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_HTML}' # noqa + }) + + try: + map_ = get_provider_by_type(v['providers'], 'map') + except ProviderTypeError: + map_ = None + + if map_: + LOGGER.debug('Adding map links') + + map_mimetype = map_['format']['mimetype'] + map_format = map_['format']['name'] + + collection['links'].append({ + 'type': map_mimetype, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', + 'title': f'Map as {map_format}', + 'href': f"{self.get_collections_url()}/{k}/map?f={map_format}" # noqa + }) + + try: + edr = get_provider_by_type(v['providers'], 'edr') + p = load_plugin('provider', edr) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + except ProviderTypeError: + edr = None + + if edr: + # TODO: translate + LOGGER.debug('Adding EDR links') + parameters = p.get_fields() + if parameters: + collection['parameter_names'] = {} + for key, value in parameters.items(): + collection['parameter_names'][key] = { + 'id': key, + 'type': 'Parameter', + 'name': value['title'], + 'unit': { + 'label': { + 'en': value['title'] + }, + 'symbol': { + 'value': value['x-ogc-unit'], + 'type': 'http://www.opengis.net/def/uom/UCUM/' # noqa + } + } + } + + for qt in p.get_query_types(): + collection['links'].append({ + 'type': 'application/json', + 'rel': 'data', + 'title': f'{qt} query for this collection as JSON', + 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'data', + 'title': f'{qt} query for this collection as HTML', + 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_HTML}' # noqa + }) + + if dataset is not None and k == dataset: + fcm = collection + break + + fcm['collections'].append(collection) + + if dataset is None: + # TODO: translate + fcm['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{self.get_collections_url()}?f={F_JSON}' + }) + fcm['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{self.get_collections_url()}?f={F_JSONLD}' + }) + fcm['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{self.get_collections_url()}?f={F_HTML}' + }) + + if request.format == F_HTML: # render + fcm['collections_path'] = self.get_collections_url() + if dataset is not None: + content = render_j2_template(self.tpl_config, + 'collections/collection.html', + fcm, request.locale) + else: + content = render_j2_template(self.tpl_config, + 'collections/index.html', fcm, + request.locale) + + return headers, HTTPStatus.OK, content + + if request.format == F_JSONLD: + jsonld = self.fcmld.copy() + if dataset is not None: + jsonld['dataset'] = jsonldify_collection(self, fcm, + request.locale) + else: + jsonld['dataset'] = [ + jsonldify_collection(self, c, request.locale) + for c in fcm.get('collections', []) + ] + return headers, HTTPStatus.OK, to_json(jsonld, self.pretty_print) + + return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) + + @gzip + @pre_process + def get_collection_schema(self, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Returns a collection schema + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**self.api_headers) + + if any([dataset is None, + dataset not in self.config['resources'].keys()]): + + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection schema') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + schema = { + 'type': 'object', + 'title': l10n.translate( + self.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{self.get_collections_url()}/{dataset}/schema' + } + + if p.type != 'coverage': + schema['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + schema['properties'][k] = v + + if k == p.id_field: + schema['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + schema['properties'][k]['x-ogc-role'] = 'primary-instant' + + if request.format == F_HTML: # render + schema['title'] = l10n.translate( + self.config['resources'][dataset]['title'], request.locale) + + schema['collections_path'] = self.get_collections_url() + + content = render_j2_template(self.tpl_config, + 'collections/schema.html', + schema, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(schema, self.pretty_print) + + def get_exception(self, status, headers, format_, code, + description) -> Tuple[dict, int, str]: + """ + Exception handler + + :param status: HTTP status code + :param headers: dict of HTTP response headers + :param format_: format string + :param code: OGC API exception code + :param description: OGC API exception code + + :returns: tuple of headers, status, and message + """ + + LOGGER.error(description) + exception = { + 'code': code, + 'type': code, + 'description': description + } + + if format_ == F_HTML: + headers['Content-Type'] = FORMAT_TYPES[F_HTML] + content = render_j2_template( + self.config, 'exception.html', exception, SYSTEM_LOCALE) + else: + content = to_json(exception, self.pretty_print) + + return headers, status, content + + def get_format_exception(self, request) -> Tuple[dict, int, str]: + """ + Returns a format exception. + + :param request: An APIRequest instance. + + :returns: tuple of (headers, status, message) + """ + + # Content-Language is in the system locale (ignore language settings) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + msg = f'Invalid format: {request.format}' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + def get_collections_url(self): + return f"{self.base_url}/collections" + + @staticmethod + def _create_crs_transform_spec( + config: dict, + query_crs_uri: Optional[str] = None, + ) -> Union[None, CrsTransformSpec]: + """Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + @staticmethod + def _set_content_crs_header( + headers: dict, + config: dict, + query_crs_uri: Optional[str] = None, + ): + """Set the *Content-Crs* header in responses from providers of Feature + type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + """ + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +def validate_bbox(value=None) -> list: + """ + Helper function to validate bbox parameter + + :param value: `list` of minx, miny, maxx, maxy + + :returns: bbox as `list` of `float` values + """ + + if value is None: + LOGGER.debug('bbox is empty') + return [] + + bbox = value.split(',') + + if len(bbox) not in [4, 6]: + msg = 'bbox should be either 4 values (minx,miny,maxx,maxy) ' \ + 'or 6 values (minx,miny,minz,maxx,maxy,maxz)' + LOGGER.debug(msg) + raise ValueError(msg) + + try: + bbox = [float(c) for c in bbox] + except ValueError as err: + msg = 'bbox values must be numbers' + err.args = (msg,) + LOGGER.debug(msg) + raise + + if (len(bbox) == 4 and bbox[1] > bbox[3]) \ + or (len(bbox) == 6 and bbox[1] > bbox[4]): + msg = 'miny should be less than maxy' + LOGGER.debug(msg) + raise ValueError(msg) + + if (len(bbox) == 4 and bbox[0] > bbox[2]) \ + or (len(bbox) == 6 and bbox[0] > bbox[3]): + msg = 'minx is greater than maxx (possibly antimeridian bbox)' + LOGGER.debug(msg) + + if len(bbox) == 6 and bbox[2] > bbox[5]: + msg = 'minz should be less than maxz' + LOGGER.debug(msg) + raise ValueError(msg) + + return bbox + + +def validate_datetime(resource_def, datetime_=None) -> str: + """ + Helper function to validate temporal parameter + + :param resource_def: `dict` of configuration resource definition + :param datetime_: `str` of datetime parameter + + :returns: `str` of datetime input, if valid + """ + + # TODO: pass datetime to query as a `datetime` object + # we would need to ensure partial dates work accordingly + # as well as setting '..' values to `None` so that underlying + # providers can just assume a `datetime.datetime` object + # + # NOTE: needs testing when passing partials from API to backend + + datetime_invalid = False + + if datetime_ is not None and 'temporal' in resource_def: + + dateparse_begin = partial(dateparse, default=datetime.min) + dateparse_end = partial(dateparse, default=datetime.max) + unix_epoch = datetime(1970, 1, 1, 0, 0, 0) + dateparse_ = partial(dateparse, default=unix_epoch) + + te = resource_def['temporal'] + + try: + if te['begin'] is not None and te['begin'].tzinfo is None: + te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) + if te['end'] is not None and te['end'].tzinfo is None: + te['end'] = te['end'].replace(tzinfo=pytz.UTC) + except AttributeError: + msg = 'Configured times should be RFC3339' + LOGGER.error(msg) + raise ValueError(msg) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + LOGGER.debug('Validating time windows') + + # normalize "" to ".." (actually changes datetime_) + datetime_ = re.sub(r'^/', '../', datetime_) + datetime_ = re.sub(r'/$', '/..', datetime_) + + datetime_begin, datetime_end = datetime_.split('/') + if datetime_begin != '..': + datetime_begin = dateparse_begin(datetime_begin) + if datetime_begin.tzinfo is None: + datetime_begin = datetime_begin.replace( + tzinfo=pytz.UTC) + + if datetime_end != '..': + datetime_end = dateparse_end(datetime_end) + if datetime_end.tzinfo is None: + datetime_end = datetime_end.replace(tzinfo=pytz.UTC) + + datetime_invalid = any([ + (te['end'] is not None and datetime_begin != '..' and + datetime_begin > te['end']), + (te['begin'] is not None and datetime_end != '..' and + datetime_end < te['begin']) + ]) + + else: # time instant + LOGGER.debug('detected time instant') + datetime__ = dateparse_(datetime_) + if datetime__ != '..': + if datetime__.tzinfo is None: + datetime__ = datetime__.replace(tzinfo=pytz.UTC) + datetime_invalid = any([ + (te['begin'] is not None and datetime__ != '..' and + datetime__ < te['begin']), + (te['end'] is not None and datetime__ != '..' and + datetime__ > te['end']) + ]) + + if datetime_invalid: + msg = 'datetime parameter out of range' + LOGGER.debug(msg) + raise ValueError(msg) + + return datetime_ + + +def validate_subset(value: str) -> dict: + """ + Helper function to validate subset parameter + + :param value: `subset` parameter + + :returns: dict of axis/values + """ + + subsets = {} + + for s in value.split(','): + LOGGER.debug(f'Processing subset {s}') + m = re.search(r'(.*)\((.*)\)', s) + subset_name, values = m.group(1, 2) + + if '"' in values: + LOGGER.debug('Values are strings') + if values.count('"') % 2 != 0: + msg = 'Invalid format: subset should be like axis("min"[:"max"])' # noqa + LOGGER.error(msg) + raise ValueError(msg) + try: + LOGGER.debug('Value is an interval') + m = re.search(r'"(\S+)":"(\S+)"', values) + values = list(m.group(1, 2)) + except AttributeError: + LOGGER.debug('Value is point') + m = re.search(r'"(.*)"', values) + values = [m.group(1)] + else: + LOGGER.debug('Values are numbers') + try: + LOGGER.debug('Value is an interval') + m = re.search(r'(\S+):(\S+)', values) + values = list(m.group(1, 2)) + except AttributeError: + LOGGER.debug('Value is point') + values = [values] + + subsets[subset_name] = list(map(get_typed_value, values)) + + return subsets diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py new file mode 100644 index 000000000..01e5638a6 --- /dev/null +++ b/pygeoapi/api/coverages.py @@ -0,0 +1,253 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError, ProviderTypeError +from pygeoapi.util import ( + filter_dict_by_key_value, get_provider_by_type, to_json +) + +from . import ( + APIRequest, API, F_JSON, SYSTEM_LOCALE, validate_bbox, validate_datetime, + validate_subset +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' +] + + +def get_collection_coverage( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection coverage + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + query_args = {} + format_ = request.format or F_JSON + + # Force response content type and language (en-US only) headers + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage') + + p = load_plugin('provider', collection_def) + except KeyError: + msg = 'collection does not exist' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, + 'InvalidParameterValue', msg) + + query_args['bbox'] = bbox + + LOGGER.debug('Processing bbox-crs parameter') + + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + query_args['bbox_crs'] = bbox_crs + + LOGGER.debug('Processing datetime parameter') + + datetime_ = request.params.get('datetime') + + try: + datetime_ = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['datetime_'] = datetime_ + query_args['format_'] = format_ + + properties = request.params.get('properties') + if properties: + LOGGER.debug('Processing properties parameter') + query_args['properties'] = [rs for + rs in properties.split(',') if rs] + LOGGER.debug(f"Fields: {query_args['properties']}") + + for a in query_args['properties']: + if a not in p.fields: + msg = 'Invalid field specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if 'subset' in request.params: + LOGGER.debug('Processing subset parameter') + try: + subsets = validate_subset(request.params['subset'] or '') + except (AttributeError, ValueError) as err: + msg = f'Invalid subset: {err}' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if not set(subsets.keys()).issubset(p.axes): + msg = 'Invalid axis name' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['subsets'] = subsets + LOGGER.debug(f"Subsets: {query_args['subsets']}") + + LOGGER.debug('Querying coverage') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + if format_ == mt: # native format + if p.filename is not None: + cd = f'attachment; filename="{p.filename}"' + headers['Content-Disposition'] = cd + + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ == F_JSON: + headers['Content-Type'] = 'application/prs.coverage+json' + return headers, HTTPStatus.OK, to_json(data, api.pretty_print) + else: + return api.get_format_exception(request) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + try: + load_plugin('provider', get_provider_by_type( + collections[k]['providers'], 'coverage')) + except ProviderTypeError: + LOGGER.debug('collection is not coverage based') + continue + + coverage_path = f'/collections/{k}/coverage' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[coverage_path] = { + 'get': { + 'summary': f'Get {title} coverage', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Coverage', + 'parameters': [ + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': '#/components/parameters/bbox-crs'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + return [{'name': 'coverages'}], {'paths': paths} diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py new file mode 100644 index 000000000..f659cc835 --- /dev/null +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -0,0 +1,332 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus +import logging +from typing import Tuple + +from shapely.errors import WKTReadingError +from shapely.wkt import loads as shapely_loads + +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + filter_providers_by_type, get_provider_by_type, render_j2_template, + to_json, filter_dict_by_key_value +) + +from . import APIRequest, API, F_HTML, validate_datetime, validate_bbox + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' +] + + +def get_collection_edr_query(api: API, request: APIRequest, + dataset, instance, query_type, + location_id=None) -> Tuple[dict, int, str]: + """ + Queries collection EDR + + :param request: APIRequest instance with query params + :param dataset: dataset name + :param instance: instance name + :param query_type: EDR query type + :param location_id: location id of a /location/ query + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + headers = request.get_response_headers(api.default_locale, + **api.api_headers) + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing parameter_names parameter') + parameternames = request.params.get('parameter_names') or [] + if isinstance(parameternames, str): + parameternames = parameternames.split(',') + + bbox = None + if query_type in ['cube', 'locations']: + LOGGER.debug('Processing cube bbox') + try: + bbox = validate_bbox(request.params.get('bbox')) + if not bbox and query_type == 'cube': + raise ValueError('bbox parameter required by cube queries') + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + LOGGER.debug('Processing coords parameter') + wkt = request.params.get('coords') + + if wkt: + try: + wkt = shapely_loads(wkt) + except WKTReadingError: + msg = 'invalid coords parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif query_type not in ['cube', 'locations']: + msg = 'missing coords parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + within = within_units = None + if query_type == 'radius': + LOGGER.debug('Processing within / within-units parameters') + within = request.params.get('within') + within_units = request.params.get('within-units') + + LOGGER.debug('Processing z parameter') + z = request.params.get('z') + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + collections[dataset]['providers'], 'edr')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if instance is not None and not p.get_instance(instance): + msg = 'Invalid instance identifier' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + if query_type not in p.get_query_types(): + msg = 'Unsupported query type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if parameternames and not any((fld in parameternames) + for fld in p.get_fields().keys()): + msg = 'Invalid parameter_names' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + query_args = dict( + query_type=query_type, + instance=instance, + format_=request.format, + datetime_=datetime_, + select_properties=parameternames, + wkt=wkt, + z=z, + bbox=bbox, + within=within, + within_units=within_units, + limit=int(api.config['server']['limit']), + location_id=location_id, + ) + + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'collections/edr/query.html', data, + api.default_locale) + else: + content = to_json(data, api.pretty_print) + + return headers, HTTPStatus.OK, content + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + LOGGER.debug('setting up edr endpoints') + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + edr_extension = filter_providers_by_type( + collections[k]['providers'], 'edr') + + if edr_extension: + collection_name_path = f'/collections/{k}' + + ep = load_plugin('provider', edr_extension) + + edr_query_endpoints = [] + + for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/{qt}', + 'qt': qt, + 'op_id': f'query{qt.capitalize()}{k.capitalize()}' + }) + if ep.instances: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa + 'qt': qt, + 'op_id': f'query{qt.capitalize()}Instance{k.capitalize()}' # noqa + }) + + for eqe in edr_query_endpoints: + if eqe['qt'] == 'cube': + spatial_parameter = 'bbox' + else: + spatial_parameter = f"{eqe['qt']}Coords" + paths[eqe['path']] = { + 'get': { + 'summary': f"query {v['description']} by {eqe['qt']}", + 'description': v['description'], + 'tags': [k], + 'operationId': eqe['op_id'], + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + if 'locations' in ep.get_query_types(): + paths[f'{collection_name_path}/locations'] = { + 'get': { + 'summary': f"Get pre-defined locations of {v['description']}", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': f'queryLOCATIONS{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + paths[f'{collection_name_path}/locations/{{locId}}'] = { + 'get': { + 'summary': f"query {v['description']} by location", + 'description': v['description'], + 'tags': [k], + 'operationId': f'queryLOCATIONSBYID{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + + return [{'name': 'edr'}], {'paths': paths} diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py new file mode 100644 index 000000000..4a746067b --- /dev/null +++ b/pygeoapi/api/itemtypes.py @@ -0,0 +1,1610 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime +from http import HTTPStatus +import logging +from typing import Any, Tuple, Union, Optional +import urllib.parse + +from pygeofilter.parsers.ecql import parse as parse_ecql_text +from pygeofilter.parsers.cql_json import parse as parse_cql_json +from pyproj.exceptions import CRSError + +from pygeoapi import l10n +from pygeoapi.formatter.base import FormatterSerializationError +from pygeoapi.linked_data import geojson2jsonld +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError, SchemaType) + +from pygeoapi.models.cql import CQLModel +from pygeoapi.util import (CrsTransformSpec, filter_providers_by_type, + filter_dict_by_key_value, get_crs_from_uri, + get_provider_by_type, get_supported_crs_list, + modify_pygeofilter, render_j2_template, str2bool, + to_json, transform_bbox) + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, + validate_bbox, validate_datetime +) + +LOGGER = logging.getLogger(__name__) + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + +CONFORMANCE_CLASSES_FEATURES = [ + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', + 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa + 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/req/core-roles-features' +] + +CONFORMANCE_CLASSES_RECORDS = [ + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' +] + + +def get_collection_queryables(api: API, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection queryables + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection queryables') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + queryables = { + 'type': 'object', + 'title': l10n.translate( + api.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{api.get_collections_url()}/{dataset}/queryables' + } + + if p.fields: + queryables['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + show_field = False + if p.properties: + if k in p.properties: + show_field = True + else: + show_field = True + + if show_field: + queryables['properties'][k] = { + 'title': k, + 'type': v['type'] + } + if 'values' in v: + queryables['properties'][k]['enum'] = v['values'] + + if k == p.id_field: + queryables['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa + + if request.format == F_HTML: # render + queryables['title'] = l10n.translate( + api.config['resources'][dataset]['title'], request.locale) + + queryables['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/queryables.html', + queryables, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(queryables, api.pretty_print) + + +def get_collection_items( + api: API, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Queries collection + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', + 'offset', 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter', 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(api.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + q = request.params.get('q') or None + + LOGGER.debug('Loading provider') + + provider_def = None + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + LOGGER.debug('Processing bbox-crs parameter') + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + # Validate bbox-crs parameter + if len(bbox) == 0: + msg = 'bbox-crs specified without bbox parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + if len(bbox_crs) == 0: + msg = 'bbox-crs specified but is empty' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa + if bbox_crs not in supported_crs_list: + msg = f'bbox-crs {bbox_crs} not supported for this collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + elif len(bbox) > 0: + # bbox but no bbox-crs parm: assume bbox is in default CRS + bbox_crs = DEFAULT_CRS + + # Transform bbox to storageCRS + # when bbox-crs different from storageCRS. + if len(bbox) > 0: + try: + # Get a pyproj CRS instance for the Collection's Storage CRS + storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + + # Do the (optional) Transform to the Storage CRS + bbox = transform_bbox(bbox, bbox_crs, storage_crs) + except CRSError as e: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', str(e)) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k in list(p.fields.keys()): + LOGGER.debug(f'Adding property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) + LOGGER.debug('processing filter parameter') + cql_text = request.params.get('filter') + if cql_text is not None: + try: + filter_ = parse_ecql_text(cql_text) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs_uri, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field'), + ) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {cql_text}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + filter_ = None + + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + # Currently only cql-text is handled, but it is optional + if filter_lang not in [None, 'cql-text']: + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + # Get provider locale (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + if provider_type == 'feature': + LOGGER.debug(f'crs: {query_crs_uri}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {properties}') + LOGGER.debug(f'select properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'language: {prv_locale}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'cql_text: {cql_text}') + LOGGER.debug(f'filter_: {filter_}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs_uri}') + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, skip_geometry=skip_geometry, + select_properties=select_properties, + crs_transform_spec=crs_transform_spec, + q=q, language=prv_locale, filterq=filter_) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + # TODO: translate titles + uri = f'{api.get_collections_url()}/{dataset}/items' + content['links'] = [{ + 'type': 'application/geo+json', + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as GeoJSON', + 'href': f'{uri}?f={F_JSON}{serialized_query_params}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}{serialized_query_params}' + }] + + if offset > 0: + prev = max(0, offset - limit) + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'prev', + 'title': 'items (prev)', + 'href': f'{uri}?offset={prev}{serialized_query_params}' + }) + + if 'numberMatched' in content: + if content['numberMatched'] > (limit + offset): + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'next', + 'title': 'items (next)', + 'href': next_href + }) + + content['links'].append( + { + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate( + collections[dataset]['title'], request.locale), + 'rel': 'collection', + 'href': uri + }) + + content['timeStamp'] = datetime.utcnow().strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + # For constructing proper URIs to items + + content['items_path'] = uri + content['dataset_path'] = '/'.join(uri.split('/')[:-1]) + content['collections_path'] = api.get_collections_url() + + content['offset'] = offset + + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + # If title exists, use it as id in html templates + content['id_field'] = content['title_field'] + content = render_j2_template(api.tpl_config, + 'collections/items/index.html', + content, request.locale) + return headers, HTTPStatus.OK, content + elif request.format == 'csv': # render + formatter = load_plugin('formatter', + {'name': 'CSV', 'geom': True}) + + try: + content = formatter.write( + data=content, + options={ + 'provider_def': get_provider_by_type( + collections[dataset]['providers'], + 'feature') + } + ) + except FormatterSerializationError as err: + LOGGER.error(err) + msg = 'Error serializing output' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + headers['Content-Type'] = formatter.mimetype + + if p.filename is None: + filename = f'{dataset}.csv' + else: + filename = f'{p.filename}' + + cd = f'attachment; filename="{filename}"' + headers['Content-Disposition'] = cd + + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, id_field=(p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def post_collection_items( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Queries collection or filter an item + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + request_headers = request.headers + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', + 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Invalid collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(api.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + val = request.params.get('q') + + q = None + if val is not None: + q = val + + LOGGER.debug('Loading provider') + + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + try: + p = load_plugin('provider', provider_def) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k not in p.fields.keys(): + msg = f'unknown query parameter: {k}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif k not in reserved_fieldnames and k in p.fields.keys(): + LOGGER.debug(f'Add property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs = request.params.get('filter-crs', DEFAULT_CRS) + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + if filter_lang != 'cql-json': # @TODO add check from the configuration + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs}') + + LOGGER.debug('Processing headers') + + LOGGER.debug('Processing request content-type header') + if (request_headers.get( + 'Content-Type') or request_headers.get( + 'content-type')) != 'application/query-cql-json': + msg = ('Invalid body content-type') + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidHeaderValue', msg) + + LOGGER.debug('Processing body') + + if not request.data: + msg = 'missing request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + filter_ = None + try: + # Parse bytes data, if applicable + data = request.data.decode() + LOGGER.debug(data) + except UnicodeDecodeError as err: + LOGGER.error(err) + msg = 'Unicode error in data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + # FIXME: remove testing backend in use once CQL support is normalized + if p.name == 'PostgreSQL': + LOGGER.debug('processing PostgreSQL CQL_JSON data') + try: + filter_ = parse_cql_json(data) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field') + ) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {data}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + LOGGER.debug('processing Elasticsearch CQL_JSON data') + try: + filter_ = CQLModel.parse_raw(data) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {data}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, + select_properties=select_properties, + skip_geometry=skip_geometry, + q=q, + filterq=filter_) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def manage_collection_item( + api: API, request: APIRequest, + action, dataset, identifier=None) -> Tuple[dict, int, str]: + """ + Adds an item to a collection + + :param request: A request object + :param action: an action among 'create', 'update', 'delete', 'options' + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'options': + headers['Allow'] = 'HEAD, GET' + if p.editable: + if identifier is None: + headers['Allow'] += ', POST' + else: + headers['Allow'] += ', PUT, DELETE' + return headers, HTTPStatus.OK, '' + + if not p.editable: + msg = 'Collection is not editable' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action in ['create', 'update'] and not request.data: + msg = 'No data found' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'create': + LOGGER.debug('Creating item') + try: + identifier = p.create(request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + headers['Location'] = f'{api.get_collections_url()}/{dataset}/items/{identifier}' # noqa + + return headers, HTTPStatus.CREATED, '' + + if action == 'update': + LOGGER.debug('Updating item') + try: + _ = p.update(identifier, request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.NO_CONTENT, '' + + if action == 'delete': + LOGGER.debug('Deleting item') + try: + _ = p.delete(identifier) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, '' + + +def get_collection_item(api: API, request: APIRequest, + dataset, identifier) -> Tuple[dict, int, str]: + """ + Get a single collection item + + :param request: A request object + :param dataset: dataset name + :param identifier: item identifier + + :returns: tuple of headers, status code, content + """ + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Processing query parameters') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + try: + LOGGER.debug(f'Fetching id {identifier}') + content = p.get( + identifier, + language=prv_locale, + crs_transform_spec=crs_transform_spec, + ) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if content is None: + msg = 'identifier not found' + return api.get_exception(HTTPStatus.BAD_REQUEST, headers, + request.format, 'NotFound', msg) + + uri = content['properties'].get(p.uri_field) if p.uri_field else \ + f'{api.get_collections_url()}/{dataset}/items/{identifier}' + + if 'links' not in content: + content['links'] = [] + + content['links'].extend([{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{api.base_url}?f={F_JSON}" + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{api.base_url}?f={F_HTML}" + }, { + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/geo+json', + 'title': 'This document as GeoJSON', + 'href': f'{uri}?f={F_JSON}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}' + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}' + }, { + 'rel': 'collection', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate(collections[dataset]['title'], + request.locale), + 'href': f'{api.get_collections_url()}/{dataset}' + }]) + + link_request_format = ( + request.format if request.format is not None else F_JSON + ) + if 'prev' in content: + content['links'].append({ + 'rel': 'prev', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa + }) + if 'next' in content: + content['links'].append({ + 'rel': 'next', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa + }) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + content['title'] = l10n.translate(collections[dataset]['title'], + request.locale) + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + content['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/items/item.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, uri, (p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +@staticmethod +def create_crs_transform_spec( + config: dict, query_crs_uri: Optional[str] = None) -> Union[None, CrsTransformSpec]: # noqa + """ + Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + +@staticmethod +def set_content_crs_header( + headers: dict, config: dict, query_crs_uri: Optional[str] = None): + """ + Set the *Content-Crs* header in responses from providers of Feature type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :returns: None + """ + + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + properties = { + 'name': 'properties', + 'in': 'query', + 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + } + + LOGGER.debug('setting up collection endpoints') + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + try: + ptype = None + + if filter_providers_by_type( + collections[k]['providers'], 'feature'): + ptype = 'feature' + + if filter_providers_by_type( + collections[k]['providers'], 'record'): + ptype = 'record' + + p = load_plugin('provider', get_provider_by_type( + collections[k]['providers'], ptype)) + + collection_name_path = f'/collections/{k}' + items_path = f'/collections/{k}/items' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + coll_properties = deepcopy(properties) + + coll_properties['schema']['items']['enum'] = list(p.fields.keys()) + + paths[items_path] = { + 'get': { + 'summary': f'Get {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Features', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/bbox-crs'}, + coll_properties, + {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa + {'$ref': '#/components/parameters/skipGeometry'}, + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa + {'$ref': '#/components/parameters/offset'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Features', + 'responses': { + '200': {'description': 'options response'} + } + } + } + + if p.editable: + LOGGER.debug('Provider is editable; adding post') + + paths[items_path]['post'] = { + 'summary': f'Add {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'add{k.capitalize()}Features', + 'requestBody': { + 'description': 'Adds item to collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '201': {'description': 'Successful creation'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.create) + paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + if ptype == 'record': + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) + if p.fields: + schema_path = f'{collection_name_path}/schema' + + paths[schema_path] = { + 'get': { + 'summary': f'Get {title} schema', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Schema', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + queryables_path = f'{collection_name_path}/queryables' + + paths[queryables_path] = { + 'get': { + 'summary': f'Get {title} queryables', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Queryables', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + if p.time_field is not None: + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + for field, type_ in p.fields.items(): + + if p.properties and field not in p.properties: + LOGGER.debug('Provider specified not to advertise property') # noqa + continue + + if field == 'q' and ptype == 'record': + LOGGER.debug('q parameter already declared, skipping') + continue + + if type_ == 'date': + schema = { + 'type': 'string', + 'format': 'date' + } + elif type_ == 'float': + schema = { + 'type': 'number', + 'format': 'float' + } + elif type_ == 'long': + schema = { + 'type': 'integer', + 'format': 'int64' + } + else: + schema = type_ + + path_ = f'{collection_name_path}/items' + paths[path_]['get']['parameters'].append({ + 'name': field, + 'in': 'query', + 'required': False, + 'schema': schema, + 'style': 'form', + 'explode': False + }) + + paths[f'{collection_name_path}/items/{{featureId}}'] = { + 'get': { + 'summary': f'Get {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'responses': { + '200': {'description': 'options response'} + } + } + } + + try: + schema_ref = p.get_schema() + paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa + 'content': { + schema_ref[0]: { + 'schema': schema_ref[1] + } + } + } + except Exception as err: + LOGGER.debug(err) + + if p.editable: + LOGGER.debug('Provider is editable; adding put/delete') + put_path = f'{collection_name_path}/items/{{featureId}}' # noqa + paths[put_path]['put'] = { # noqa + 'summary': f'Update {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'update{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'requestBody': { + 'description': 'Updates item in collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.replace) + paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { # noqa + 'summary': f'Delete {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'delete{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + ], + 'responses': { + '200': {'description': 'Successful delete'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + except ProviderTypeError: + LOGGER.debug('collection is not feature/item based') + + return [{'name': 'records'}, {'name': 'features'}], {'paths': paths} diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py new file mode 100644 index 000000000..03866fb25 --- /dev/null +++ b/pygeoapi/api/maps.py @@ -0,0 +1,340 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from http import HTTPStatus +import logging +from typing import Tuple + +from pygeoapi.openapi import get_oas_30_parameters +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_providers_by_type, + filter_dict_by_key_value +) + +from . import APIRequest, API, validate_datetime + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' +] + + +def get_collection_map(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + query_args = { + 'crs': 'CRS84' + } + + format_ = request.format or 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + query_args['format_'] = request.params.get('f', 'png') + query_args['style'] = style + query_args['crs'] = request.params.get('bbox-crs', 4326) + query_args['transparent'] = request.params.get('transparent', True) + + try: + query_args['width'] = int(request.params.get('width', 500)) + query_args['height'] = int(request.params.get('height', 300)) + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid width/height' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + + LOGGER.debug('Processing bbox parameter') + try: + bbox = request.params.get('bbox').split(',') + if len(bbox) != 4: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values should be minx,miny,maxx,maxy' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + except AttributeError: + bbox = api.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa + try: + query_args['bbox'] = [float(c) for c in bbox] + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values must be numbers' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + query_args['datetime_'] = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Generating map') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ in [None, 'html']: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + +def get_collection_map_legend(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map legend + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + format_ = 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Generating legend') + try: + data = p.get_legend(style, request.params.get('f', 'png')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + LOGGER.debug('setting up maps endpoints') + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + parameters = get_oas_30_parameters(cfg, locale) + for k, v in get_visible_collections(cfg).items(): + map_extension = filter_providers_by_type( + collections[k]['providers'], 'map') + + if map_extension: + mp = load_plugin('provider', map_extension) + + map_f = deepcopy(parameters['f']) + map_f['schema']['enum'] = [map_extension['format']['name']] + map_f['schema']['default'] = map_extension['format']['name'] + + pth = f'/collections/{k}/map' + paths[pth] = { + 'get': { + 'summary': 'Get map', + 'description': f"{v['description']} map", + 'tags': [k], + 'operationId': 'getMap', + 'parameters': [ + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + { + 'name': 'width', + 'in': 'query', + 'description': 'Response image width', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'height', + 'in': 'query', + 'description': 'Response image height', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'transparent', + 'in': 'query', + 'description': 'Background transparency of map (default=true).', # noqa + 'required': False, + 'schema': { + 'type': 'boolean', + 'default': True, + }, + 'style': 'form', + 'explode': False + }, + {'$ref': '#/components/parameters/bbox-crs-epsg'}, + map_f + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/json': {} + } + }, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + if mp.time_field is not None: + paths[pth]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + return [{'name': 'maps'}], {'paths': paths} diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py new file mode 100644 index 000000000..6cc7a7180 --- /dev/null +++ b/pygeoapi/api/processes.py @@ -0,0 +1,740 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime, timezone +from http import HTTPStatus +import json +import logging +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.util import ( + json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, + to_json, DATETIME_FORMAT) +from pygeoapi.process.base import ( + JobNotFoundError, JobResultNotFoundError, ProcessorExecuteError +) +from pygeoapi.process.manager.base import get_manager, Subscriber + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback' +] + + +def describe_processes(api: API, request: APIRequest, + process=None) -> Tuple[dict, int, str]: + """ + Provide processes metadata + + :param request: A request object + :param process: process identifier, defaults to None to obtain + information about all processes + + :returns: tuple of headers, status code, content + """ + + processes = [] + + headers = request.get_response_headers(**api.api_headers) + + if process is not None: + if process not in api.manager.processes.keys(): + msg = 'Identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + if len(api.manager.processes) > 0: + if process is not None: + relevant_processes = [process] + else: + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + relevant_processes = list(api.manager.processes)[:limit] + except TypeError: + LOGGER.debug('returning all processes') + relevant_processes = api.manager.processes.keys() + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + for key in relevant_processes: + p = api.manager.get_processor(key) + p2 = l10n.translate_struct(deepcopy(p.metadata), + request.locale) + p2['id'] = key + + if process is None: + p2.pop('inputs') + p2.pop('outputs') + p2.pop('example', None) + + p2['jobControlOptions'] = ['sync-execute'] + if api.manager.is_async: + p2['jobControlOptions'].append('async-execute') + + p2['outputTransmission'] = ['value'] + p2['links'] = p2.get('links', []) + + jobs_url = f"{api.base_url}/jobs" + process_url = f"{api.base_url}/processes/{key}" + + # TODO translation support + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'href': f'{process_url}?f={F_JSON}', + 'title': 'Process description as JSON', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'href': f'{process_url}?f={F_HTML}', + 'title': 'Process description as HTML', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_HTML}', + 'title': 'jobs for this process as HTML', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_JSON}', + 'title': 'jobs for this process as JSON', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', + 'href': f'{process_url}/execution?f={F_JSON}', + 'title': 'Execution for this process as JSON', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + processes.append(p2) + + if process is not None: + response = processes[0] + else: + process_url = f"{api.base_url}/processes" + response = { + 'processes': processes, + 'links': [{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{process_url}?f={F_JSON}' + }, { + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{process_url}?f={F_JSONLD}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{process_url}?f={F_HTML}' + }] + } + + if request.format == F_HTML: # render + if process is not None: + response = render_j2_template(api.tpl_config, + 'processes/process.html', + response, request.locale) + else: + response = render_j2_template(api.tpl_config, + 'processes/index.html', response, + request.locale) + + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(response, api.pretty_print) + + +# TODO: get_jobs doesn't have tests +def get_jobs(api: API, request: APIRequest, + job_id=None) -> Tuple[dict, int, str]: + """ + Get process jobs + + :param request: A request object + :param job_id: id of job + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if job_id is None: + jobs = sorted(api.manager.get_jobs(), + key=lambda k: k['job_start_datetime'], + reverse=True) + else: + try: + jobs = [api.manager.get_job(job_id)] + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, + 'InvalidParameterValue', job_id) + + serialized_jobs = { + 'jobs': [], + 'links': [{ + 'href': f"{api.base_url}/jobs?f={F_HTML}", + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'Jobs list as HTML' + }, { + 'href': f"{api.base_url}/jobs?f={F_JSON}", + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Jobs list as JSON' + }] + } + for job_ in jobs: + job2 = { + 'type': 'process', + 'processID': job_['process_id'], + 'jobID': job_['identifier'], + 'status': job_['status'], + 'message': job_['message'], + 'progress': job_['progress'], + 'parameters': job_.get('parameters'), + 'job_start_datetime': job_['job_start_datetime'], + 'job_end_datetime': job_['job_end_datetime'] + } + + # TODO: translate + if JobStatus[job_['status']] in ( + JobStatus.successful, JobStatus.running, JobStatus.accepted): + + job_result_url = f"{api.base_url}/jobs/{job_['identifier']}/results" # noqa + + job2['links'] = [{ + 'href': f'{job_result_url}?f={F_HTML}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_HTML], + 'title': f'results of job {job_id} as HTML' + }, { + 'href': f'{job_result_url}?f={F_JSON}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_JSON], + 'title': f'results of job {job_id} as JSON' + }] + + if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_HTML]): + + job2['links'].append({ + 'href': job_result_url, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa + 'type': job_['mimetype'], + 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa + }) + + serialized_jobs['jobs'].append(job2) + + if job_id is None: + j2_template = 'jobs/index.html' + else: + serialized_jobs = serialized_jobs['jobs'][0] + j2_template = 'jobs/job.html' + + if request.format == F_HTML: + data = { + 'jobs': serialized_jobs, + 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) + } + response = render_j2_template(api.tpl_config, j2_template, data, + request.locale) + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(serialized_jobs, + api.pretty_print) + + +def execute_process(api: API, request: APIRequest, + process_id) -> Tuple[dict, int, str]: + """ + Execute process + + :param request: A request object + :param process_id: id of process + + :returns: tuple of headers, status code, content + """ + + # Responses are always in US English only + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if process_id not in api.manager.processes: + msg = 'identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + data = request.data + if not data: + # TODO not all processes require input, e.g. time-dependent or + # random value generators + msg = 'missing request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + try: + # Parse bytes data, if applicable + data = data.decode() + LOGGER.debug(data) + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input does not appear to be valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + data_dict = data.get('inputs', {}) + LOGGER.debug(data_dict) + + subscriber = None + subscriber_dict = data.get('subscriber') + if subscriber_dict: + try: + success_uri = subscriber_dict['successUri'] + except KeyError: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', 'Missing successUri') + else: + subscriber = Subscriber( + # NOTE: successUri is mandatory according to the standard + success_uri=success_uri, + in_progress_uri=subscriber_dict.get('inProgressUri'), + failed_uri=subscriber_dict.get('failedUri'), + ) + + try: + execution_mode = RequestedProcessExecutionMode( + request.headers.get('Prefer', request.headers.get('prefer')) + ) + except ValueError: + execution_mode = None + try: + LOGGER.debug('Executing process') + result = api.manager.execute_process( + process_id, data_dict, execution_mode=execution_mode, + subscriber=subscriber) + job_id, mime_type, outputs, status, additional_headers = result + headers.update(additional_headers or {}) + headers['Location'] = f'{api.base_url}/jobs/{job_id}' + except ProcessorExecuteError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, + request.format, err.ogc_exception_code, err.message) + + response = {} + if status == JobStatus.failed: + response = outputs + + if data.get('response', 'raw') == 'raw': + headers['Content-Type'] = mime_type + response = outputs + elif status not in (JobStatus.failed, JobStatus.accepted): + response['outputs'] = [outputs] + + if status == JobStatus.accepted: + http_status = HTTPStatus.CREATED + elif status == JobStatus.failed: + http_status = HTTPStatus.BAD_REQUEST + else: + http_status = HTTPStatus.OK + + if mime_type == 'application/json': + response2 = to_json(response, api.pretty_print) + else: + response2 = response + + return headers, http_status, response2 + + +def get_job_result(api: API, request: APIRequest, + job_id) -> Tuple[dict, int, str]: + """ + Get result of job (instance of a process) + + :param request: A request object + :param job_id: ID of job + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + try: + job = api.manager.get_job(job_id) + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchJob', job_id + ) + + status = JobStatus[job['status']] + + if status == JobStatus.running: + msg = 'job still running' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.accepted: + # NOTE: this case is not mentioned in the specification + msg = 'job accepted but not yet running' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.failed: + msg = 'job failed' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + mimetype, job_output = api.manager.get_job_result(job_id) + except JobResultNotFoundError: + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'JobResultNotFound', job_id + ) + + if mimetype not in (None, FORMAT_TYPES[F_JSON]): + headers['Content-Type'] = mimetype + content = job_output + else: + if request.format == F_JSON: + content = json.dumps(job_output, sort_keys=True, indent=4, + default=json_serial) + else: + # HTML + headers['Content-Type'] = "text/html" + data = { + 'job': {'id': job_id}, + 'result': job_output + } + content = render_j2_template( + api.config, 'jobs/results/index.html', + data, request.locale) + + return headers, HTTPStatus.OK, content + + +def delete_job( + api: API, request: APIRequest, job_id +) -> Tuple[dict, int, str]: + """ + Delete a process job + + :param job_id: job identifier + + :returns: tuple of headers, status code, content + """ + response_headers = request.get_response_headers( + SYSTEM_LOCALE, **api.api_headers) + try: + success = api.manager.delete_job(job_id) + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, response_headers, request.format, + 'NoSuchJob', job_id + ) + else: + if success: + http_status = HTTPStatus.OK + jobs_url = f"{api.base_url}/jobs" + + response = { + 'jobID': job_id, + 'status': JobStatus.dismissed.value, + 'message': 'Job dismissed', + 'progress': 100, + 'links': [{ + 'href': jobs_url, + 'rel': 'up', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'The job list for the current process' + }] + } + else: + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, + request.format, 'InternalError', job_id + ) + LOGGER.info(response) + # TODO: this response does not have any headers + return {}, http_status, response + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML + + LOGGER.debug('setting up processes endpoints') + + oas = {'tags': []} + + paths = {} + + process_manager = get_manager(cfg) + + if len(process_manager.processes) > 0: + paths['/processes'] = { + 'get': { + 'summary': 'Processes', + 'description': 'Processes', + 'tags': ['server'], + 'operationId': 'getProcesses', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + LOGGER.debug('setting up processes') + + for k, v in process_manager.processes.items(): + if k.startswith('_'): + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + name = l10n.translate(k, locale) + p = process_manager.get_processor(k) + md_desc = l10n.translate(p.metadata['description'], locale) + process_name_path = f'/processes/{name}' + tag = { + 'name': name, + 'description': md_desc, + 'externalDocs': {} + } + for link in p.metadata.get('links', []): + if link['type'] == 'information': + translated_link = l10n.translate(link, locale) + tag['externalDocs']['description'] = translated_link[ + 'type'] + tag['externalDocs']['url'] = translated_link['url'] + break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] + + oas['tags'].append(tag) + + paths[process_name_path] = { + 'get': { + 'summary': 'Get process metadata', + 'description': md_desc, + 'tags': [name], + 'operationId': f'describe{name.capitalize()}Process', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths[f'{process_name_path}/execution'] = { + 'post': { + 'summary': f"Process {l10n.translate(p.metadata['title'], locale)} execution", # noqa + 'description': md_desc, + 'tags': [name], + 'operationId': f'execute{name.capitalize()}Job', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + }, + 'requestBody': { + 'description': 'Mandatory execute request JSON', + 'required': True, + 'content': { + 'application/json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa + } + } + } + } + } + } + if 'example' in p.metadata: + paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa + + name_in_path = { + 'name': 'jobId', + 'in': 'path', + 'description': 'job identifier', + 'required': True, + 'schema': { + 'type': 'string' + } + } + + paths['/jobs'] = { + 'get': { + 'summary': 'Retrieve jobs list', + 'description': 'Retrieve a list of jobs', + 'tags': ['jobs'], + 'operationId': 'getJobs', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths['/jobs/{jobId}'] = { + 'get': { + 'summary': 'Retrieve job details', + 'description': 'Retrieve job details', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJob', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + }, + 'delete': { + 'summary': 'Cancel / delete job', + 'description': 'Cancel / delete job', + 'tags': ['jobs'], + 'parameters': [ + name_in_path + ], + 'operationId': 'deleteJob', + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + }, + } + + paths['/jobs/{jobId}/results'] = { + 'get': { + 'summary': 'Retrieve job results', + 'description': 'Retrive job resiults', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJobResults', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + return [{'name': 'proceses'}, {'name': 'jobs'}], {'paths': paths} diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py new file mode 100644 index 000000000..948b68857 --- /dev/null +++ b/pygeoapi/api/stac.py @@ -0,0 +1,256 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus +import logging +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin + +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderNotFoundError +) +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + render_j2_template +) + +from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML + + +LOGGER = logging.getLogger(__name__) + + +CONFORMANCE_CLASSES = [] + + +# TODO: no tests for this? +def get_stac_root(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide STAC root page + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + id_ = 'pygeoapi-stac' + stac_version = '1.0.0-rc.2' + stac_url = f'{api.base_url}/stac' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'title': l10n.translate( + api.config['metadata']['identification']['title'], + request.locale), + 'description': l10n.translate( + api.config['metadata']['identification']['description'], + request.locale), + 'links': [] + } + + stac_collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'stac-collection') + + for key, value in stac_collections.items(): + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}?f={F_JSON}', + 'type': FORMAT_TYPES[F_JSON] + }) + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}', + 'type': FORMAT_TYPES[F_HTML] + }) + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'stac/collection.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +# TODO: no tests for this? +def get_stac_path(api: API, request: APIRequest, + path) -> Tuple[dict, int, str]: + """ + Provide STAC resource path + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + dataset = None + LOGGER.debug(f'Path: {path}') + dir_tokens = path.split('/') + if dir_tokens: + dataset = dir_tokens[0] + + stac_collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'stac-collection') + + if dataset not in stac_collections: + msg = 'Collection not found' + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + stac_collections[dataset]['providers'], 'stac')) + except ProviderConnectionError as err: + LOGGER.error(err) + msg = 'connection error (check logs)' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + id_ = f'{dataset}-stac' + stac_version = '1.0.0-rc.2' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'description': l10n.translate( + stac_collections[dataset]['description'], request.locale), + 'links': [] + } + try: + stac_data = p.get_data_path( + f'{api.base_url}/stac', + path, + path.replace(dataset, '', 1) + ) + except ProviderNotFoundError as err: + LOGGER.error(err) + msg = 'resource not found' + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + except Exception as err: + LOGGER.error(err) + msg = 'data query error' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + if isinstance(stac_data, dict): + content.update(stac_data) + content['links'].extend( + stac_collections[dataset].get('links', [])) + + if request.format == F_HTML: # render + content['path'] = path + if 'assets' in content: # item view + if content['type'] == 'Collection': + content = render_j2_template( + api.tpl_config, + 'stac/collection_base.html', + content, + request.locale + ) + elif content['type'] == 'Feature': + content = render_j2_template( + api.tpl_config, + 'stac/item.html', + content, + request.locale + ) + else: + msg = f'Unknown STAC type {content.type}' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, + request.format, + 'NoApplicableCode', + msg) + else: + content = render_j2_template(api.tpl_config, + 'stac/catalog.html', + content, request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + else: # send back file + headers.pop('Content-Type', None) + return headers, HTTPStatus.OK, stac_data + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + LOGGER.debug('setting up STAC') + stac_collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'stac-collection') + paths = {} + if stac_collections: + paths['/stac'] = { + 'get': { + 'summary': 'SpatioTemporal Asset Catalog', + 'description': 'SpatioTemporal Asset Catalog', + 'tags': ['stac'], + 'operationId': 'getStacCatalog', + 'parameters': [], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + return [{'name': 'stac'}], {'paths': paths} diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py new file mode 100644 index 000000000..5f6dd5fec --- /dev/null +++ b/pygeoapi/api/tiles.py @@ -0,0 +1,534 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.models.provider.base import (TilesMetadataFormat, + TileMatrixSetEnum) +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError +) + +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + filter_providers_by_type, render_j2_template +) + +from . import ( + APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML, SYSTEM_LOCALE, F_JSONLD +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' +] + + +def get_collection_tiles(api: API, request: APIRequest, + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection tiles + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except (KeyError, ProviderTypeError): + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + tiles = { + 'links': [], + 'tilesets': [] + } + + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSON}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_HTML}' + }) + + tile_services = p.get_tiles_service( + baseurl=api.base_url, + servicepath=f'{api.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa + ) + + for service in tile_services['links']: + tiles['links'].append(service) + + tiling_schemes = p.get_tiling_schemes() + + for matrix in tiling_schemes: + tile_matrix = { + 'title': dataset, + 'tileMatrixSetURI': matrix.tileMatrixSetURI, + 'crs': matrix.crs, + 'dataType': 'vector', + 'links': [] + } + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', + 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa + 'href': f'{api.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa + }) + + tiles['tilesets'].append(tile_matrix) + + if request.format == F_HTML: # render + tiles['id'] = dataset + tiles['title'] = l10n.translate( + api.config['resources'][dataset]['title'], SYSTEM_LOCALE) + tiles['tilesets'] = [ + scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] + tiles['bounds'] = \ + api.config['resources'][dataset]['extents']['spatial']['bbox'] + tiles['minzoom'] = p.options['zoom']['min'] + tiles['maxzoom'] = p.options['zoom']['max'] + tiles['collections_path'] = api.get_collections_url() + tiles['tile_type'] = p.tile_type + + content = render_j2_template(api.tpl_config, + 'collections/tiles/index.html', tiles, + request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tiles, api.pretty_print) + + +# TODO: no test for this function? +def get_collection_tiles_data( + api: API, request: APIRequest, + dataset=None, matrix_id=None, + z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + :param z_idx: z index + :param y_idx: y index + :param x_idx: x index + + :returns: tuple of headers, status code, content + """ + + format_ = request.format + if not format_: + return api.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + LOGGER.debug('Processing tiles') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading tile provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + + format_ = p.format_type + headers['Content-Type'] = format_ + + LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa + content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, + z=z_idx, y=y_idx, x=x_idx, format_=format_) + if content is None: + msg = 'identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) + else: + return headers, HTTPStatus.OK, content + + # @TODO: figure out if the spec requires to return json errors + except KeyError: + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + +# TODO: no test for this function? +def get_collection_tiles_metadata( + api: API, request: APIRequest, + dataset=None, matrix_id=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid([TilesMetadataFormat.TILEJSON]): + return api.get_format_exception(request) + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except KeyError: + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(t, request.raw_locale) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + tiles_metadata = p.get_metadata( + dataset=dataset, server_url=api.base_url, + layer=p.get_layer(), tileset=matrix_id, + metadata_format=request._format, title=l10n.translate( + api.config['resources'][dataset]['title'], + request.locale), + description=l10n.translate( + api.config['resources'][dataset]['description'], + request.locale), + language=prv_locale) + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'collections/tiles/metadata.html', + tiles_metadata, request.locale) + + return headers, HTTPStatus.OK, content + else: + return headers, HTTPStatus.OK, tiles_metadata + + +def tilematrixsets(api: API, + request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide tileMatrixSets definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + # Retrieve available TileMatrixSets + enums = [e.value for e in TileMatrixSetEnum] + + tms = {"tileMatrixSets": []} + + for e in enums: + tms['tileMatrixSets'].append({ + "title": e.title, + "id": e.tileMatrixSet, + "uri": e.tileMatrixSetURI, + "links": [ + { + "rel": "self", + "type": "text/html", + "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa + }, + { + "rel": "self", + "type": "application/json", + "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa + } + ] + }) + + tms['links'] = [{ + "rel": "alternate", + "type": "text/html", + "title": "This document as HTML", + "href": f"{api.base_url}/tileMatrixSets?f=html" + }, { + "rel": "self", + "type": "application/json", + "title": "This document", + "href": f"{api.base_url}/tileMatrixSets?f=json" + }] + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'tilematrixsets/index.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + + +def tilematrixset(api: API, + request: APIRequest, + tileMatrixSetId) -> Tuple[dict, + int, str]: + """ + Provide tile matrix definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + # Retrieve relevant TileMatrixSet + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + try: + for e in enums: + if tileMatrixSetId == e.tileMatrixSet: + enum = e + if not enum: + raise ValueError('could not find this tilematrixset') + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + tms = { + "title": enum.tileMatrixSet, + "crs": enum.crs, + "id": enum.tileMatrixSet, + "uri": enum.tileMatrixSetURI, + "orderedAxes": enum.orderedAxes, + "wellKnownScaleSet": enum.wellKnownScaleSet, + "tileMatrices": enum.tileMatrices + } + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'tilematrixsets/tilematrixset.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + paths = {} + + LOGGER.debug('setting up tiles endpoints') + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + tile_extension = filter_providers_by_type( + collections[k]['providers'], 'tile') + + if tile_extension: + tp = load_plugin('provider', tile_extension) + + tiles_path = f'/collections/{k}/tiles' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[tiles_path] = { + 'get': { + 'summary': f'Fetch a {title} tiles description', + 'description': description, + 'tags': [k], + 'operationId': f'describe{k.capitalize()}Tiles', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Tiles'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + tiles_data_path = f'{tiles_path}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa + + paths[tiles_data_path] = { + 'get': { + 'summary': f'Get a {title} tile', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Tiles', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa + { + 'name': 'f', + 'in': 'query', + 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [tp.format_type], + 'default': tp.format_type + }, + 'style': 'form', + 'explode': False + } + ], + 'responses': { + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + mimetype = tile_extension['format']['mimetype'] + paths[tiles_data_path]['get']['responses']['200'] = { + 'description': 'successful operation', + 'content': { + mimetype: { + 'schema': { + 'type': 'string', + 'format': 'binary' + } + } + } + } + + return [{'name': 'tiles'}], {'paths': paths} diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 974828ba9..fe841b2fd 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -8,7 +8,7 @@ # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 Luca Delucchi # Copyright (c) 2022 Krishna Lodha -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -40,7 +40,14 @@ from django.conf import settings from django.http import HttpRequest, HttpResponse -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -157,8 +164,8 @@ def collection_queryables(request: HttpRequest, :returns: Django HTTP Response """ - response_ = _feed_response( - request, 'get_collection_queryables', collection_id + response_ = execute_from_django( + itemtypes_api.get_collection_queryables, request, collection_id ) response = _to_django_response(*response_) @@ -176,22 +183,26 @@ def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: """ if request.method == 'GET': - response_ = _feed_response( + response_ = execute_from_django( + itemtypes_api.get_collection_items, request, - 'get_collection_items', collection_id, + skip_valid_check=True, ) elif request.method == 'POST': if request.content_type is not None: if request.content_type == 'application/geo+json': - response_ = _feed_response(request, 'manage_collection_item', - request, 'create', collection_id) + response_ = execute_from_django( + itemtypes_api.manage_collection_item, request, + 'create', collection_id, skip_valid_check=True) else: - response_ = _feed_response(request, 'post_collection_items', - request, collection_id) + response_ = execute_from_django( + itemtypes_api.post_collection_items, + request, collection_id, skip_valid_check=True,) elif request.method == 'OPTIONS': - response_ = _feed_response(request, 'manage_collection_item', - request, 'options', collection_id) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'options', collection_id, + skip_valid_check=True) response = _to_django_response(*response_) @@ -206,12 +217,9 @@ def collection_map(request: HttpRequest, collection_id: str): :returns: HTTP response """ - - response_ = _feed_response(request, 'get_collection_map', collection_id) - - response = _to_django_response(*response_) - - return response + return execute_from_django( + maps_api.get_collection_map, request, collection_id + ) def collection_style_map(request: HttpRequest, collection_id: str, @@ -252,17 +260,17 @@ def collection_item(request: HttpRequest, elif request.method == 'PUT': response_ = _feed_response( request, 'manage_collection_item', request, 'update', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'DELETE': response_ = _feed_response( request, 'manage_collection_item', request, 'delete', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': response_ = _feed_response( request, 'manage_collection_item', request, 'options', - collection_id, item_id) + collection_id, item_id, skip_valid_check=True) response = _to_django_response(*response_) @@ -280,8 +288,8 @@ def collection_coverage(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response( - request, 'get_collection_coverage', collection_id + response_ = execute_from_django( + coverages_api.get_collection_coverage, request, collection_id ) response = _to_django_response(*response_) @@ -298,10 +306,8 @@ def collection_tiles(request: HttpRequest, collection_id: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_collection_tiles', collection_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(tiles_api.get_collection_tiles, request, + collection_id) def collection_tiles_metadata(request: HttpRequest, collection_id: str, @@ -316,15 +322,11 @@ def collection_tiles_metadata(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( - request, - 'get_collection_tiles_metadata', - collection_id, - tileMatrixSetId, + return execute_from_django( + tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, + skip_valid_check=True, ) - response = _to_django_response(*response_) - - return response def collection_item_tiles(request: HttpRequest, collection_id: str, @@ -343,18 +345,16 @@ def collection_item_tiles(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( + return execute_from_django( + tiles_api.get_collection_tiles_data, request, - 'get_collection_tiles_data', collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True, ) - response = _to_django_response(*response_) - - return response def processes(request: HttpRequest, @@ -368,10 +368,8 @@ def processes(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'describe_processes', process_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.describe_processes, request, + process_id) def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: @@ -384,11 +382,7 @@ def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: :returns: Django HTTP response """ - - response_ = _feed_response(request, 'get_jobs', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_jobs, request, job_id) def job_results(request: HttpRequest, @@ -402,10 +396,7 @@ def job_results(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_job_result', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_job_result, request, job_id) def job_results_resource(request: HttpRequest, process_id: str, job_id: str, @@ -420,6 +411,7 @@ def job_results_resource(request: HttpRequest, process_id: str, job_id: str, :returns: Django HTTP response """ + # TODO: this api method does not exist response_ = _feed_response( request, 'get_job_result_resource', @@ -451,17 +443,15 @@ def get_collection_edr_query( query_type = 'locations' else: query_type = request.path.split('/')[-1] - response_ = _feed_response( + return execute_from_django( + edr_api.get_collection_edr_query, request, - 'get_collection_edr_query', collection_id, instance_id, query_type, - location_id + location_id, + skip_valid_check=True, ) - response = _to_django_response(*response_) - - return response def stac_catalog_root(request: HttpRequest) -> HttpResponse: @@ -473,10 +463,7 @@ def stac_catalog_root(request: HttpRequest) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_root') - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_root, request) def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: @@ -489,10 +476,7 @@ def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_path', path) - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_path, request, path) def admin_config(request: HttpRequest) -> HttpResponse: @@ -551,6 +535,7 @@ def admin_config_resource(request: HttpRequest, resource_id) +# TODO: remove this when all views have been refactored def _feed_response(request: HttpRequest, api_definition: str, *args, **kwargs) -> Tuple[Dict, int, str]: """Use pygeoapi api to process the input request""" @@ -566,8 +551,31 @@ def _feed_response(request: HttpRequest, api_definition: str, return api(request, *args, **kwargs) +def execute_from_django(api_function, request: HttpRequest, *args, + skip_valid_check=False) -> HttpResponse: + + api_: API | "Admin" + if settings.PYGEOAPI_CONFIG['server'].get('admin'): # noqa + from pygeoapi.admin import Admin + api_ = Admin(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) + else: + api_ = API(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) + + api_request = APIRequest.from_django(request, api_.locales) + content: str | bytes + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) + + return _to_django_response(headers, status, content) + + +# TODO: inline this to execute_from_django after refactoring def _to_django_response(headers: Mapping, status_code: int, - content: str) -> HttpResponse: + content: str | bytes) -> HttpResponse: """Convert API payload to a django response""" response = HttpResponse(content, status=status_code) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 5c2247563..e4d8e82d1 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -34,9 +34,17 @@ import click -from flask import Flask, Blueprint, make_response, request, send_from_directory - -from pygeoapi.api import API +from flask import (Flask, Blueprint, make_response, request, + send_from_directory, Response, Request) + +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -110,6 +118,7 @@ def schemas(path): mimetype=get_mimetype(basename_)) +# TODO: inline in execute_from_flask when all views have been refactored def get_response(result: tuple): """ Creates a Flask Response object and updates matching headers. @@ -117,7 +126,7 @@ def get_response(result: tuple): :param result: The result of the API call. This should be a tuple of (headers, status, content). - :returns: A Response instance. + :returns: A Response instance """ headers, status, content = result @@ -128,6 +137,33 @@ def get_response(result: tuple): return response +def execute_from_flask(api_function, request: Request, *args, + skip_valid_check=False) -> Response: + """ + Executes API function from Flask + + :param api_function: API function + :param request: request object + :param *args: variable length additional arguments + :param skip_validity_check: bool + + :returns: A Response instance + """ + + api_request = APIRequest.from_flask(request, api_.locales) + + content: str | bytes + + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) + # handle jsonld too? + + return get_response((headers, status, content)) + + @BLUEPRINT.route('/') def landing_page(): """ @@ -145,6 +181,7 @@ def openapi(): :returns: HTTP response """ + return get_response(api_.openapi_(request)) @@ -155,6 +192,7 @@ def conformance(): :returns: HTTP response """ + return get_response(api_.conformance(request)) @@ -164,9 +202,12 @@ def get_tilematrix_set(tileMatrixSetId=None): OGC API TileMatrixSet endpoint :param tileMatrixSetId: identifier of tile matrix set + :returns: HTTP response """ - return get_response(api_.tilematrixset(request, tileMatrixSetId)) + + return execute_from_flask(tiles_api.tilematrixset, request, + tileMatrixSetId) @BLUEPRINT.route('/TileMatrixSets') @@ -176,7 +217,8 @@ def get_tilematrix_sets(): :returns: HTTP response """ - return get_response(api_.tilematrixsets(request)) + + return execute_from_flask(tiles_api.tilematrixsets, request) @BLUEPRINT.route('/collections') @@ -189,6 +231,7 @@ def collections(collection_id=None): :returns: HTTP response """ + return get_response(api_.describe_collections(request, collection_id)) @@ -201,6 +244,7 @@ def collection_schema(collection_id): :returns: HTTP response """ + return get_response(api_.get_collection_schema(request, collection_id)) @@ -213,7 +257,9 @@ def collection_queryables(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_queryables(request, collection_id)) + + return execute_from_flask(itemtypes_api.get_collection_queryables, request, + collection_id) @BLUEPRINT.route('/collections//items', @@ -234,36 +280,40 @@ def collection_items(collection_id, item_id=None): if item_id is None: if request.method == 'GET': # list items - return get_response( - api_.get_collection_items(request, collection_id)) + return execute_from_flask(itemtypes_api.get_collection_items, + request, collection_id, + skip_valid_check=True) elif request.method == 'POST': # filter or manage items if request.content_type is not None: if request.content_type == 'application/geo+json': - return get_response( - api_.manage_collection_item(request, 'create', - collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, + request, 'create', collection_id, + skip_valid_check=True) else: - return get_response( - api_.post_collection_items(request, collection_id)) + return execute_from_flask( + itemtypes_api.post_collection_items, request, + collection_id, skip_valid_check=True) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, request, 'options', + collection_id, skip_valid_check=True) elif request.method == 'DELETE': - return get_response( - api_.manage_collection_item(request, 'delete', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'delete', collection_id, item_id, + skip_valid_check=True) elif request.method == 'PUT': - return get_response( - api_.manage_collection_item(request, 'update', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'update', collection_id, item_id, + skip_valid_check=True) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'options', collection_id, item_id, + skip_valid_check=True) else: - return get_response( - api_.get_collection_item(request, collection_id, item_id)) + return execute_from_flask(itemtypes_api.get_collection_item, request, + collection_id, item_id) @BLUEPRINT.route('/collections//coverage') @@ -275,7 +325,9 @@ def collection_coverage(collection_id): :returns: HTTP response """ - return get_response(api_.get_collection_coverage(request, collection_id)) + + return execute_from_flask(coverages_api.get_collection_coverage, request, + collection_id) @BLUEPRINT.route('/collections//tiles') @@ -287,8 +339,9 @@ def get_collection_tiles(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles( - request, collection_id)) + + return execute_from_flask(tiles_api.get_collection_tiles, request, + collection_id) @BLUEPRINT.route('/collections//tiles/') @@ -302,8 +355,10 @@ def get_collection_tiles_metadata(collection_id=None, tileMatrixSetId=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles_metadata( - request, collection_id, tileMatrixSetId)) + + return execute_from_flask(tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, + skip_valid_check=True) @BLUEPRINT.route('/collections//tiles/\ @@ -321,8 +376,12 @@ def get_collection_tiles_data(collection_id=None, tileMatrixSetId=None, :returns: HTTP response """ - return get_response(api_.get_collection_tiles_data( - request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol)) + + return execute_from_flask( + tiles_api.get_collection_tiles_data, + request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True, + ) @BLUEPRINT.route('/collections//map') @@ -337,15 +396,9 @@ def collection_map(collection_id, style_id=None): :returns: HTTP response """ - headers, status_code, content = api_.get_collection_map( - request, collection_id, style_id) - - response = make_response(content, status_code) - - if headers: - response.headers = headers - - return response + return execute_from_flask( + maps_api.get_collection_map, request, collection_id, style_id + ) @BLUEPRINT.route('/processes') @@ -358,7 +411,9 @@ def get_processes(process_id=None): :returns: HTTP response """ - return get_response(api_.describe_processes(request, process_id)) + + return execute_from_flask(processes_api.describe_processes, request, + process_id) @BLUEPRINT.route('/jobs') @@ -374,12 +429,12 @@ def get_jobs(job_id=None): """ if job_id is None: - return get_response(api_.get_jobs(request)) + return execute_from_flask(processes_api.get_jobs, request) else: if request.method == 'DELETE': # dismiss job - return get_response(api_.delete_job(request, job_id)) + return execute_from_flask(processes_api.delete_jobs, request) else: # Return status of a specific job - return get_response(api_.get_jobs(request, job_id)) + return execute_from_flask(processes_api.get_jobs, request, job_id) @BLUEPRINT.route('/processes//execution', methods=['POST']) @@ -392,7 +447,8 @@ def execute_process_jobs(process_id): :returns: HTTP response """ - return get_response(api_.execute_process(request, process_id)) + return execute_from_flask(processes_api.execute_process, request, + process_id) @BLUEPRINT.route('/jobs//results', @@ -405,7 +461,8 @@ def get_job_result(job_id=None): :returns: HTTP response """ - return get_response(api_.get_job_result(request, job_id)) + + return execute_from_flask(processes_api.get_job_result, request, job_id) @BLUEPRINT.route('/jobs//results/', @@ -419,6 +476,8 @@ def get_job_result_resource(job_id, resource): :returns: HTTP response """ + + # TODO: this does not seem to exist? return get_response(api_.get_job_result_resource( request, job_id, resource)) @@ -450,14 +509,17 @@ def get_collection_edr_query(collection_id, instance_id=None, :returns: HTTP response """ + if location_id: query_type = 'locations' else: query_type = request.path.split('/')[-1] - return get_response(api_.get_collection_edr_query(request, collection_id, - instance_id, query_type, - location_id)) + return execute_from_flask( + edr_api.get_collection_edr_query, request, collection_id, instance_id, + query_type, location_id, + skip_valid_check=True, + ) @BLUEPRINT.route('/stac') @@ -467,7 +529,8 @@ def stac_catalog_root(): :returns: HTTP response """ - return get_response(api_.get_stac_root(request)) + + return execute_from_flask(stac_api.get_stac_root, request) @BLUEPRINT.route('/stac/') @@ -479,7 +542,8 @@ def stac_catalog_path(path): :returns: HTTP response """ - return get_response(api_.get_stac_path(request, path)) + + return execute_from_flask(stac_api.get_stac_path, request, path) @ADMIN_BLUEPRINT.route('/admin/config', methods=['GET', 'PUT', 'PATCH']) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index a68c37ceb..274e19dad 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -37,19 +37,16 @@ import logging import os from pathlib import Path -from typing import Tuple, Union +from typing import Union import click from jsonschema import validate as jsonschema_validate import yaml from pygeoapi import l10n +from pygeoapi.api import all_apis from pygeoapi.models.openapi import OAPIFormat -from pygeoapi.plugin import load_plugin -from pygeoapi.process.manager.base import get_manager -from pygeoapi.provider.base import ProviderTypeError, SchemaType -from pygeoapi.util import (filter_dict_by_key_value, get_provider_by_type, - filter_providers_by_type, to_json, yaml_load, +from pygeoapi.util import (filter_dict_by_key_value, to_json, yaml_load, get_api_rules, get_base_url) LOGGER = logging.getLogger(__name__) @@ -151,8 +148,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: paths = {} # TODO: make openapi multilingual (default language only for now) - server_locales = l10n.get_locales(cfg) - locale_ = server_locales[0] + locale_ = l10n.get_locales(cfg)[0] api_rules = get_api_rules(cfg) @@ -282,11 +278,6 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'url': cfg['metadata']['identification']['url']} } ) - oas['tags'].append({ - 'name': 'stac', - 'description': 'SpatioTemporal Asset Catalog' - } - ) oas['components'] = { 'responses': { @@ -307,19 +298,183 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'schema': {'$ref': '#/components/schemas/queryables'} } } + } + }, + 'parameters': get_oas_30_parameters(cfg=cfg, locale_=locale_), + 'schemas': { + # TODO: change this schema once OGC will definitively publish it + 'queryable': { + 'type': 'object', + 'required': [ + 'queryable', + 'type' + ], + 'properties': { + 'queryable': { + 'description': 'the token that may be used in a CQL predicate', # noqa + 'type': 'string' + }, + 'title': { + 'description': 'a human readable title for the queryable', # noqa + 'type': 'string' + }, + 'description': { + 'description': 'a human-readable narrative describing the queryable', # noqa + 'type': 'string' + }, + 'language': { + 'description': 'the language used for the title and description', # noqa + 'type': 'string', + 'default': [ + 'en' + ] + }, + 'type': { + 'description': 'the data type of the queryable', # noqa + 'type': 'string' + }, + 'type-ref': { + 'description': 'a reference to the formal definition of the type', # noqa + 'type': 'string', + 'format': 'url' + } + } }, - 'Tiles': { - 'description': 'Retrieves the tiles description for this collection', # noqa - 'content': { - 'application/json': { - 'schema': { - '$ref': '#/components/schemas/tiles' + 'queryables': { + 'type': 'object', + 'required': [ + 'queryables' + ], + 'properties': { + 'queryables': { + 'type': 'array', + 'items': {'$ref': '#/components/schemas/queryable'} + } + } + } + } + } + + items_f = deepcopy(oas['components']['parameters']['f']) + items_f['schema']['enum'].append('csv') + + LOGGER.debug('setting up datasets') + + for k, v in get_visible_collections(cfg).items(): + name = l10n.translate(k, locale_) + title = l10n.translate(v['title'], locale_) + desc = l10n.translate(v['description'], locale_) + collection_name_path = f'/collections/{k}' + tag = { + 'name': name, + 'description': desc, + 'externalDocs': {} + } + for link in l10n.translate(v.get('links', []), locale_): + if link['type'] == 'information': + tag['externalDocs']['description'] = link['type'] + tag['externalDocs']['url'] = link['url'] + break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] + + oas['tags'].append(tag) + + paths[collection_name_path] = { + 'get': { + 'summary': f'Get {title} metadata', + 'description': desc, + 'tags': [name], + 'operationId': f'describe{name.capitalize()}Collection', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Collection"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + oas['components']['responses'].update({ + 'Tiles': { + 'description': 'Retrieves the tiles description for this collection', # noqa + 'content': { + 'application/json': { + 'schema': { + '$ref': '#/components/schemas/tiles' + } } } } } - }, - 'parameters': { + ) + + oas['components']['schemas'].update({ + 'tilematrixsetlink': { + 'type': 'object', + 'required': ['tileMatrixSet'], + 'properties': { + 'tileMatrixSet': { + 'type': 'string' + }, + 'tileMatrixSetURI': { + 'type': 'string' + } + } + }, + 'tiles': { + 'type': 'object', + 'required': [ + 'tileMatrixSetLinks', + 'links' + ], + 'properties': { + 'tileMatrixSetLinks': { + 'type': 'array', + 'items': { + '$ref': '#/components/schemas/tilematrixsetlink' # noqa + } + }, + 'links': { + 'type': 'array', + 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa + } + } + } + } + ) + + oas['paths'] = paths + + for api_name, api_module in all_apis().items(): + LOGGER.debug(f'Adding OpenAPI definitions for {api_name}') + + try: + sub_tags, sub_paths = api_module.get_oas_30(cfg, locale_) + oas['paths'].update(sub_paths['paths']) + oas['tags'].extend(sub_tags) + except Exception as err: + if fail_on_invalid_collection: + raise + else: + LOGGER.warning(f'Resource not added to OpenAPI: {err}') + + if cfg['server'].get('admin', False): + schema_dict = get_config_schema() + oas['definitions'] = schema_dict['definitions'] + LOGGER.debug('Adding admin endpoints') + oas['paths'].update(get_admin()) + + return oas + + +def get_oas_30_parameters(cfg: dict, locale_: str): + server_locales = l10n.get_locales(cfg) + return { 'f': { 'name': 'f', 'in': 'query', @@ -344,20 +499,6 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'default': l10n.locale2str(locale_) } }, - 'properties': { - 'name': 'properties', - 'in': 'query', - 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'items': { - 'type': 'string' - } - } - }, 'skipGeometry': { 'name': 'skipGeometry', 'in': 'query', @@ -461,330 +602,31 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'type': 'string' } } - }, - 'schemas': { - # TODO: change this schema once OGC will definitively publish it - 'queryable': { - 'type': 'object', - 'required': [ - 'queryable', - 'type' - ], - 'properties': { - 'queryable': { - 'description': 'the token that may be used in a CQL predicate', # noqa - 'type': 'string' - }, - 'title': { - 'description': 'a human readable title for the queryable', # noqa - 'type': 'string' - }, - 'description': { - 'description': 'a human-readable narrative describing the queryable', # noqa - 'type': 'string' - }, - 'language': { - 'description': 'the language used for the title and description', # noqa - 'type': 'string', - 'default': [ - 'en' - ] - }, - 'type': { - 'description': 'the data type of the queryable', # noqa - 'type': 'string' - }, - 'type-ref': { - 'description': 'a reference to the formal definition of the type', # noqa - 'type': 'string', - 'format': 'url' - } - } - }, - 'queryables': { - 'type': 'object', - 'required': [ - 'queryables' - ], - 'properties': { - 'queryables': { - 'type': 'array', - 'items': {'$ref': '#/components/schemas/queryable'} - } - } - }, - 'tilematrixsetlink': { - 'type': 'object', - 'required': ['tileMatrixSet'], - 'properties': { - 'tileMatrixSet': { - 'type': 'string' - }, - 'tileMatrixSetURI': { - 'type': 'string' - } - } - }, - 'tiles': { - 'type': 'object', - 'required': [ - 'tileMatrixSetLinks', - 'links' - ], - 'properties': { - 'tileMatrixSetLinks': { - 'type': 'array', - 'items': { - '$ref': '#/components/schemas/tilematrixsetlink' # noqa - } - }, - 'links': { - 'type': 'array', - 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa - } - } - } } - } - LOGGER.debug('setting up datasets') + +def get_visible_collections(cfg: dict) -> dict: collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') - for k, v in collections.items(): - try: - LOGGER.debug(f'Generating OpenAPI tags/paths for collection {k}') - r_tags, r_paths = handle_collection(locale_, oas['components'], - k, v) - oas['tags'].extend(r_tags) - paths.update(r_paths) - except Exception as err: - if fail_on_invalid_collection: - raise - else: - LOGGER.warning(f'Resource {k} not added to OpenAPI: {err}') + return { + k: v + for k, v in collections.items() + if v.get('visibility', 'default') != 'hidden' + } - LOGGER.debug('setting up STAC') - stac_collections = filter_dict_by_key_value(cfg['resources'], - 'type', 'stac-collection') - if stac_collections: - paths['/stac'] = { - 'get': { - 'summary': 'SpatioTemporal Asset Catalog', - 'description': 'SpatioTemporal Asset Catalog', - 'tags': ['stac'], - 'operationId': 'getStacCatalog', - 'parameters': [], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } - process_manager = get_manager(cfg) +def get_config_schema(): + schema_file = os.path.join(THISDIR, 'schemas', 'config', + 'pygeoapi-config-0.x.yml') - if len(process_manager.processes) > 0: - paths['/processes'] = { - 'get': { - 'summary': 'Processes', - 'description': 'Processes', - 'tags': ['server'], - 'operationId': 'getProcesses', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } - } - } - LOGGER.debug('setting up processes') - - for k, v in process_manager.processes.items(): - if k.startswith('_'): - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - name = l10n.translate(k, locale_) - p = process_manager.get_processor(k) - md_desc = l10n.translate(p.metadata['description'], locale_) - process_name_path = f'/processes/{name}' - tag = { - 'name': name, - 'description': md_desc, # noqa - 'externalDocs': {} - } - for link in p.metadata.get('links', []): - if link['type'] == 'information': - translated_link = l10n.translate(link, locale_) - tag['externalDocs']['description'] = translated_link[ - 'type'] - tag['externalDocs']['url'] = translated_link['url'] - break - if len(tag['externalDocs']) == 0: - del tag['externalDocs'] - - oas['tags'].append(tag) - - paths[process_name_path] = { - 'get': { - 'summary': 'Get process metadata', - 'description': md_desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Process', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } + with open(schema_file) as fh2: + return yaml_load(fh2) - paths[f'{process_name_path}/execution'] = { - 'post': { - 'summary': f"Process {l10n.translate(p.metadata['title'], locale_)} execution", # noqa - 'description': md_desc, - 'tags': [name], - 'operationId': f'execute{name.capitalize()}Job', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - }, - 'requestBody': { - 'description': 'Mandatory execute request JSON', - 'required': True, - 'content': { - 'application/json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa - } - } - } - } - } - } - if 'example' in p.metadata: - paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa - name_in_path = { - 'name': 'jobId', - 'in': 'path', - 'description': 'job identifier', - 'required': True, - 'schema': { - 'type': 'string' - } - } +def get_admin(): - paths['/jobs'] = { - 'get': { - 'summary': 'Retrieve jobs list', - 'description': 'Retrieve a list of jobs', - 'tags': ['jobs'], - 'operationId': 'getJobs', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - paths['/jobs/{jobId}'] = { - 'get': { - 'summary': 'Retrieve job details', - 'description': 'Retrieve job details', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJob', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - 'delete': { - 'summary': 'Cancel / delete job', - 'description': 'Cancel / delete job', - 'tags': ['jobs'], - 'parameters': [ - name_in_path - ], - 'operationId': 'deleteJob', - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - } - - paths['/jobs/{jobId}/results'] = { - 'get': { - 'summary': 'Retrieve job results', - 'description': 'Retrive job resiults', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJobResults', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - } - } - - tag = { - 'name': 'jobs', - 'description': 'Process jobs', - } - oas['tags'].insert(1, tag) - - oas['paths'] = paths - - if cfg['server'].get('admin', False): - schema_dict = get_config_schema() - oas['definitions'] = schema_dict['definitions'] - LOGGER.debug('Adding admin endpoints') - oas['paths'].update(get_admin()) - - return oas - - -def get_config_schema() -> dict: - """ - Get configuration schema - - :returns: `dict` of configuration schema - """ - - schema_file = os.path.join(THISDIR, 'schemas', 'config', - 'pygeoapi-config-0.x.yml') - - with open(schema_file) as fh2: - return yaml_load(fh2) - - -def get_admin() -> dict: - """ - Generate admin paths for OpenAPI Document - - :returns: `dict` of paths - """ - - schema_dict = get_config_schema() + schema_dict = get_config_schema() paths = {} @@ -800,6 +642,7 @@ def get_admin() -> dict: ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict @@ -861,6 +704,7 @@ def get_admin() -> dict: ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -903,6 +747,7 @@ def get_admin() -> dict: ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -976,630 +821,6 @@ def get_admin() -> dict: return paths -def handle_collection(locale_: str, components: dict, collection_id: str, - collection_def: dict) -> Tuple[list, dict]: - """ - Generate relevant OpenAPI constructs for a given collection - - :param locale_: locale - :param components: OpenAPI components - :param collection_id: collection identifier - :param collection_def: collection definition - - :returns: `tuple` of `list` of tags and `dict` of paths - """ - - paths = {} - tags = [] - - items_f = deepcopy(components['parameters']['f']) - items_f['schema']['enum'].append('csv') - items_l = deepcopy(components['parameters']['lang']) - - if collection_def.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {collection_id}') - return [], {} - - name = l10n.translate(collection_id, locale_) - title = l10n.translate(collection_def['title'], locale_) - desc = l10n.translate(collection_def['description'], locale_) - collection_name_path = f'/collections/{collection_id}' - - tag = { - 'name': name, - 'description': desc, - 'externalDocs': {} - } - - for link in l10n.translate(collection_def.get('links', []), locale_): - if link['type'] == 'information': - tag['externalDocs']['description'] = link['type'] - tag['externalDocs']['url'] = link['url'] - break - - if len(tag['externalDocs']) == 0: - del tag['externalDocs'] - - tags.append(tag) - - paths[collection_name_path] = { - 'get': { - 'summary': f'Get {title} metadata', - 'description': desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Collection', - 'parameters': [ - {'$ref': '#/components/parameters/f'}, - {'$ref': '#/components/parameters/lang'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Collection"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - LOGGER.debug('setting up collection endpoints') - try: - ptype = None - - if filter_providers_by_type(collection_def['providers'], 'feature'): - ptype = 'feature' - - if filter_providers_by_type(collection_def['providers'], 'record'): - ptype = 'record' - - p = load_plugin('provider', get_provider_by_type( - collection_def['providers'], ptype)) - - items_path = f'{collection_name_path}/items' - - coll_properties = deepcopy(components['parameters']['properties']) # noqa - - coll_properties['schema']['items']['enum'] = list(p.fields.keys()) - - paths[items_path] = { - 'get': { - 'summary': f'Get {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Features', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/bbox-crs'}, # noqa - coll_properties, - {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa - {'$ref': '#/components/parameters/skipGeometry'}, - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa - {'$ref': '#/components/parameters/offset'}, - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Features', - 'responses': { - '200': {'description': 'options response'} - } - } - } - - if p.editable: - LOGGER.debug('Provider is editable; adding post') - - paths[items_path]['post'] = { - 'summary': f'Add {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'add{name.capitalize()}Features', - 'requestBody': { - 'description': 'Adds item to collection', - 'content': { - 'application/geo+json': { - 'schema': {} - } - }, - 'required': True - }, - 'responses': { - '201': {'description': 'Successful creation'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.create) - paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - if ptype == 'record': - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) - if p.fields: - schema_path = f'{collection_name_path}/schema' - - paths[schema_path] = { - 'get': { - 'summary': f'Get {title} schema', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - - queryables_path = f'{collection_name_path}/queryables' - - paths[queryables_path] = { - 'get': { - 'summary': f'Get {title} queryables', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - - if p.time_field is not None: - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - for field, type_ in p.fields.items(): - - if p.properties and field not in p.properties: - LOGGER.debug('Provider specified not to advertise property') - continue - - if field == 'q' and ptype == 'record': - LOGGER.debug('q parameter already declared, skipping') - continue - - if type_ == 'date': - schema = { - 'type': 'string', - 'format': 'date' - } - elif type_ == 'float': - schema = { - 'type': 'number', - 'format': 'float' - } - elif type_ == 'long': - schema = { - 'type': 'integer', - 'format': 'int64' - } - else: - schema = type_ - - path_ = f'{collection_name_path}/items' - paths[path_]['get']['parameters'].append({ - 'name': field, - 'in': 'query', - 'required': False, - 'schema': schema, - 'style': 'form', - 'explode': False - }) - - paths[f'{collection_name_path}/items/{{featureId}}'] = { - 'get': { - 'summary': f'Get {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/f'}, - {'$ref': '#/components/parameters/lang'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa - ], - 'responses': { - '200': {'description': 'options response'} - } - } - } - - try: - schema_ref = p.get_schema() - paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa - 'content': { - schema_ref[0]: { - 'schema': schema_ref[1] - } - } - } - except Exception as err: - LOGGER.debug(err) - - if p.editable: - LOGGER.debug('Provider is editable; adding put/delete') - put_path = f'{collection_name_path}/items/{{featureId}}' - paths[put_path]['put'] = { # noqa - 'summary': f'Update {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'update{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa - ], - 'requestBody': { - 'description': 'Updates item in collection', - 'content': { - 'application/geo+json': { - 'schema': {} - } - }, - 'required': True - }, - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.replace) - paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { - 'summary': f'Delete {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'delete{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - ], - 'responses': { - '200': {'description': 'Successful delete'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - except ProviderTypeError: - LOGGER.debug('collection is not feature based') - - LOGGER.debug('setting up coverage endpoints') - try: - load_plugin('provider', get_provider_by_type( - collection_def['providers'], 'coverage')) - - coverage_path = f'{collection_name_path}/coverage' - - paths[coverage_path] = { - 'get': { - 'summary': f'Get {title} coverage', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Coverage', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': '#/components/parameters/bbox-crs'}, - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - except ProviderTypeError: - LOGGER.debug('collection is not coverage based') - - LOGGER.debug('setting up tiles endpoints') - tile_extension = filter_providers_by_type( - collection_def['providers'], 'tile') - - if tile_extension: - tp = load_plugin('provider', tile_extension) - - tiles_path = f'{collection_name_path}/tiles' - - paths[tiles_path] = { - 'get': { - 'summary': f'Fetch a {title} tiles description', - 'description': desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Tiles', - 'parameters': [ - items_f, - # items_l TODO: is this useful? - ], - 'responses': { - '200': {'$ref': '#/components/responses/Tiles'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - tiles_data_path = f'{collection_name_path}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa - - paths[tiles_data_path] = { - 'get': { - 'summary': f'Get a {title} tile', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Tiles', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa - { - 'name': 'f', - 'in': 'query', - 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': [tp.format_type], - 'default': tp.format_type - }, - 'style': 'form', - 'explode': False - } - ], - 'responses': { - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - mimetype = tile_extension['format']['mimetype'] - paths[tiles_data_path]['get']['responses']['200'] = { - 'description': 'successful operation', - 'content': { - mimetype: { - 'schema': { - 'type': 'string', - 'format': 'binary' - } - } - } - } - - LOGGER.debug('setting up edr endpoints') - edr_extension = filter_providers_by_type( - collection_def['providers'], 'edr') - - if edr_extension: - ep = load_plugin('provider', edr_extension) - - edr_query_endpoints = [] - - for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/{qt}', - 'qt': qt, - 'op_id': f'query{qt.capitalize()}{collection_id.capitalize()}' - }) - if ep.instances: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa - 'qt': qt, - 'op_id': f'query{qt.capitalize()}Instance{collection_id.capitalize()}' # noqa - }) - - for eqe in edr_query_endpoints: - if eqe['qt'] == 'cube': - spatial_parameter = 'bbox' - else: - spatial_parameter = f"{eqe['qt']}Coords" - paths[eqe['path']] = { - 'get': { - 'summary': f"query {collection_def['description']} by {eqe['qt']}", # noqa - 'description': collection_def['description'], - 'tags': [collection_id], - 'operationId': eqe['op_id'], - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - if 'locations' in ep.get_query_types(): - paths[f'{collection_name_path}/locations'] = { - 'get': { - 'summary': f"Get pre-defined locations of {v['description']}", # noqa - 'description': collection_def['description'], - 'tags': [collection_id], - 'operationId': f'queryLOCATIONS{collection_id.capitalize()}', # noqa - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - paths[f'{collection_name_path}/locations/{{locId}}'] = { - 'get': { - 'summary': f"query {collection_defv['description']} by location", # noqa - 'description': collection_def['description'], - 'tags': [collection_id], - 'operationId': f'queryLOCATIONSBYID{collection_id.capitalize()}', # noqa - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - - LOGGER.debug('setting up maps endpoints') - map_extension = filter_providers_by_type( - collection_def['providers'], 'map') - - if map_extension: - mp = load_plugin('provider', map_extension) - - map_f = deepcopy(components['parameters']['f']) - map_f['schema']['enum'] = [map_extension['format']['name']] - map_f['schema']['default'] = map_extension['format']['name'] - - pth = f'/collections/{collection_def}/map' - paths[pth] = { - 'get': { - 'summary': 'Get map', - 'description': f"{collection_def['description']} map", - 'tags': [collection_id], - 'operationId': 'getMap', - 'parameters': [ - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - { - 'name': 'width', - 'in': 'query', - 'description': 'Response image width', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'height', - 'in': 'query', - 'description': 'Response image height', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'transparent', - 'in': 'query', - 'description': 'Background transparency of map (default=true).', # noqa - 'required': False, - 'schema': { - 'type': 'boolean', - 'default': True, - }, - 'style': 'form', - 'explode': False - }, - {'$ref': '#/components/parameters/bbox-crs-epsg'}, - map_f - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/json': {} - } - }, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - if mp.time_field is not None: - paths[pth]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - return tags, paths - - def get_oas(cfg: dict, fail_on_invalid_collection: bool = True, version='3.0') -> dict: """ @@ -1647,9 +868,9 @@ def generate_openapi_document(cfg_file: Union[Path, io.TextIOWrapper], :param cfg_file: configuration Path instance (`str` of filepath or parsed `dict`) + :param output_format: output format for OpenAPI document :param fail_on_invalid_collection: `bool` of whether to fail on an invalid collection - :param output_format: output format for OpenAPI document :returns: `str` of the OpenAPI document in the output format requested """ @@ -1670,7 +891,6 @@ def generate_openapi_document(cfg_file: Union[Path, io.TextIOWrapper], content = yaml.safe_dump(oas, default_flow_style=False) else: content = to_json(oas, pretty=pretty_print) - return content diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index ad208f30f..d4d8f23b4 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -52,7 +52,7 @@ JobStatus, ProcessExecutionMode, RequestedProcessExecutionMode, - Subscriber, + Subscriber ) LOGGER = logging.getLogger(__name__) @@ -318,7 +318,6 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, jfmt = 'application/json' self.update_job(job_id, job_metadata) - self._send_failed_notification(subscriber) return jfmt, outputs, current_status @@ -328,7 +327,7 @@ def execute_process( process_id: str, data_dict: dict, execution_mode: Optional[RequestedProcessExecutionMode] = None, - subscriber: Optional[Subscriber] = None, + subscriber: Optional[Subscriber] = None ) -> Tuple[str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -390,6 +389,7 @@ def execute_process( # managers **({'subscriber': subscriber} if self.supports_subscribing else {}) ) + return job_id, mime_type, outputs, status, response_headers def _send_in_progress_notification(self, subscriber: Optional[Subscriber]): diff --git a/pygeoapi/process/manager/dummy.py b/pygeoapi/process/manager/dummy.py index 6676617ed..eb35eee09 100644 --- a/pygeoapi/process/manager/dummy.py +++ b/pygeoapi/process/manager/dummy.py @@ -72,7 +72,7 @@ def execute_process( process_id: str, data_dict: dict, execution_mode: Optional[RequestedProcessExecutionMode] = None, - subscriber: Optional[Subscriber] = None, + subscriber: Optional[Subscriber] = None ) -> Tuple[str, str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index d1595c4d3..2609e2947 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -398,12 +398,15 @@ def _get_coverage_properties(self): } if 'crs' in self._data.variables.keys(): - properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa + try: + properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa - properties['inverse_flattening'] = self._data.crs.\ - inverse_flattening + properties['inverse_flattening'] = self._data.crs.\ + inverse_flattening - properties['crs_type'] = 'ProjectedCRS' + properties['crs_type'] = 'ProjectedCRS' + except AttributeError: + pass properties['axes'] = [ properties['x_axis_label'], diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 8264566dd..6d6b7054c 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -50,7 +50,14 @@ ) import uvicorn -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -113,10 +120,12 @@ async def get_response( """ loop = asyncio.get_running_loop() - result = await loop.run_in_executor( + headers, status, content = await loop.run_in_executor( None, call_api_threadsafe, loop, api_call, *args) + return _to_response(headers, status, content) - headers, status, content = result + +def _to_response(headers, status, content): if headers['Content-Type'] == 'text/html': response = HTMLResponse(content=content, status_code=status) else: @@ -130,6 +139,27 @@ async def get_response( return response +async def execute_from_starlette(api_function, request: Request, *args, + skip_valid_check=False) -> Response: + api_request = await APIRequest.from_starlette(request, api_.locales) + content: str | bytes + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + loop = asyncio.get_running_loop() + headers, status, content = await loop.run_in_executor( + None, call_api_threadsafe, loop, api_function, + api_, api_request, *args) + # NOTE: that gzip currently doesn't work in starlette + # https://github.com/geopython/pygeoapi/issues/1591 + content = apply_gzip(headers, content) + + response = _to_response(headers, status, content) + + return response + + async def landing_page(request: Request): """ OGC API landing page endpoint @@ -173,8 +203,9 @@ async def get_tilematrix_set(request: Request, tileMatrixSetId=None): if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.tilematrixset, request, tileMatrixSetId) + return await execute_from_starlette( + tiles_api.tilematrixset, request, tileMatrixSetId, + ) async def get_tilematrix_sets(request: Request): @@ -183,7 +214,7 @@ async def get_tilematrix_sets(request: Request): :returns: HTTP response """ - return await get_response(api_.tilematrixsets, request) + return await execute_from_starlette(tiles_api.tilematrixsets, request) async def collection_schema(request: Request, collection_id=None): @@ -198,8 +229,8 @@ async def collection_schema(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_schema, request, collection_id) + return await get_response(api_.get_collection_schema, request, + collection_id) async def collection_queryables(request: Request, collection_id=None): @@ -214,8 +245,9 @@ async def collection_queryables(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_queryables, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_queryables, request, collection_id, + ) async def get_collection_tiles(request: Request, collection_id=None): @@ -229,8 +261,9 @@ async def get_collection_tiles(request: Request, collection_id=None): """ if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_tiles, request, collection_id) + + return await execute_from_starlette( + tiles_api.get_collection_tiles, request, collection_id) async def get_collection_tiles_metadata(request: Request, collection_id=None, @@ -247,9 +280,10 @@ async def get_collection_tiles_metadata(request: Request, collection_id=None, collection_id = request.path_params['collection_id'] if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.get_collection_tiles_metadata, request, - collection_id, tileMatrixSetId + + return await execute_from_starlette( + tiles_api.get_collection_tiles_metadata, request, + collection_id, tileMatrixSetId, skip_valid_check=True, ) @@ -278,9 +312,10 @@ async def get_collection_items_tiles(request: Request, collection_id=None, tileRow = request.path_params['tileRow'] if 'tileCol' in request.path_params: tileCol = request.path_params['tileCol'] - return await get_response( - api_.get_collection_tiles_data, request, collection_id, - tileMatrixSetId, tile_matrix, tileRow, tileCol + return await execute_from_starlette( + tiles_api.get_collection_tiles_data, request, collection_id, + tileMatrixSetId, tile_matrix, tileRow, tileCol, + skip_valid_check=True, ) @@ -301,45 +336,47 @@ async def collection_items(request: Request, collection_id=None, item_id=None): item_id = request.path_params['item_id'] if item_id is None: if request.method == 'GET': # list items - return await get_response( - api_.get_collection_items, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_items, request, collection_id, + skip_valid_check=True) elif request.method == 'POST': # filter or manage items content_type = request.headers.get('content-type') if content_type is not None: if content_type == 'application/geo+json': - return await get_response( - api_.manage_collection_item, request, - 'create', collection_id) + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, + 'create', collection_id, skip_valid_check=True) else: - return await get_response( - api_.post_collection_items, + return await execute_from_starlette( + itemtypes_api.post_collection_items, request, - collection_id + collection_id, + skip_valid_check=True, ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, - 'options', collection_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, + 'options', collection_id, skip_valid_check=True, ) elif request.method == 'DELETE': - return await get_response( - api_.manage_collection_item, request, 'delete', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'delete', + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'PUT': - return await get_response( - api_.manage_collection_item, request, 'update', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'update', + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, 'options', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'options', + collection_id, item_id, skip_valid_check=True, ) else: - return await get_response( - api_.get_collection_item, request, collection_id, item_id) + return await execute_from_starlette( + itemtypes_api.get_collection_item, request, collection_id, item_id) async def collection_coverage(request: Request, collection_id=None): @@ -354,8 +391,8 @@ async def collection_coverage(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_coverage, request, collection_id) + return await execute_from_starlette( + coverages_api.get_collection_coverage, request, collection_id) async def collection_map(request: Request, collection_id, style_id=None): @@ -373,8 +410,9 @@ async def collection_map(request: Request, collection_id, style_id=None): if 'style_id' in request.path_params: style_id = request.path_params['style_id'] - return await get_response( - api_.get_collection_map, request, collection_id, style_id) + return await execute_from_starlette( + maps_api.get_collection_map, request, collection_id, style_id + ) async def get_processes(request: Request, process_id=None): @@ -389,7 +427,8 @@ async def get_processes(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.describe_processes, request, process_id) + return await execute_from_starlette(processes_api.describe_processes, + request, process_id) async def get_jobs(request: Request, job_id=None): @@ -406,12 +445,14 @@ async def get_jobs(request: Request, job_id=None): job_id = request.path_params['job_id'] if job_id is None: # list of submit job - return await get_response(api_.get_jobs, request) + return await execute_from_starlette(processes_api.get_jobs, request) else: # get or delete job if request.method == 'DELETE': - return await get_response(api_.delete_job, job_id) + return await execute_from_starlette(processes_api.delete_job, + request, job_id) else: # Return status of a specific job - return await get_response(api_.get_jobs, request, job_id) + return await execute_from_starlette(processes_api.get_jobs, + request, job_id) async def execute_process_jobs(request: Request, process_id=None): @@ -427,7 +468,8 @@ async def execute_process_jobs(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.execute_process, request, process_id) + return await execute_from_starlette(processes_api.execute_process, + request, process_id) async def get_job_result(request: Request, job_id=None): @@ -443,7 +485,8 @@ async def get_job_result(request: Request, job_id=None): if 'job_id' in request.path_params: job_id = request.path_params['job_id'] - return await get_response(api_.get_job_result, request, job_id) + return await execute_from_starlette(processes_api.get_job_result, + request, job_id) async def get_job_result_resource(request: Request, @@ -463,6 +506,7 @@ async def get_job_result_resource(request: Request, if 'resource' in request.path_params: resource = request.path_params['resource'] + # TODO: this api function currently doesn't exist return await get_response( api_.get_job_result_resource, request, job_id, resource) @@ -484,9 +528,10 @@ async def get_collection_edr_query(request: Request, collection_id=None, instanc instance_id = request.path_params['instance_id'] query_type = request["path"].split('/')[-1] # noqa - return await get_response( - api_.get_collection_edr_query, request, collection_id, - instance_id, query_type + return await execute_from_starlette( + edr_api.get_collection_edr_query, request, collection_id, + instance_id, query_type, + skip_valid_check=True, ) @@ -513,7 +558,7 @@ async def stac_catalog_root(request: Request): :returns: Starlette HTTP response """ - return await get_response(api_.get_stac_root, request) + return await execute_from_starlette(stac_api.get_stac_root, request) async def stac_catalog_path(request: Request): @@ -525,7 +570,7 @@ async def stac_catalog_path(request: Request): :returns: Starlette HTTP response """ path = request.path_params["path"] - return await get_response(api_.get_stac_path, request, path) + return await execute_from_starlette(stac_api.get_stac_path, request, path) async def admin_config(request: Request): diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 2ccbeb25f..7e1f64e22 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -609,10 +609,12 @@ class JobStatus(Enum): @dataclass(frozen=True) class Subscriber: - """Store subscriber urls as defined in: + """ + Store subscriber URLs as defined in: https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/subscriber.yaml # noqa """ + success_uri: str in_progress_uri: Optional[str] failed_uri: Optional[str] diff --git a/tests/api/test_api.py b/tests/api/test_api.py new file mode 100644 index 000000000..c11afca28 --- /dev/null +++ b/tests/api/test_api.py @@ -0,0 +1,869 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +import gzip +from http import HTTPStatus + +from pyld import jsonld +import pytest + +from pygeoapi.api import ( + API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP, + __version__, validate_bbox, validate_datetime, + validate_subset +) +from pygeoapi.util import yaml_load, get_api_rules, get_base_url + +from tests.util import (get_test_file_path, mock_flask, mock_starlette, + mock_request) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_with_rules() -> dict: + """ Returns a pygeoapi configuration with default API rules. """ + with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_enclosure() -> dict: + """ Returns a pygeoapi configuration with enclosure links. """ + with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_hidden_resources(): + filename = 'pygeoapi-test-config-hidden-resources.yml' + with open(get_test_file_path(filename)) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def enclosure_api(config_enclosure, openapi): + """ Returns an API instance with a collection with enclosure links. """ + return API(config_enclosure, openapi) + + +@pytest.fixture() +def rules_api(config_with_rules, openapi): + """ Returns an API instance with URL prefix and strict slashes policy. + The API version is extracted from the current version here. + """ + return API(config_with_rules, openapi) + + +@pytest.fixture() +def api_hidden_resources(config_hidden_resources, openapi): + return API(config_hidden_resources, openapi) + + +def test_apirequest(api_): + # Test without (valid) locales + with pytest.raises(ValueError): + req = mock_request() + APIRequest(req, []) + APIRequest(req, None) + APIRequest(req, ['zz']) + + # Test all supported formats from query args + for f, mt in FORMAT_TYPES.items(): + req = mock_request({'f': f}) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test all supported formats from Accept header + for f, mt in FORMAT_TYPES.items(): + req = mock_request(HTTP_ACCEPT=mt) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test nonsense format + req = mock_request({'f': 'foo'}) + apireq = APIRequest(req, api_.locales) + assert not apireq.is_valid() + assert apireq.format == 'foo' + assert apireq.is_valid(('foo',)) + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + + # Test without format + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format is None + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + assert apireq.get_linkrel(F_JSON) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Test complex format string + hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + assert apireq.get_linkrel(F_HTML) == 'self' + assert apireq.get_linkrel(F_JSON) == 'alternate' + + # Test accept header with multiple valid formats + hh = 'plain/text,application/ld+json,application/json;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_JSONLD + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSONLD] + assert apireq.get_linkrel(F_JSONLD) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Overrule HTTP content negotiation + req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + + # Test data + for d in (None, '', 'test', {'key': 'value'}): + req = mock_request(data=d) + apireq = APIRequest.with_data(req, api_.locales) + if not d: + assert apireq.data == b'' + elif isinstance(d, dict): + assert d == json.loads(apireq.data) + else: + assert apireq.data == d.encode() + + # Test multilingual + test_lang = { + 'nl': ('en', 'en-US'), # unsupported lang should return default + 'en-US': ('en', 'en-US'), + 'de_CH': ('en', 'en-US'), + 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), + 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), + } + sup_lang = ('en-US', 'fr_CA') + for lang_in, (lang_out, cl_out) in test_lang.items(): + # Using l query parameter + req = mock_request({'lang': lang_in}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Using Accept-Language header + req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Test language override + req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'fr' + assert apireq.locale.language == 'fr' + assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' + + # Test locale territory + req = mock_request({'lang': 'en-GB'}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'en-GB' + assert apireq.locale.language == 'en' + assert apireq.locale.territory == 'US' + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query parameter + # (should return default language from YAML config) + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.raw_locale is None + assert apireq.locale.language == api_.default_locale.language + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query param + # (should return first in custom list of languages) + sup_lang = ('de', 'fr', 'en') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale is None + assert apireq.locale.language == 'de' + assert apireq.get_response_headers()['Content-Language'] == 'de' + + +def test_apirules_active(config_with_rules, rules_api): + assert rules_api.config == config_with_rules + rules = get_api_rules(config_with_rules) + base_url = get_base_url(config_with_rules) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: + # Test happy path + response = flask_client.get(f'{flask_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = flask_client.get(f'{flask_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = flask_client.get(f'{flask_prefix}/') + assert response.status_code == 200 + response = flask_client.get(flask_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = flask_client.get(flask_prefix, follow_redirects=True) + assert response.status_code == 200 + assert response.is_json + links = response.json['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa + # Test happy path + response = starlette_client.get(f'{starlette_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = starlette_client.get(f'{starlette_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = starlette_client.get(f'{starlette_prefix}/') + assert response.status_code == 200 + response = starlette_client.get(starlette_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa + assert response.status_code == 200 + links = response.json()['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + +def test_apirules_inactive(config, api_): + assert api_.config == config + rules = get_api_rules(config) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + assert flask_prefix == '' + with mock_flask('pygeoapi-test-config.yml') as flask_client: + response = flask_client.get('') + assert response.status_code == 200 + response = flask_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = flask_client.get('/') + assert response.status_code == 200 + response = flask_client.get('/conformance/') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + assert starlette_prefix == '' + with mock_starlette('pygeoapi-test-config.yml') as starlette_client: + response = starlette_client.get('') + assert response.status_code == 200 + response = starlette_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert str(response.url) == f"{starlette_client.base_url}/conformance" + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = starlette_client.get('/') + assert response.status_code == 200 + response = starlette_client.get('/conformance/', follow_redirects=True) + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + +def test_api(config, api_, openapi): + assert api_.config == config + assert isinstance(api_.config, dict) + + req = mock_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + root = json.loads(response) + assert isinstance(root, dict) + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request(HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'Swagger UI' in response + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'ReDoc' in response + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + assert api_.get_collections_url() == 'http://localhost:5000/collections' + + +def test_api_exception(config, api_): + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + # When a language is set, the exception should still be English + req = mock_request({'f': 'foo', 'lang': 'fr'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + +def test_gzip(config, api_, openapi): + # Requests for each response type and gzip encoding + req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', + HTTP_ACCEPT_ENCODING=F_GZIP) + + # Responses from server config without gzip compression + rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Add gzip to server and use utf-16 encoding + config['server']['gzip'] = True + enc_16 = 'utf-16' + config['server']['encoding'] = enc_16 + api_ = API(config, openapi) + + # Responses from server with gzip compression + rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) + rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) + rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) + rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) + + # Validate compressed json response + assert rsp_json_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' + assert rsp_json_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) + assert isinstance(parsed_gzip_json, str) + parsed_gzip_json = json.loads(parsed_gzip_json) + assert isinstance(parsed_gzip_json, dict) + assert parsed_gzip_json == json.loads(rsp_json) + + # Validate compressed jsonld response + assert rsp_jsonld_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' + assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + assert isinstance(parsed_gzip_jsonld, str) + parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) + assert isinstance(parsed_gzip_jsonld, dict) + assert parsed_gzip_jsonld == json.loads(rsp_jsonld) + + # Validate compressed html response + assert rsp_html_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' + assert rsp_html_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) + assert isinstance(parsed_gzip_html, str) + assert parsed_gzip_html == rsp_html + + # Validate compressed gzip response + assert rsp_gzip_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' + assert rsp_gzip_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) + assert isinstance(parsed_gzip_gzip, str) + parsed_gzip_gzip = json.loads(parsed_gzip_gzip) + assert isinstance(parsed_gzip_gzip, dict) + + # Requests without content encoding header + req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) + req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) + req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) + + # Responses without content encoding + _, _, rsp_json_ = api_.landing_page(req_json) + _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) + _, _, rsp_html_ = api_.landing_page(req_html) + + # Confirm each request is the same when decompressed + assert rsp_json_ == rsp_json == \ + gzip.decompress(rsp_gzip_json).decode(enc_16) + + assert rsp_jsonld_ == rsp_jsonld == \ + gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + + assert rsp_html_ == rsp_html == \ + gzip.decompress(rsp_gzip_html).decode(enc_16) + + +def test_root(config, api_): + req = mock_request() + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/json' == \ + FORMAT_TYPES[F_JSON] + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + assert rsp_headers['Content-Language'] == 'en-US' + + assert isinstance(root, dict) + assert 'links' in root + assert root['links'][0]['rel'] == 'self' + assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] + assert root['links'][0]['href'].endswith('?f=json') + assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' + for link in root['links']) + assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' + for link in root['links']) + assert len(root['links']) == 11 + assert 'title' in root + assert root['title'] == 'pygeoapi default instance' + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_root_structured_data(config, api_): + req = mock_request({"f": "jsonld"}) + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/ld+json' == \ + FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + + assert isinstance(root, dict) + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + assert '@context' in root + assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' + expanded = jsonld.expand(root)[0] + assert '@type' in expanded + assert 'http://schema.org/DataCatalog' in expanded['@type'] + assert 'http://schema.org/description' in expanded + assert root['description'] == expanded['http://schema.org/description'][0][ + '@value'] + assert 'http://schema.org/keywords' in expanded + assert len(expanded['http://schema.org/keywords']) == 3 + assert '@value' in expanded['http://schema.org/keywords'][0].keys() + assert 'http://schema.org/provider' in expanded + assert expanded['http://schema.org/provider'][0]['@type'][ + 0] == 'http://schema.org/Organization' + assert expanded['http://schema.org/name'][0]['@value'] == root['name'] + + +def test_conformance(config, api_): + req = mock_request() + rsp_headers, code, response = api_.conformance(req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'conformsTo' in root + assert len(root['conformsTo']) == 37 + assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ + in root['conformsTo'] + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.conformance(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.conformance(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections(config, api_): + req = mock_request({"f": "foo"}) + rsp_headers, code, response = api_.describe_collections(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({"f": "html"}) + rsp_headers, code, response = api_.describe_collections(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request() + rsp_headers, code, response = api_.describe_collections(req) + collections = json.loads(response) + + assert len(collections) == 2 + assert len(collections['collections']) == 9 + assert len(collections['links']) == 3 + + rsp_headers, code, response = api_.describe_collections(req, 'foo') + collection = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'en-US' + assert collection['id'] == 'obs' + assert collection['title'] == 'Observations' + assert collection['description'] == 'My cool observations' + assert len(collection['links']) == 14 + assert collection['extent'] == { + 'spatial': { + 'bbox': [[-180, -90, 180, 90]], + 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + }, + 'temporal': { + 'interval': [ + ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] + ], + 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' + } + } + + # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults + assert collection['crs'] is not None + crs_set = [ + 'http://www.opengis.net/def/crs/EPSG/0/28992', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/EPSG/0/4326', + ] + for crs in crs_set: + assert crs in collection['crs'] + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert 'storageCrsCoordinateEpoch' not in collection + + # French language request + req = mock_request({'lang': 'fr'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'fr-CA' + assert collection['title'] == 'Observations' + assert collection['description'] == 'Mes belles observations' + + # Check HTML request in an unsupported language + req = mock_request({'f': 'html', 'lang': 'de'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + # hiearchical collections + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'naturalearth/lakes') + + collection = json.loads(response) + assert collection['id'] == 'naturalearth/lakes' + + # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured + assert collection['crs'] is not None + default_crs_list = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', + ] + contains_default = False + for crs in default_crs_list: + if crs in default_crs_list: + contains_default = True + assert contains_default + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert collection['storageCrsCoordinateEpoch'] == 2017.23 + + +def test_describe_collections_hidden_resources( + config_hidden_resources, api_hidden_resources): + req = mock_request({}) + rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa + assert code == HTTPStatus.OK + + assert len(config_hidden_resources['resources']) == 3 + + collections = json.loads(response) + assert len(collections['collections']) == 1 + + +def test_describe_collections_json_ld(config, api_): + req = mock_request({'f': 'jsonld'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert '@context' in collection + expanded = jsonld.expand(collection)[0] + # Metadata is about a schema:DataCollection that contains a schema:Dataset + assert not expanded['@id'].endswith('obs') + assert 'http://schema.org/dataset' in expanded + assert len(expanded['http://schema.org/dataset']) == 1 + dataset = expanded['http://schema.org/dataset'][0] + assert dataset['@type'][0] == 'http://schema.org/Dataset' + assert len(dataset['http://schema.org/distribution']) == 14 + assert all(dist['@type'][0] == 'http://schema.org/DataDownload' + for dist in dataset['http://schema.org/distribution']) + + assert 'http://schema.org/Organization' in expanded[ + 'http://schema.org/provider'][0]['@type'] + + assert 'http://schema.org/Place' in dataset[ + 'http://schema.org/spatial'][0]['@type'] + assert 'http://schema.org/GeoShape' in dataset[ + 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] + assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ + 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' + + assert 'http://schema.org/temporalCoverage' in dataset + assert dataset['http://schema.org/temporalCoverage'][0][ + '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections_enclosures(config_enclosure, enclosure_api): + original_enclosures = { + lnk['title']: lnk + for lnk in config_enclosure['resources']['objects']['links'] + if lnk['rel'] == 'enclosure' + } + + req = mock_request() + _, _, response = enclosure_api.describe_collections(req, 'objects') + features = json.loads(response) + modified_enclosures = { + lnk['title']: lnk for lnk in features['links'] + if lnk['rel'] == 'enclosure' + } + + # If type and length is set, do not verify/update link + assert original_enclosures['download link 1'] == \ + modified_enclosures['download link 1'] + # If length is missing, modify link type and length + assert original_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 2']['type'] + assert modified_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 3']['type'] + assert 'length' not in original_enclosures['download link 2'] + assert modified_enclosures['download link 2']['length'] > 0 + assert modified_enclosures['download link 2']['length'] == \ + modified_enclosures['download link 3']['length'] + assert original_enclosures['download link 3']['type'] != \ + modified_enclosures['download link 3']['type'] + + +def test_get_collection_schema(config, api_): + req = mock_request() + rsp_headers, code, response = api_.get_collection_schema(req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 5 + + +def test_validate_bbox(): + assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] + assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] + assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] + assert (validate_bbox('-142.1,42.12,-52.22,84.4') == + [-142.1, 42.12, -52.22, 84.4]) + assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == + [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) + + assert (validate_bbox('177.0,65.0,-177.0,70.0') == + [177.0, 65.0, -177.0, 70.0]) + + with pytest.raises(ValueError): + validate_bbox('1,2,4') + + with pytest.raises(ValueError): + validate_bbox('1,2,4,5,6') + + with pytest.raises(ValueError): + validate_bbox('3,4,1,2') + + with pytest.raises(ValueError): + validate_bbox('1,2,6,4,5,3') + + +def test_validate_datetime(): + config = yaml_load(''' + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + ''') + + # test time instant + assert validate_datetime(config, '2004') == '2004' + assert validate_datetime(config, '2004-10') == '2004-10' + assert validate_datetime(config, '2001-10-30') == '2001-10-30' + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009-10-30') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-09-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-10-30T17:24:39Z') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-10-30T08:58:29Z') + + # test time envelope + assert validate_datetime(config, '2004/2005') == '2004/2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + assert validate_datetime(config, '2004/..') == '2004/..' + assert validate_datetime(config, '../2005') == '../2005' + assert validate_datetime(config, '2004/') == '2004/..' + assert validate_datetime(config, '/2005') == '../2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-11-01/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../2000-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../1999') + + +@pytest.mark.parametrize("value, expected", [ + ('time(2000-11-11)', {'time': ['2000-11-11']}), + ('time("2000-11-11")', {'time': ['2000-11-11']}), + ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), + ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa + ('lat(40)', {'lat': [40]}), + ('lat(0:40)', {'lat': [0, 40]}), + ('foo("bar")', {'foo': ['bar']}), + ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) +]) +def test_validate_subset(value, expected): + assert validate_subset(value) == expected + + with pytest.raises(ValueError): + validate_subset('foo("bar)') + + +def test_get_exception(config, api_): + d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') + assert d[0] == {} + assert d[1] == 500 + content = json.loads(d[2]) + assert content['code'] == 'NoApplicableCode' + assert content['description'] == 'oops' + + d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') diff --git a/tests/api/test_coverages.py b/tests/api/test_coverages.py new file mode 100644 index 000000000..39b275228 --- /dev/null +++ b/tests/api/test_coverages.py @@ -0,0 +1,177 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +from http import HTTPStatus + +import pytest + +from pygeoapi.api.coverages import get_collection_coverage +from pygeoapi.util import yaml_load + +from tests.util import get_test_file_path, mock_request, mock_api_request + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +def test_describe_collections(config, api_): + + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'gdps-temperature') + + collection = json.loads(response) + + assert collection['id'] == 'gdps-temperature' + assert len(collection['links']) == 10 + assert collection['extent']['spatial']['grid'][0]['cellsCount'] == 2400 + assert collection['extent']['spatial']['grid'][0]['resolution'] == 0.15000000000000002 # noqa + assert collection['extent']['spatial']['grid'][1]['cellsCount'] == 1201 + assert collection['extent']['spatial']['grid'][1]['resolution'] == 0.15 + + +def test_get_collection_schema(config, api_): + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') + + assert rsp_headers['Content-Type'] == 'application/schema+json' + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') + + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 1 + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 1 + assert schema['properties']['1']['type'] == 'number' + assert schema['properties']['1']['title'] == 'Temperature [C]' + + +def test_get_collection_coverage(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_coverage( + api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'properties': '12'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'subset': 'bad_axis(10:20)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'blah'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + assert rsp_headers['Content-Type'] == 'text/html' + + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + # NOTE: This test used to assert the code to be 200 OK, + # but it requested HTML, which is not available, + # so it should be 400 Bad Request + assert code == HTTPStatus.BAD_REQUEST + assert rsp_headers['Content-Type'] == 'text/html' + + req = mock_api_request({'subset': 'Lat(5:10),Long(5:10)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + content = json.loads(response) + + assert content['domain']['axes']['x']['num'] == 35 + assert content['domain']['axes']['y']['num'] == 35 + assert 'TMP' in content['parameters'] + assert 'TMP' in content['ranges'] + assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] + + req = mock_api_request({'bbox': '-79,45,-75,49'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + content = json.loads(response) + + assert content['domain']['axes']['x']['start'] == -79.0 + assert content['domain']['axes']['x']['stop'] == -75.0 + assert content['domain']['axes']['y']['start'] == 49.0 + assert content['domain']['axes']['y']['stop'] == 45.0 + + req = mock_api_request({ + 'subset': 'Lat(5:10),Long(5:10)', + 'f': 'GRIB' + }) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + assert isinstance(response, bytes) + + req = mock_api_request(HTTP_ACCEPT='application/x-netcdf') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'cmip5') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'application/x-netcdf' diff --git a/tests/api/test_environmental_data_retrieval.py b/tests/api/test_environmental_data_retrieval.py new file mode 100644 index 000000000..0de57f023 --- /dev/null +++ b/tests/api/test_environmental_data_retrieval.py @@ -0,0 +1,226 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus + +from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query + +from tests.util import mock_api_request + + +def test_get_collection_edr_query(config, api_): + # edr resource + req = mock_api_request() + rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') + collection = json.loads(response) + parameter_names = list(collection['parameter_names'].keys()) + parameter_names.sort() + assert len(parameter_names) == 4 + assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] + + # no coords parameter + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # bad query type + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'corridor') + assert code == HTTPStatus.BAD_REQUEST + + # bad coords parameter + req = mock_api_request({'coords': 'gah'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # bad parameter_names parameter + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'parameter_names': 'bad' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # all parameters + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + + axes = list(data['domain']['axes'].keys()) + axes.sort() + assert len(axes) == 3 + assert axes == ['TIME', 'x', 'y'] + + assert data['domain']['axes']['x']['start'] == 11.0 + assert data['domain']['axes']['x']['stop'] == 11.0 + assert data['domain']['axes']['y']['start'] == 11.0 + assert data['domain']['axes']['y']['stop'] == 11.0 + + parameters = list(data['parameters'].keys()) + parameters.sort() + assert len(parameters) == 4 + assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] + + # single parameter + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'parameter_names': 'SST' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + + assert len(data['parameters'].keys()) == 1 + assert list(data['parameters'].keys())[0] == 'SST' + + # Zulu time zone + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + # bounded date range + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-01-17/2000-06-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-02-15T16:29:05.999999999' + assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' + assert time_dict['num'] == 5 + + # unbounded date range - start + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '../2000-06-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-01-16T06:00:00.000000000' + assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' + assert time_dict['num'] == 6 + + # unbounded date range - end + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-06-16/..' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-06-16T10:25:30.000000000' + assert time_dict['stop'] == '2000-12-16T01:20:05.999999996' + assert time_dict['num'] == 7 + + # some data + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + # no data + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.NO_CONTENT + + # position no coords + req = mock_api_request({ + 'datetime': '2000-01-17' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # cube bbox parameter 4 dimensional + req = mock_api_request({ + 'bbox': '0,0,10,10' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.OK + + # cube bad bbox parameter + req = mock_api_request({ + 'bbox': '0,0,10' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.BAD_REQUEST + + # cube no bbox parameter + req = mock_api_request({}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.BAD_REQUEST + + # cube decreasing latitude coords and S3 + req = mock_api_request({ + 'bbox': '-100,40,-99,45', + 'parameter_names': 'tmn', + 'datetime': '1994-01-01/1994-12-31', + }) + + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'usgs-prism', None, 'cube') + assert code == HTTPStatus.OK diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py new file mode 100644 index 000000000..1ced8ea45 --- /dev/null +++ b/tests/api/test_itemtypes.py @@ -0,0 +1,659 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import copy +import gzip +import json +from http import HTTPStatus + +from pyld import jsonld +import pytest +import pyproj +from shapely.geometry import Point + +from pygeoapi.api import (API, FORMAT_TYPES, F_GZIP, F_HTML, F_JSONLD, + apply_gzip) +from pygeoapi.api.itemtypes import ( + get_collection_queryables, get_collection_item, + get_collection_items, manage_collection_item) +from pygeoapi.util import yaml_load, get_crs_from_uri + +from tests.util import get_test_file_path, mock_api_request + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +def test_get_collection_queryables(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_queryables( + api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 5 + + # test with provider filtered properties + api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] + + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 2 + assert 'geometry' in queryables['properties'] + assert queryables['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Geometry.json' # noqa + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_get_collection_items(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'foo') + features = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'foo'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3,4c'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + # bbox-crs must be in configured values for Collection + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + # bbox-crs must be in configured values for Collection (CSV will ignore) + req = mock_api_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + # bbox-crs can be a default even if not configured + req = mock_api_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + # bbox-crs can be a default even if not configured + req = mock_api_request({'bbox': '4,52,5,53'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'f': 'html', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'fr-CA' + + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + assert len(features['features']) == 5 + + req = mock_api_request({'resulttype': 'hits'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 0 + + # Invalid limit + req = mock_api_request({'limit': 0}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'stn_id': '35'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 2 + assert features['numberMatched'] == 2 + + req = mock_api_request({'stn_id': '35', 'value': '93.9'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 1 + assert features['numberMatched'] == 1 + + req = mock_api_request({'limit': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 2 + assert features['features'][1]['properties']['stn_id'] == 35 + + links = features['links'] + assert len(links) == 4 + assert '/collections/obs/items?f=json' in links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld' in links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html' in links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs' in links[3]['href'] + assert links[3]['rel'] == 'collection' + + # Invalid offset + req = mock_api_request({'offset': -1}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'offset': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 3 + assert features['features'][1]['properties']['stn_id'] == 2147 + + links = features['links'] + assert len(links) == 5 + assert '/collections/obs/items?f=json' in links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld' in links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html' in links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0' in links[3]['href'] + assert links[3]['rel'] == 'prev' + assert '/collections/obs' in links[4]['href'] + assert links[4]['rel'] == 'collection' + + req = mock_api_request({ + 'offset': 1, + 'limit': 1, + 'bbox': '-180,90,180,90' + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 1 + + links = features['links'] + assert len(links) == 5 + assert '/collections/obs/items?f=json&limit=1&bbox=-180,90,180,90' in \ + links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld&limit=1&bbox=-180,90,180,90' in \ + links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html&limit=1&bbox=-180,90,180,90' in \ + links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0&limit=1&bbox=-180,90,180,90' \ + in links[3]['href'] + assert links[3]['rel'] == 'prev' + assert '/collections/obs' in links[4]['href'] + assert links[4]['rel'] == 'collection' + + req = mock_api_request({ + 'sortby': 'bad-property', + 'stn_id': '35' + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'sortby': 'stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'sortby': '+stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'sortby': '-stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'f': 'csv'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' + + req = mock_api_request({'datetime': '2003'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'datetime': '2010-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'datetime': '2001-11-11/2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '../2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '2001-11-11/..'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999/2005-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999/2000-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + api_.config['resources']['obs']['extents'].pop('temporal') + + req = mock_api_request({'datetime': '2002/2014-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'scalerank': 1}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') + features = json.loads(response) + + assert len(features['features']) == 10 + assert features['numberMatched'] == 11 + assert features['numberReturned'] == 10 + + req = mock_api_request({'datetime': '2005-04-22'}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'skipGeometry': 'true'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert json.loads(response)['features'][0]['geometry'] is None + + req = mock_api_request({'properties': 'foo,bar'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + +def test_collection_items_gzip_csv(config, api_, openapi): + # Add gzip to server + config['server']['gzip'] = True + api_ = API(config, openapi) + + req_csv = mock_api_request({'f': 'csv'}) + rsp_csv_headers, _, rsp_csv = get_collection_items(api_, req_csv, 'obs') + rsp_csv = apply_gzip(rsp_csv_headers, rsp_csv) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv = rsp_csv.decode('utf-8') + + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') + assert rsp_csv == rsp_csv_ + + # Use utf-16 encoding + config['server']['encoding'] = 'utf-16' + api_ = API(config, openapi) + + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') + assert rsp_csv == rsp_csv_ + + +def test_get_collection_items_crs(config, api_): + + # Invalid CRS query parameter + req = mock_api_request({'crs': '4326'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') + + assert code == HTTPStatus.BAD_REQUEST + + # Unsupported CRS + req = mock_api_request( + {'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') + + assert code == HTTPStatus.BAD_REQUEST + + # Supported CRSs + default_crs = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + storage_crs = 'http://www.opengis.net/def/crs/EPSG/0/25833' + crs_4258 = 'http://www.opengis.net/def/crs/EPSG/0/4258' + supported_crs_list = [default_crs, storage_crs, crs_4258] + + for crs in supported_crs_list: + req = mock_api_request({'crs': crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{crs}>' + + # With CRS query parameter, using storageCRS + req = mock_api_request({'crs': storage_crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' + + features_25833 = json.loads(response) + + # With CRS query parameter resulting in coordinates transformation + req = mock_api_request({'crs': crs_4258}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' + + features_4258 = json.loads(response) + transform_func = pyproj.Transformer.from_crs( + pyproj.CRS.from_epsg(25833), + pyproj.CRS.from_epsg(4258), + always_xy=False, + ).transform + for feat_orig in features_25833['features']: + id_ = feat_orig['id'] + x, y, *_ = feat_orig['geometry']['coordinates'] + loc_transf = Point(transform_func(x, y)) + for feat_out in features_4258['features']: + if id_ == feat_out['id']: + loc_out = Point(feat_out['geometry']['coordinates'][:2]) + + assert loc_out.equals_exact(loc_transf, 1e-5) + break + + # Without CRS query parameter: assume Transform to default WGS84 lon,lat + req = mock_api_request({}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{default_crs}>' + + features_wgs84 = json.loads(response) + + # With CRS query parameter resulting in coordinates transformation + transform_func = pyproj.Transformer.from_crs( + pyproj.CRS.from_epsg(4258), + get_crs_from_uri(default_crs), + always_xy=False, + ).transform + for feat_orig in features_4258['features']: + id_ = feat_orig['id'] + x, y, *_ = feat_orig['geometry']['coordinates'] + loc_transf = Point(transform_func(x, y)) + for feat_out in features_wgs84['features']: + if id_ == feat_out['id']: + loc_out = Point(feat_out['geometry']['coordinates'][:2]) + + assert loc_out.equals_exact(loc_transf, 1e-5) + break + + +def test_manage_collection_item_read_only_options_req(config, api_): + """Test OPTIONS request on a read-only items endpoint""" + req = mock_api_request() + _, code, _ = manage_collection_item(api_, req, 'options', 'foo') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET' + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET' + + +def test_manage_collection_item_editable_options_req(config, openapi): + """Test OPTIONS request on a editable items endpoint""" + config = copy.deepcopy(config) + config['resources']['obs']['providers'][0]['editable'] = True + api_ = API(config, openapi) + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET, POST' + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' + + +def test_get_collection_items_json_ld(config, api_): + req = mock_api_request({ + 'f': 'jsonld', + 'limit': 2 + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + collection = json.loads(response) + + assert '@context' in collection + assert all((f in collection['@context'][0] for + f in ('schema', 'type', 'features', 'FeatureCollection'))) + assert len(collection['@context']) > 1 + assert collection['@context'][1]['schema'] == 'https://schema.org/' + expanded = jsonld.expand(collection)[0] + featuresUri = 'https://schema.org/itemListElement' + assert len(expanded[featuresUri]) == 2 + + +def test_get_collection_item(config, api_): + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_item( + api_, req, 'gdps-temperature', '371') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'foo', '371') + + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = get_collection_item( + api_, req, 'obs', 'notfound') + + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + feature = json.loads(response) + + assert feature['properties']['stn_id'] == 35 + assert 'prev' not in feature['links'] + assert 'next' not in feature['links'] + + +def test_get_collection_item_json_ld(config, api_): + req = mock_api_request({'f': 'jsonld'}) + rsp_headers, _, response = get_collection_item(api_, req, 'objects', '3') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + feature = json.loads(response) + assert '@context' in feature + assert all((f in feature['@context'][0] for + f in ('schema', 'type', 'gsp'))) + assert len(feature['@context']) == 1 + assert 'schema' in feature['@context'][0] + assert feature['@context'][0]['schema'] == 'https://schema.org/' + assert feature['id'] == 3 + expanded = jsonld.expand(feature)[0] + + assert expanded['@id'].startswith('http://') + assert expanded['@id'].endswith('/collections/objects/items/3') + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'POINT (-85 33)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/latitude'][0][ + '@value'] == 33 + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/longitude'][0][ + '@value'] == -85 + + _, _, response = get_collection_item(api_, req, 'objects', '2') + feature = json.loads(response) + assert feature['geometry']['type'] == 'MultiPoint' + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTIPOINT (10 40, 40 30, 20 20, 30 10)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" + + _, _, response = get_collection_item(api_, req, 'objects', '1') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'LINESTRING (30 10, 10 30, 40 40)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/line'][0][ + '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' + + _, _, response = get_collection_item(api_, req, 'objects', '4') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTILINESTRING ((10 10, 20 20, 10 40), ' \ + '(40 40, 30 30, 40 20, 30 10))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/line'][0][ + '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ + '30.0,30.0 40.0,20.0 30.0,10.0' + + _, _, response = get_collection_item(api_, req, 'objects', '5') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' + + _, _, response = get_collection_item(api_, req, 'objects', '7') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '\ + '((15 5, 40 10, 10 20, 5 10, 15 5)))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ + '45.0,40.0 40.0,10.0 15.0,5.0' + + req = mock_api_request({'f': 'jsonld', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'fr-CA' diff --git a/tests/api/test_maps.py b/tests/api/test_maps.py new file mode 100644 index 000000000..3b284ac0a --- /dev/null +++ b/tests/api/test_maps.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus + +from pygeoapi.api.maps import get_collection_map + +from tests.util import mock_api_request + + +def test_get_collection_map(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_map(api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request() + rsp_headers, code, response = get_collection_map( + api_, req, 'mapserver_world_map') + assert code == HTTPStatus.OK + assert isinstance(response, bytes) + assert response[1:4] == b'PNG' diff --git a/tests/api/test_processes.py b/tests/api/test_processes.py new file mode 100644 index 000000000..b83305fae --- /dev/null +++ b/tests/api/test_processes.py @@ -0,0 +1,428 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus +import time +from unittest import mock + +from pygeoapi.api import FORMAT_TYPES, F_HTML, F_JSON +from pygeoapi.api.processes import ( + describe_processes, execute_process, delete_job, get_job_result, +) + +from tests.util import mock_api_request + + +def test_describe_processes(config, api_): + req = mock_api_request({'limit': 1}) + # Test for description of single processes + rsp_headers, code, response = describe_processes(api_, req) + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 1 + assert len(data['links']) == 3 + + req = mock_api_request() + + # Test for undefined process + rsp_headers, code, response = describe_processes(api_, req, 'foo') + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert data['code'] == 'NoSuchProcess' + + # Test for description of all processes + rsp_headers, code, response = describe_processes(api_, req) + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 2 + assert len(data['links']) == 3 + + # Test for particular, defined process + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + process = json.loads(response) + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert process['id'] == 'hello-world' + assert process['version'] == '0.2.0' + assert process['title'] == 'Hello World' + assert len(process['keywords']) == 3 + assert len(process['links']) == 6 + assert len(process['inputs']) == 2 + assert len(process['outputs']) == 1 + assert len(process['outputTransmission']) == 1 + assert len(process['jobControlOptions']) == 2 + assert 'sync-execute' in process['jobControlOptions'] + assert 'async-execute' in process['jobControlOptions'] + + # Check HTML response when requested in headers + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested in headers + req = mock_api_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'en-US' + + # Check HTML response when requested with query parameter + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested with query parameter + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested with French language parameter + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'fr-CA' + process = json.loads(response) + assert process['title'] == 'Bonjour le Monde' + + # Check JSON response when language requested in headers + req = mock_api_request(HTTP_ACCEPT_LANGUAGE='fr') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'fr-CA' + + # Test for undefined process + req = mock_api_request() + rsp_headers, code, response = describe_processes(api_, req, + 'goodbye-world') + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert data['code'] == 'NoSuchProcess' + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Test describe doesn't crash if example is missing + req = mock_api_request() + processor = api_.manager.get_processor("hello-world") + example = processor.metadata.pop("example") + rsp_headers, code, response = describe_processes(api_, req) + processor.metadata['example'] = example + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 2 + + +def test_execute_process(config, api_): + req_body_0 = { + 'inputs': { + 'name': 'Test' + } + } + req_body_1 = { + 'inputs': { + 'name': 'Test' + }, + 'response': 'document' + } + req_body_2 = { + 'inputs': { + 'name': 'Tést' + } + } + req_body_3 = { + 'inputs': { + 'name': 'Tést', + 'message': 'This is a test.' + } + } + req_body_4 = { + 'inputs': { + 'foo': 'Tést' + } + } + req_body_5 = { + 'inputs': {} + } + req_body_6 = { + 'inputs': { + 'name': None + } + } + req_body_7 = { + 'inputs': { + 'name': 'Test' + }, + 'subscriber': { + 'successUri': 'https://example.com/success', + 'inProgressUri': 'https://example.com/inProgress', + 'failedUri': 'https://example.com/failed', + } + } + + cleanup_jobs = set() + + # Test posting empty payload to existing process + req = mock_api_request(data='') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert rsp_headers['Content-Language'] == 'en-US' + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' not in rsp_headers + assert data['code'] == 'MissingParameterValue' + + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'foo') + + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert 'Location' not in rsp_headers + assert data['code'] == 'NoSuchProcess' + + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + + assert len(data.keys()) == 2 + assert data['id'] == 'echo' + assert data['value'] == 'Hello Test!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_1) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + + assert len(data.keys()) == 1 + assert data['outputs'][0]['id'] == 'echo' + assert data['outputs'][0]['value'] == 'Hello Test!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_2) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Tést!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_3) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Tést! This is a test.' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_4) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_5) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + assert data['description'] == 'Error updating job' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_6) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + assert data['description'] == 'Error updating job' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'goodbye-world') + + response = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert 'Location' not in rsp_headers + assert response['code'] == 'NoSuchProcess' + + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + response = json.loads(response) + assert code == HTTPStatus.OK + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_1, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + assert 'Location' in rsp_headers + response = json.loads(response) + assert isinstance(response, dict) + assert code == HTTPStatus.CREATED + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_7) + with mock.patch( + 'pygeoapi.process.manager.base.requests.post' + ) as post_mocker: + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert code == HTTPStatus.OK + post_mocker.assert_any_call( + req_body_7['subscriber']['inProgressUri'], json={} + ) + post_mocker.assert_any_call( + req_body_7['subscriber']['successUri'], + json={'id': 'echo', 'value': 'Hello Test!'} + ) + assert post_mocker.call_count == 2 + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + # Cleanup + time.sleep(2) # Allow time for any outstanding async jobs + for _, job_id in cleanup_jobs: + rsp_headers, code, response = delete_job(api_, mock_api_request(), + job_id) + assert code == HTTPStatus.OK + + +def _execute_a_job(api_): + req_body_sync = { + 'inputs': { + 'name': 'Sync Test' + } + } + + req = mock_api_request(data=req_body_sync) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Sync Test!' + + job_id = rsp_headers['Location'].split('/')[-1] + return job_id + + +def test_delete_job(api_): + rsp_headers, code, response = delete_job(api_, mock_api_request(), + 'does-not-exist') + + assert code == HTTPStatus.NOT_FOUND + req_body_async = { + 'inputs': { + 'name': 'Async Test Deletion' + } + } + job_id = _execute_a_job(api_) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + + assert code == HTTPStatus.OK + + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request(data=req_body_async, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + assert code == HTTPStatus.CREATED + assert 'Location' in rsp_headers + + time.sleep(2) # Allow time for async execution to complete + job_id = rsp_headers['Location'].split('/')[-1] + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.OK + + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.NOT_FOUND + + +def test_get_job_result(api_): + rsp_headers, code, response = get_job_result( + api_, mock_api_request(), 'not-exist', + ) + assert code == HTTPStatus.NOT_FOUND + + job_id = _execute_a_job(api_) + rsp_headers, code, response = get_job_result(api_, mock_api_request(), + job_id) + # default response is html + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'text/html' + assert 'Hello Sync Test!' in response + + rsp_headers, code, response = get_job_result( + api_, mock_api_request({'f': 'json'}), job_id, + ) + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'application/json' + assert json.loads(response)['value'] == "Hello Sync Test!" diff --git a/tests/api/test_tiles.py b/tests/api/test_tiles.py new file mode 100644 index 000000000..df11ca5f3 --- /dev/null +++ b/tests/api/test_tiles.py @@ -0,0 +1,110 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus + +from pygeoapi.api import FORMAT_TYPES, F_HTML +from pygeoapi.api.tiles import ( + get_collection_tiles, tilematrixset, tilematrixsets, +) +from pygeoapi.models.provider.base import TileMatrixSetEnum + +from tests.util import mock_api_request + + +def test_get_collection_tiles(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_tiles(api_, req, 'obs') + assert code == HTTPStatus.BAD_REQUEST + + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') + assert code == HTTPStatus.OK + + # Language settings should be ignored (return system default) + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') + assert rsp_headers['Content-Language'] == 'en-US' + content = json.loads(response) + assert len(content['links']) > 0 + assert len(content['tilesets']) > 0 + + +def test_tilematrixsets(config, api_): + req = mock_api_request() + rsp_headers, code, response = tilematrixsets(api_, req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'tileMatrixSets' in root + assert len(root['tileMatrixSets']) == 2 + assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad' \ + in root['tileMatrixSets'][0]['uri'] + assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ + in root['tileMatrixSets'][1]['uri'] + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixsets(api_, req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_tilematrixset(config, api_): + req = mock_api_request() + + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + for e in enums: + enum = e.tileMatrixSet + rsp_headers, code, response = tilematrixset(api_, req, enum) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'id' in root + assert root['id'] == enum + assert 'tileMatrices' in root + assert len(root['tileMatrices']) == 30 + + rsp_headers, code, response = tilematrixset(api_, req, 'foo') + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixset(api_, req, enum) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..ddeb3b496 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Bernhard Mallinger +# +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import pytest + +from pygeoapi.api import API +from pygeoapi.util import yaml_load + +from tests.util import get_test_file_path + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def openapi(): + with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def api_(config, openapi): + return API(config, openapi) diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index 3c13af48d..000000000 --- a/tests/test_api.py +++ /dev/null @@ -1,2262 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# John A Stevenson -# Colin Blackburn -# -# Copyright (c) 2023 Tom Kralidis -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -import copy -import json -import logging -import time -import gzip -from http import HTTPStatus -from unittest import mock - -from pyld import jsonld -import pytest -import pyproj -from shapely.geometry import Point - -from pygeoapi.api import ( - API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, - validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ -) -from pygeoapi.util import (yaml_load, get_crs_from_uri, - get_api_rules, get_base_url) - -from .util import (get_test_file_path, mock_request, - mock_flask, mock_starlette) - -from pygeoapi.models.provider.base import TileMatrixSetEnum - -LOGGER = logging.getLogger(__name__) - - -@pytest.fixture() -def config(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_with_rules() -> dict: - """ Returns a pygeoapi configuration with default API rules. """ - with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_enclosure() -> dict: - """ Returns a pygeoapi configuration with enclosure links. """ - with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_hidden_resources(): - filename = 'pygeoapi-test-config-hidden-resources.yml' - with open(get_test_file_path(filename)) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def openapi(): - with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def api_(config, openapi): - return API(config, openapi) - - -@pytest.fixture() -def enclosure_api(config_enclosure, openapi): - """ Returns an API instance with a collection with enclosure links. """ - return API(config_enclosure, openapi) - - -@pytest.fixture() -def rules_api(config_with_rules, openapi): - """ Returns an API instance with URL prefix and strict slashes policy. - The API version is extracted from the current version here. - """ - return API(config_with_rules, openapi) - - -@pytest.fixture() -def api_hidden_resources(config_hidden_resources, openapi): - return API(config_hidden_resources, openapi) - - -def test_apirequest(api_): - # Test without (valid) locales - with pytest.raises(ValueError): - req = mock_request() - APIRequest(req, []) - APIRequest(req, None) - APIRequest(req, ['zz']) - - # Test all supported formats from query args - for f, mt in FORMAT_TYPES.items(): - req = mock_request({'f': f}) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test all supported formats from Accept header - for f, mt in FORMAT_TYPES.items(): - req = mock_request(HTTP_ACCEPT=mt) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test nonsense format - req = mock_request({'f': 'foo'}) - apireq = APIRequest(req, api_.locales) - assert not apireq.is_valid() - assert apireq.format == 'foo' - assert apireq.is_valid(('foo',)) - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - - # Test without format - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format is None - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - assert apireq.get_linkrel(F_JSON) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Test complex format string - hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - assert apireq.get_linkrel(F_HTML) == 'self' - assert apireq.get_linkrel(F_JSON) == 'alternate' - - # Test accept header with multiple valid formats - hh = 'plain/text,application/ld+json,application/json;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_JSONLD - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSONLD] - assert apireq.get_linkrel(F_JSONLD) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Overrule HTTP content negotiation - req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - - # Test data - for d in (None, '', 'test', {'key': 'value'}): - req = mock_request(data=d) - apireq = APIRequest.with_data(req, api_.locales) - if not d: - assert apireq.data == b'' - elif isinstance(d, dict): - assert d == json.loads(apireq.data) - else: - assert apireq.data == d.encode() - - # Test multilingual - test_lang = { - 'nl': ('en', 'en-US'), # unsupported lang should return default - 'en-US': ('en', 'en-US'), - 'de_CH': ('en', 'en-US'), - 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), - 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), - } - sup_lang = ('en-US', 'fr_CA') - for lang_in, (lang_out, cl_out) in test_lang.items(): - # Using l query parameter - req = mock_request({'lang': lang_in}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Using Accept-Language header - req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Test language override - req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'fr' - assert apireq.locale.language == 'fr' - assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' - - # Test locale territory - req = mock_request({'lang': 'en-GB'}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'en-GB' - assert apireq.locale.language == 'en' - assert apireq.locale.territory == 'US' - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query parameter - # (should return default language from YAML config) - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.raw_locale is None - assert apireq.locale.language == api_.default_locale.language - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query param - # (should return first in custom list of languages) - sup_lang = ('de', 'fr', 'en') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale is None - assert apireq.locale.language == 'de' - assert apireq.get_response_headers()['Content-Language'] == 'de' - - -def test_apirules_active(config_with_rules, rules_api): - assert rules_api.config == config_with_rules - rules = get_api_rules(config_with_rules) - base_url = get_base_url(config_with_rules) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: - # Test happy path - response = flask_client.get(f'{flask_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = flask_client.get(f'{flask_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = flask_client.get(f'{flask_prefix}/') - assert response.status_code == 200 - response = flask_client.get(flask_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = flask_client.get(flask_prefix, follow_redirects=True) - assert response.status_code == 200 - assert response.is_json - links = response.json['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa - # Test happy path - response = starlette_client.get(f'{starlette_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = starlette_client.get(f'{starlette_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = starlette_client.get(f'{starlette_prefix}/') - assert response.status_code == 200 - response = starlette_client.get(starlette_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa - assert response.status_code == 200 - links = response.json()['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - -def test_apirules_inactive(config, api_): - assert api_.config == config - rules = get_api_rules(config) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - assert flask_prefix == '' - with mock_flask('pygeoapi-test-config.yml') as flask_client: - response = flask_client.get('') - assert response.status_code == 200 - response = flask_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = flask_client.get('/') - assert response.status_code == 200 - response = flask_client.get('/conformance/') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - assert starlette_prefix == '' - with mock_starlette('pygeoapi-test-config.yml') as starlette_client: - response = starlette_client.get('') - assert response.status_code == 200 - response = starlette_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert str(response.url) == f"{starlette_client.base_url}/conformance" - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = starlette_client.get('/') - assert response.status_code == 200 - response = starlette_client.get('/conformance/', follow_redirects=True) - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - -def test_api(config, api_, openapi): - assert api_.config == config - assert isinstance(api_.config, dict) - - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - root = json.loads(response) - assert isinstance(root, dict) - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request(HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'Swagger UI' in response - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'ReDoc' in response - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - assert api_.get_collections_url() == 'http://localhost:5000/collections' - - -def test_api_exception(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - # When a language is set, the exception should still be English - req = mock_request({'f': 'foo', 'lang': 'fr'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - -def test_gzip(config, api_): - # Requests for each response type and gzip encoding - req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', - HTTP_ACCEPT_ENCODING=F_GZIP) - - # Responses from server config without gzip compression - rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Add gzip to server and use utf-16 encoding - config['server']['gzip'] = True - enc_16 = 'utf-16' - config['server']['encoding'] = enc_16 - api_ = API(config, openapi) - - # Responses from server with gzip compression - rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) - rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) - rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) - rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) - - # Validate compressed json response - assert rsp_json_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' - assert rsp_json_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) - assert isinstance(parsed_gzip_json, str) - parsed_gzip_json = json.loads(parsed_gzip_json) - assert isinstance(parsed_gzip_json, dict) - assert parsed_gzip_json == json.loads(rsp_json) - - # Validate compressed jsonld response - assert rsp_jsonld_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' - assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - assert isinstance(parsed_gzip_jsonld, str) - parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) - assert isinstance(parsed_gzip_jsonld, dict) - assert parsed_gzip_jsonld == json.loads(rsp_jsonld) - - # Validate compressed html response - assert rsp_html_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' - assert rsp_html_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) - assert isinstance(parsed_gzip_html, str) - assert parsed_gzip_html == rsp_html - - # Validate compressed gzip response - assert rsp_gzip_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' - assert rsp_gzip_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) - assert isinstance(parsed_gzip_gzip, str) - parsed_gzip_gzip = json.loads(parsed_gzip_gzip) - assert isinstance(parsed_gzip_gzip, dict) - - # Requests without content encoding header - req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) - req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) - req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) - - # Responses without content encoding - _, _, rsp_json_ = api_.landing_page(req_json) - _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) - _, _, rsp_html_ = api_.landing_page(req_html) - - # Confirm each request is the same when decompressed - assert rsp_json_ == rsp_json == \ - gzip.decompress(rsp_gzip_json).decode(enc_16) - - assert rsp_jsonld_ == rsp_jsonld == \ - gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - - assert rsp_html_ == rsp_html == \ - gzip.decompress(rsp_gzip_html).decode(enc_16) - - -def test_gzip_csv(config, api_): - req_csv = mock_request({'f': 'csv'}) - rsp_csv_headers, _, rsp_csv = api_.get_collection_items(req_csv, 'obs') - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv = rsp_csv.decode('utf-8') - - req_csv = mock_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = api_.get_collection_items(req_csv, 'obs') # noqa - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') - assert rsp_csv == rsp_csv_ - - # Use utf-16 encoding - config['server']['encoding'] = 'utf-16' - api_ = API(config, openapi) - - req_csv = mock_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = api_.get_collection_items(req_csv, 'obs') # noqa - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') - assert rsp_csv == rsp_csv_ - - -def test_root(config, api_): - req = mock_request() - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/json' == \ - FORMAT_TYPES[F_JSON] - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - assert rsp_headers['Content-Language'] == 'en-US' - - assert isinstance(root, dict) - assert 'links' in root - assert root['links'][0]['rel'] == 'self' - assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] - assert root['links'][0]['href'].endswith('?f=json') - assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' - for link in root['links']) - assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' - for link in root['links']) - assert len(root['links']) == 11 - assert 'title' in root - assert root['title'] == 'pygeoapi default instance' - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_root_structured_data(config, api_): - req = mock_request({"f": "jsonld"}) - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/ld+json' == \ - FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - - assert isinstance(root, dict) - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - assert '@context' in root - assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' - expanded = jsonld.expand(root)[0] - assert '@type' in expanded - assert 'http://schema.org/DataCatalog' in expanded['@type'] - assert 'http://schema.org/description' in expanded - assert root['description'] == expanded['http://schema.org/description'][0][ - '@value'] - assert 'http://schema.org/keywords' in expanded - assert len(expanded['http://schema.org/keywords']) == 3 - assert '@value' in expanded['http://schema.org/keywords'][0].keys() - assert 'http://schema.org/provider' in expanded - assert expanded['http://schema.org/provider'][0]['@type'][ - 0] == 'http://schema.org/Organization' - assert expanded['http://schema.org/name'][0]['@value'] == root['name'] - - -def test_conformance(config, api_): - req = mock_request() - rsp_headers, code, response = api_.conformance(req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'conformsTo' in root - assert len(root['conformsTo']) == 36 - assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ - in root['conformsTo'] - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.conformance(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.conformance(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_tilematrixsets(config, api_): - req = mock_request() - rsp_headers, code, response = api_.tilematrixsets(req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'tileMatrixSets' in root - assert len(root['tileMatrixSets']) == 2 - assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad' \ - in root['tileMatrixSets'][0]['uri'] - assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ - in root['tileMatrixSets'][1]['uri'] - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.tilematrixsets(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.tilematrixsets(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_tilematrixset(config, api_): - req = mock_request() - - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - for e in enums: - enum = e.tileMatrixSet - rsp_headers, code, response = api_.tilematrixset(req, enum) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'id' in root - assert root['id'] == enum - assert 'tileMatrices' in root - assert len(root['tileMatrices']) == 30 - - rsp_headers, code, response = api_.tilematrixset(req, 'foo') - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.tilematrixset(req, enum) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections(config, api_): - req = mock_request({"f": "foo"}) - rsp_headers, code, response = api_.describe_collections(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({"f": "html"}) - rsp_headers, code, response = api_.describe_collections(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req) - collections = json.loads(response) - - assert len(collections) == 2 - assert len(collections['collections']) == 9 - assert len(collections['links']) == 3 - - rsp_headers, code, response = api_.describe_collections(req, 'foo') - collection = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'en-US' - assert collection['id'] == 'obs' - assert collection['title'] == 'Observations' - assert collection['description'] == 'My cool observations' - assert len(collection['links']) == 14 - assert collection['extent'] == { - 'spatial': { - 'bbox': [[-180, -90, 180, 90]], - 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - }, - 'temporal': { - 'interval': [ - ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] - ], - 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' - } - } - - # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults - assert collection['crs'] is not None - crs_set = [ - 'http://www.opengis.net/def/crs/EPSG/0/28992', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/EPSG/0/4326', - ] - for crs in crs_set: - assert crs in collection['crs'] - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert 'storageCrsCoordinateEpoch' not in collection - - # French language request - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'fr-CA' - assert collection['title'] == 'Observations' - assert collection['description'] == 'Mes belles observations' - - # Check HTML request in an unsupported language - req = mock_request({'f': 'html', 'lang': 'de'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req, - 'gdps-temperature') - collection = json.loads(response) - - assert collection['id'] == 'gdps-temperature' - assert len(collection['links']) == 10 - assert collection['extent']['spatial']['grid'][0]['cellsCount'] == 2400 - assert collection['extent']['spatial']['grid'][0]['resolution'] == 0.15000000000000002 # noqa - assert collection['extent']['spatial']['grid'][1]['cellsCount'] == 1201 - assert collection['extent']['spatial']['grid'][1]['resolution'] == 0.15 - - # hiearchical collections - rsp_headers, code, response = api_.describe_collections( - req, 'naturalearth/lakes') - collection = json.loads(response) - assert collection['id'] == 'naturalearth/lakes' - - # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured - assert collection['crs'] is not None - default_crs_list = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', - ] - contains_default = False - for crs in default_crs_list: - if crs in default_crs_list: - contains_default = True - assert contains_default - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert collection['storageCrsCoordinateEpoch'] == 2017.23 - - -def test_describe_collections_hidden_resources( - config_hidden_resources, api_hidden_resources): - req = mock_request({}) - rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa - assert code == HTTPStatus.OK - - assert len(config_hidden_resources['resources']) == 3 - - collections = json.loads(response) - assert len(collections['collections']) == 1 - - -def test_get_collection_schema(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_schema(req, - 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_schema(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_schema(req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 5 - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_schema( - req, 'gdps-temperature') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 1 - assert schema['properties']['1']['type'] == 'number' - - -def test_get_collection_queryables(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_queryables(req, - 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - queryables = json.loads(response) - - assert 'properties' in queryables - assert len(queryables['properties']) == 5 - - # test with provider filtered properties - api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] - - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') - queryables = json.loads(response) - - assert 'properties' in queryables - assert len(queryables['properties']) == 2 - assert 'geometry' in queryables['properties'] - assert queryables['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Geometry.json' # noqa - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert '@context' in collection - expanded = jsonld.expand(collection)[0] - # Metadata is about a schema:DataCollection that contains a schema:Dataset - assert not expanded['@id'].endswith('obs') - assert 'http://schema.org/dataset' in expanded - assert len(expanded['http://schema.org/dataset']) == 1 - dataset = expanded['http://schema.org/dataset'][0] - assert dataset['@type'][0] == 'http://schema.org/Dataset' - assert len(dataset['http://schema.org/distribution']) == 14 - assert all(dist['@type'][0] == 'http://schema.org/DataDownload' - for dist in dataset['http://schema.org/distribution']) - - assert 'http://schema.org/Organization' in expanded[ - 'http://schema.org/provider'][0]['@type'] - - assert 'http://schema.org/Place' in dataset[ - 'http://schema.org/spatial'][0]['@type'] - assert 'http://schema.org/GeoShape' in dataset[ - 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] - assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ - 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' - - assert 'http://schema.org/temporalCoverage' in dataset - assert dataset['http://schema.org/temporalCoverage'][0][ - '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_get_collection_items(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_items(req, 'foo') - features = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox': '1,2,3'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox': '1,2,3,4c'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox-crs': 'bad_value'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - # bbox-crs must be in configured values for Collection - req = mock_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - # bbox-crs must be in configured values for Collection (CSV will ignore) - req = mock_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - # bbox-crs can be a default even if not configured - req = mock_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - # bbox-crs can be a default even if not configured - req = mock_request({'bbox': '4,52,5,53'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'f': 'html', 'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'fr-CA' - - req = mock_request() - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - assert len(features['features']) == 5 - - req = mock_request({'resulttype': 'hits'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 0 - - # Invalid limit - req = mock_request({'limit': 0}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'stn_id': '35'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 2 - assert features['numberMatched'] == 2 - - req = mock_request({'stn_id': '35', 'value': '93.9'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 1 - assert features['numberMatched'] == 1 - - req = mock_request({'limit': 2}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 2 - assert features['features'][1]['properties']['stn_id'] == 35 - - links = features['links'] - assert len(links) == 4 - assert '/collections/obs/items?f=json' in links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld' in links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html' in links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs' in links[3]['href'] - assert links[3]['rel'] == 'collection' - - # Invalid offset - req = mock_request({'offset': -1}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'offset': 2}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 3 - assert features['features'][1]['properties']['stn_id'] == 2147 - - links = features['links'] - assert len(links) == 5 - assert '/collections/obs/items?f=json' in links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld' in links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html' in links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0' in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' - - req = mock_request({ - 'offset': 1, - 'limit': 1, - 'bbox': '-180,90,180,90' - }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 1 - - links = features['links'] - assert len(links) == 5 - assert '/collections/obs/items?f=json&limit=1&bbox=-180,90,180,90' in \ - links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld&limit=1&bbox=-180,90,180,90' in \ - links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html&limit=1&bbox=-180,90,180,90' in \ - links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0&limit=1&bbox=-180,90,180,90' \ - in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' - - req = mock_request({ - 'sortby': 'bad-property', - 'stn_id': '35' - }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'sortby': 'stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_request({'sortby': '+stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_request({'sortby': '-stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_request({'f': 'csv'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' - - req = mock_request({'datetime': '2003'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '1999'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'datetime': '2010-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'datetime': '2001-11-11/2003-12-18'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '../2003-12-18'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '2001-11-11/..'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '1999/2005-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '1999/2000-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - api_.config['resources']['obs']['extents'].pop('temporal') - - req = mock_request({'datetime': '2002/2014-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'scalerank': 1}) - rsp_headers, code, response = api_.get_collection_items( - req, 'naturalearth/lakes') - features = json.loads(response) - - assert len(features['features']) == 10 - assert features['numberMatched'] == 11 - assert features['numberReturned'] == 10 - - req = mock_request({'datetime': '2005-04-22'}) - rsp_headers, code, response = api_.get_collection_items( - req, 'naturalearth/lakes') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'skipGeometry': 'true'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert json.loads(response)['features'][0]['geometry'] is None - - req = mock_request({'properties': 'foo,bar'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - -def test_get_collection_items_crs(config, api_): - - # Invalid CRS query parameter - req = mock_request({'crs': '4326'}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.BAD_REQUEST - - # Unsupported CRS - req = mock_request({'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.BAD_REQUEST - - # Supported CRSs - default_crs = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - storage_crs = 'http://www.opengis.net/def/crs/EPSG/0/25833' - crs_4258 = 'http://www.opengis.net/def/crs/EPSG/0/4258' - supported_crs_list = [default_crs, storage_crs, crs_4258] - - for crs in supported_crs_list: - req = mock_request({'crs': crs}) - rsp_headers, code, response = api_.get_collection_items( - req, 'norway_pop', - ) - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{crs}>' - - # With CRS query parameter, using storageCRS - req = mock_request({'crs': storage_crs}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' - - features_25833 = json.loads(response) - - # With CRS query parameter resulting in coordinates transformation - req = mock_request({'crs': crs_4258}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' - - features_4258 = json.loads(response) - transform_func = pyproj.Transformer.from_crs( - pyproj.CRS.from_epsg(25833), - pyproj.CRS.from_epsg(4258), - always_xy=False, - ).transform - for feat_orig in features_25833['features']: - id_ = feat_orig['id'] - x, y, *_ = feat_orig['geometry']['coordinates'] - loc_transf = Point(transform_func(x, y)) - for feat_out in features_4258['features']: - if id_ == feat_out['id']: - loc_out = Point(feat_out['geometry']['coordinates'][:2]) - - assert loc_out.equals_exact(loc_transf, 1e-5) - break - - # Without CRS query parameter: assume Transform to default WGS84 lon,lat - req = mock_request({}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{default_crs}>' - - features_wgs84 = json.loads(response) - - # With CRS query parameter resulting in coordinates transformation - transform_func = pyproj.Transformer.from_crs( - pyproj.CRS.from_epsg(4258), - get_crs_from_uri(default_crs), - always_xy=False, - ).transform - for feat_orig in features_4258['features']: - id_ = feat_orig['id'] - x, y, *_ = feat_orig['geometry']['coordinates'] - loc_transf = Point(transform_func(x, y)) - for feat_out in features_wgs84['features']: - if id_ == feat_out['id']: - loc_out = Point(feat_out['geometry']['coordinates'][:2]) - - assert loc_out.equals_exact(loc_transf, 1e-5) - break - - -def test_manage_collection_item_read_only_options_req(config, api_): - """Test OPTIONS request on a read-only items endpoint""" - req = mock_request() - _, code, _ = api_.manage_collection_item(req, 'options', 'foo') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item(req, 'options', 'obs') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET' - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item( - req, 'options', 'obs', 'ressource_id') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET' - - -def test_manage_collection_item_editable_options_req(config): - """Test OPTIONS request on a editable items endpoint""" - config = copy.deepcopy(config) - config['resources']['obs']['providers'][0]['editable'] = True - api_ = API(config, openapi) - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item(req, 'options', 'obs') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET, POST' - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item( - req, 'options', 'obs', 'ressource_id') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' - - -def test_describe_collections_enclosures(config_enclosure, enclosure_api): - original_enclosures = { - lnk['title']: lnk - for lnk in config_enclosure['resources']['objects']['links'] - if lnk['rel'] == 'enclosure' - } - - req = mock_request() - _, _, response = enclosure_api.describe_collections(req, 'objects') - features = json.loads(response) - modified_enclosures = { - lnk['title']: lnk for lnk in features['links'] - if lnk['rel'] == 'enclosure' - } - - # If type and length is set, do not verify/update link - assert original_enclosures['download link 1'] == \ - modified_enclosures['download link 1'] - # If length is missing, modify link type and length - assert original_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 2']['type'] - assert modified_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 3']['type'] - assert 'length' not in original_enclosures['download link 2'] - assert modified_enclosures['download link 2']['length'] > 0 - assert modified_enclosures['download link 2']['length'] == \ - modified_enclosures['download link 3']['length'] - assert original_enclosures['download link 3']['type'] != \ - modified_enclosures['download link 3']['type'] - - -def test_get_collection_items_json_ld(config, api_): - req = mock_request({ - 'f': 'jsonld', - 'limit': 2 - }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - collection = json.loads(response) - - assert '@context' in collection - assert all((f in collection['@context'][0] for - f in ('schema', 'type', 'features', 'FeatureCollection'))) - assert len(collection['@context']) > 1 - assert collection['@context'][1]['schema'] == 'https://schema.org/' - expanded = jsonld.expand(collection)[0] - featuresUri = 'https://schema.org/itemListElement' - assert len(expanded[featuresUri]) == 2 - - -def test_get_collection_item(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_item( - req, 'gdps-temperature', '371') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request() - rsp_headers, code, response = api_.get_collection_item(req, 'foo', '371') - - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = api_.get_collection_item( - req, 'obs', 'notfound') - - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - req = mock_request() - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - feature = json.loads(response) - - assert feature['properties']['stn_id'] == 35 - assert 'prev' not in feature['links'] - assert 'next' not in feature['links'] - - -def test_get_collection_item_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, _, response = api_.get_collection_item(req, 'objects', '3') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - feature = json.loads(response) - assert '@context' in feature - assert all((f in feature['@context'][0] for - f in ('schema', 'type', 'gsp'))) - assert len(feature['@context']) == 1 - assert 'schema' in feature['@context'][0] - assert feature['@context'][0]['schema'] == 'https://schema.org/' - assert feature['id'] == 3 - expanded = jsonld.expand(feature)[0] - - assert expanded['@id'].startswith('http://') - assert expanded['@id'].endswith('/collections/objects/items/3') - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'POINT (-85 33)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/latitude'][0][ - '@value'] == 33 - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/longitude'][0][ - '@value'] == -85 - - _, _, response = api_.get_collection_item(req, 'objects', '2') - feature = json.loads(response) - assert feature['geometry']['type'] == 'MultiPoint' - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTIPOINT (10 40, 40 30, 20 20, 30 10)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" - - _, _, response = api_.get_collection_item(req, 'objects', '1') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'LINESTRING (30 10, 10 30, 40 40)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/line'][0][ - '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' - - _, _, response = api_.get_collection_item(req, 'objects', '4') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTILINESTRING ((10 10, 20 20, 10 40), ' \ - '(40 40, 30 30, 40 20, 30 10))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/line'][0][ - '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ - '30.0,30.0 40.0,20.0 30.0,10.0' - - _, _, response = api_.get_collection_item(req, 'objects', '5') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' - - _, _, response = api_.get_collection_item(req, 'objects', '7') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '\ - '((15 5, 40 10, 10 20, 5 10, 15 5)))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ - '45.0,40.0 40.0,10.0 15.0,5.0' - - req = mock_request({'f': 'jsonld', 'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'fr-CA' - - -def test_get_collection_coverage(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_coverage( - req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'properties': '12'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'subset': 'bad_axis(10:20)'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'blah'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - assert rsp_headers['Content-Type'] == 'text/html' - - req = mock_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - # NOTE: This test used to assert the code to be 200 OK, - # but it requested HTML, which is not available, - # so it should be 400 Bad Request - assert code == HTTPStatus.BAD_REQUEST - assert rsp_headers['Content-Type'] == 'text/html' - - req = mock_request({'subset': 'Lat(5:10),Long(5:10)'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.OK - content = json.loads(response) - - assert content['domain']['axes']['x']['num'] == 35 - assert content['domain']['axes']['y']['num'] == 35 - assert 'TMP' in content['parameters'] - assert 'TMP' in content['ranges'] - assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] - - req = mock_request({'bbox': '-79,45,-75,49'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.OK - content = json.loads(response) - - assert content['domain']['axes']['x']['start'] == -79.0 - assert content['domain']['axes']['x']['stop'] == -75.0 - assert content['domain']['axes']['y']['start'] == 49.0 - assert content['domain']['axes']['y']['stop'] == 45.0 - - req = mock_request({ - 'subset': 'Lat(5:10),Long(5:10)', - 'f': 'GRIB' - }) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.OK - assert isinstance(response, bytes) - - req = mock_request(HTTP_ACCEPT='application/x-netcdf') - rsp_headers, code, response = api_.get_collection_coverage( - req, 'cmip5') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'application/x-netcdf' - - # req = mock_request({ - # 'subset': 'time("2006-07-01T06:00:00":"2007-07-01T06:00:00")' - # }) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') - # - # assert code == HTTPStatus.OK - # assert isinstance(json.loads(response), dict) - - # req = mock_request({'subset': 'lat(1:2'}) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') - # - # assert code == HTTPStatus.BAD_REQUEST - # - # req = mock_request({'subset': 'lat(1:2)'}) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') - # - # assert code == HTTPStatus.NO_CONTENT - - -def test_get_collection_map(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_map(req, 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request() - rsp_headers, code, response = api_.get_collection_map( - req, 'mapserver_world_map') - assert code == HTTPStatus.OK - assert isinstance(response, bytes) - assert response[1:4] == b'PNG' - - -def test_get_collection_tiles(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_tiles(req, 'obs') - assert code == HTTPStatus.BAD_REQUEST - - rsp_headers, code, response = api_.get_collection_tiles( - req, 'naturalearth/lakes') - assert code == HTTPStatus.OK - - # Language settings should be ignored (return system default) - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_tiles( - req, 'naturalearth/lakes') - assert rsp_headers['Content-Language'] == 'en-US' - content = json.loads(response) - assert len(content['links']) > 0 - assert len(content['tilesets']) > 0 - - -def test_describe_processes(config, api_): - req = mock_request({'limit': 1}) - # Test for description of single processes - rsp_headers, code, response = api_.describe_processes(req) - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 1 - assert len(data['links']) == 3 - - req = mock_request() - - # Test for undefined process - rsp_headers, code, response = api_.describe_processes(req, 'foo') - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert data['code'] == 'NoSuchProcess' - - # Test for description of all processes - rsp_headers, code, response = api_.describe_processes(req) - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 2 - assert len(data['links']) == 3 - - # Test for particular, defined process - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - process = json.loads(response) - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert process['id'] == 'hello-world' - assert process['version'] == '0.2.0' - assert process['title'] == 'Hello World' - assert len(process['keywords']) == 3 - assert len(process['links']) == 6 - assert len(process['inputs']) == 2 - assert len(process['outputs']) == 1 - assert len(process['outputTransmission']) == 1 - assert len(process['jobControlOptions']) == 2 - assert 'sync-execute' in process['jobControlOptions'] - assert 'async-execute' in process['jobControlOptions'] - - # Check HTML response when requested in headers - req = mock_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested in headers - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'en-US' - - # Check HTML response when requested with query parameter - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested with query parameter - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested with French language parameter - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'fr-CA' - process = json.loads(response) - assert process['title'] == 'Bonjour le Monde' - - # Check JSON response when language requested in headers - req = mock_request(HTTP_ACCEPT_LANGUAGE='fr') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'fr-CA' - - # Test for undefined process - req = mock_request() - rsp_headers, code, response = api_.describe_processes(req, 'goodbye-world') - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert data['code'] == 'NoSuchProcess' - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Test describe doesn't crash if example is missing - req = mock_request() - processor = api_.manager.get_processor("hello-world") - example = processor.metadata.pop("example") - rsp_headers, code, response = api_.describe_processes(req) - processor.metadata['example'] = example - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 2 - - -def test_execute_process(config, api_): - req_body_0 = { - 'inputs': { - 'name': 'Test' - } - } - req_body_1 = { - 'inputs': { - 'name': 'Test' - }, - 'response': 'document' - } - req_body_2 = { - 'inputs': { - 'name': 'Tést' - } - } - req_body_3 = { - 'inputs': { - 'name': 'Tést', - 'message': 'This is a test.' - } - } - req_body_4 = { - 'inputs': { - 'foo': 'Tést' - } - } - req_body_5 = { - 'inputs': {} - } - req_body_6 = { - 'inputs': { - 'name': None - } - } - req_body_7 = { - 'inputs': { - 'name': 'Test' - }, - 'subscriber': { - 'successUri': 'https://example.com/success', - 'inProgressUri': 'https://example.com/inProgress', - 'failedUri': 'https://example.com/failed', - } - } - - cleanup_jobs = set() - - # Test posting empty payload to existing process - req = mock_request(data='') - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - assert rsp_headers['Content-Language'] == 'en-US' - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' not in rsp_headers - assert data['code'] == 'MissingParameterValue' - - req = mock_request(data=req_body_0) - rsp_headers, code, response = api_.execute_process(req, 'foo') - - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert 'Location' not in rsp_headers - assert data['code'] == 'NoSuchProcess' - - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - - assert len(data.keys()) == 2 - assert data['id'] == 'echo' - assert data['value'] == 'Hello Test!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_1) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - - assert len(data.keys()) == 1 - assert data['outputs'][0]['id'] == 'echo' - assert data['outputs'][0]['value'] == 'Hello Test!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_2) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Tést!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_3) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Tést! This is a test.' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_4) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_5) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - assert data['description'] == 'Error updating job' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_6) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - assert data['description'] == 'Error updating job' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_0) - rsp_headers, code, response = api_.execute_process(req, 'goodbye-world') - - response = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert 'Location' not in rsp_headers - assert response['code'] == 'NoSuchProcess' - - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - response = json.loads(response) - assert code == HTTPStatus.OK - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_1, HTTP_Prefer='respond-async') - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - assert 'Location' in rsp_headers - response = json.loads(response) - assert isinstance(response, dict) - assert code == HTTPStatus.CREATED - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_7) - with mock.patch( - 'pygeoapi.process.manager.base.requests.post' - ) as post_mocker: - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - assert code == HTTPStatus.OK - post_mocker.assert_any_call( - req_body_7['subscriber']['inProgressUri'], json={} - ) - post_mocker.assert_any_call( - req_body_7['subscriber']['successUri'], - json={'id': 'echo', 'value': 'Hello Test!'} - ) - assert post_mocker.call_count == 2 - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - # Cleanup - time.sleep(2) # Allow time for any outstanding async jobs - for _, job_id in cleanup_jobs: - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.OK - - -def _execute_a_job(api_): - req_body_sync = { - 'inputs': { - 'name': 'Sync Test' - } - } - - req = mock_request(data=req_body_sync) - rsp_headers, code, response = api_.execute_process( - req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Sync Test!' - - job_id = rsp_headers['Location'].split('/')[-1] - return job_id - - -def test_delete_job(api_): - rsp_headers, code, response = api_.delete_job( - mock_request(), 'does-not-exist') - - assert code == HTTPStatus.NOT_FOUND - req_body_async = { - 'inputs': { - 'name': 'Async Test Deletion' - } - } - job_id = _execute_a_job(api_) - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - - assert code == HTTPStatus.OK - - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.NOT_FOUND - - req = mock_request(data=req_body_async, HTTP_Prefer='respond-async') - rsp_headers, code, response = api_.execute_process( - req, 'hello-world') - - assert code == HTTPStatus.CREATED - assert 'Location' in rsp_headers - - time.sleep(2) # Allow time for async execution to complete - job_id = rsp_headers['Location'].split('/')[-1] - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.OK - - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.NOT_FOUND - - -def test_get_job_result(api_): - rsp_headers, code, response = api_.get_job_result(mock_request(), - 'not-exist') - assert code == HTTPStatus.NOT_FOUND - - job_id = _execute_a_job(api_) - rsp_headers, code, response = api_.get_job_result(mock_request(), job_id) - # default response is html - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'text/html' - assert 'Hello Sync Test!' in response - - rsp_headers, code, response = api_.get_job_result( - mock_request({'f': 'json'}), job_id, - ) - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'application/json' - assert json.loads(response)['value'] == "Hello Sync Test!" - - -def test_get_collection_edr_query(config, api_): - # edr resource - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') - collection = json.loads(response) - parameter_names = list(collection['parameter_names'].keys()) - parameter_names.sort() - assert len(parameter_names) == 4 - assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] - - # no coords parameter - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # bad query type - req = mock_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'corridor') - assert code == HTTPStatus.BAD_REQUEST - - # bad coords parameter - req = mock_request({'coords': 'gah'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # bad parameter_names parameter - req = mock_request({ - 'coords': 'POINT(11 11)', 'parameter_names': 'bad' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # all parameters - req = mock_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - - axes = list(data['domain']['axes'].keys()) - axes.sort() - assert len(axes) == 3 - assert axes == ['TIME', 'x', 'y'] - - assert data['domain']['axes']['x']['start'] == 11.0 - assert data['domain']['axes']['x']['stop'] == 11.0 - assert data['domain']['axes']['y']['start'] == 11.0 - assert data['domain']['axes']['y']['stop'] == 11.0 - - parameters = list(data['parameters'].keys()) - parameters.sort() - assert len(parameters) == 4 - assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] - - # single parameter - req = mock_request({ - 'coords': 'POINT(11 11)', 'parameter_names': 'SST' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - - assert len(data['parameters'].keys()) == 1 - assert list(data['parameters'].keys())[0] == 'SST' - - # Zulu time zone - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - # bounded date range - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-01-17/2000-06-16' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-02-15T16:29:05.999999999' - assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' - assert time_dict['num'] == 5 - - # unbounded date range - start - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '../2000-06-16' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-01-16T06:00:00.000000000' - assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' - assert time_dict['num'] == 6 - - # unbounded date range - end - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-06-16/..' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-06-16T10:25:30.000000000' - assert time_dict['stop'] == '2000-12-16T01:20:05.999999996' - assert time_dict['num'] == 7 - - # some data - req = mock_request({ - 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - # no data - req = mock_request({ - 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.NO_CONTENT - - # position no coords - req = mock_request({ - 'datetime': '2000-01-17' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # cube bbox parameter 4 dimensional - req = mock_request({ - 'bbox': '0,0,10,10' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.OK - - # cube bad bbox parameter - req = mock_request({ - 'bbox': '0,0,10' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.BAD_REQUEST - - # cube no bbox parameter - req = mock_request({}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.BAD_REQUEST - - # cube decreasing latitude coords and S3 - req = mock_request({ - 'bbox': '-100,40,-99,45', - 'parameter_names': 'tmn', - 'datetime': '1994-01-01/1994-12-31', - }) - - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'usgs-prism', None, 'cube') - assert code == HTTPStatus.OK - - -def test_validate_bbox(): - assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] - assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] - assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] - assert (validate_bbox('-142.1,42.12,-52.22,84.4') == - [-142.1, 42.12, -52.22, 84.4]) - assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == - [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) - - assert (validate_bbox('177.0,65.0,-177.0,70.0') == - [177.0, 65.0, -177.0, 70.0]) - - with pytest.raises(ValueError): - validate_bbox('1,2,4') - - with pytest.raises(ValueError): - validate_bbox('1,2,4,5,6') - - with pytest.raises(ValueError): - validate_bbox('3,4,1,2') - - with pytest.raises(ValueError): - validate_bbox('1,2,6,4,5,3') - - -def test_validate_datetime(): - config = yaml_load(''' - temporal: - begin: 2000-10-30T18:24:39Z - end: 2007-10-30T08:57:29Z - ''') - - # test time instant - assert validate_datetime(config, '2004') == '2004' - assert validate_datetime(config, '2004-10') == '2004-10' - assert validate_datetime(config, '2001-10-30') == '2001-10-30' - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009-10-30') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-09-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-10-30T17:24:39Z') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-10-30T08:58:29Z') - - # test time envelope - assert validate_datetime(config, '2004/2005') == '2004/2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - assert validate_datetime(config, '2004/..') == '2004/..' - assert validate_datetime(config, '../2005') == '../2005' - assert validate_datetime(config, '2004/') == '2004/..' - assert validate_datetime(config, '/2005') == '../2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-11-01/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../2000-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../1999') - - -@pytest.mark.parametrize("value, expected", [ - ('time(2000-11-11)', {'time': ['2000-11-11']}), - ('time("2000-11-11")', {'time': ['2000-11-11']}), - ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), - ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa - ('lat(40)', {'lat': [40]}), - ('lat(0:40)', {'lat': [0, 40]}), - ('foo("bar")', {'foo': ['bar']}), - ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) -]) -def test_validate_subset(value, expected): - assert validate_subset(value) == expected - - with pytest.raises(ValueError): - validate_subset('foo("bar)') - - -def test_get_exception(config, api_): - d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') - assert d[0] == {} - assert d[1] == 500 - content = json.loads(d[2]) - assert content['code'] == 'NoApplicableCode' - assert content['description'] == 'oops' - - d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') - assert d[0] == {'Content-Type': 'text/html'} diff --git a/tests/test_api_ogr_provider.py b/tests/test_api_ogr_provider.py index 65d358783..c29fc629e 100644 --- a/tests/test_api_ogr_provider.py +++ b/tests/test_api_ogr_provider.py @@ -4,7 +4,7 @@ # Authors: Tom Kralidis # # Copyright (c) 2019 Just van den Broecke -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -35,8 +35,10 @@ import pytest from pygeoapi.api import API +from pygeoapi.api.itemtypes import get_collection_item, get_collection_items from pygeoapi.util import yaml_load, geojson_to_geom -from .util import get_test_file_path, mock_request + +from .util import get_test_file_path, mock_api_request LOGGER = logging.getLogger(__name__) @@ -70,16 +72,16 @@ def test_get_collection_items_bbox_crs(config, api_): COLLECTIONS = ['dutch_addresses_4326', 'dutch_addresses_28992'] for coll in COLLECTIONS: # bbox-crs full extent - req = mock_request({'bbox': '5.670670, 52.042700, 5.829110, 52.123700', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '5.670670, 52.042700, 5.829110, 52.123700', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 # bbox-crs partial extent, 1 feature, request with multiple CRSs for crs in CRS_BBOX_DICT: - req = mock_request({'bbox': CRS_BBOX_DICT[crs], 'bbox-crs': crs}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': CRS_BBOX_DICT[crs], 'bbox-crs': crs}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 1 @@ -88,29 +90,29 @@ def test_get_collection_items_bbox_crs(config, api_): assert properties['huisnummer'] == '2' # bbox-crs outside extent - req = mock_request({'bbox': '5, 51.9, 5.1, 52.0', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '5, 51.9, 5.1, 52.0', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs outside extent - req = mock_request({'bbox': '130000, 440000, 140000, 450000', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/28992'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '130000, 440000, 140000, 450000', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/28992'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs outside extent - axis reversed CRS - req = mock_request({'bbox': '51.9, 5, 52.0, 5.1', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '51.9, 5, 52.0, 5.1', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs full extent - axis reversed CRS - req = mock_request({'bbox': '52.042700, 5.670670, 52.123700, 5.829110', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '52.042700, 5.670670, 52.123700, 5.829110', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -135,8 +137,8 @@ def test_get_collection_items_crs(config, api_): COLLECTIONS = ['dutch_addresses_4326', 'dutch_addresses_28992'] for coll in COLLECTIONS: # crs full extent to get target feature - req = mock_request({}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -145,13 +147,13 @@ def test_get_collection_items_crs(config, api_): # request with multiple CRSs for crs in CRS_DICT: # Do for query (/items) - req = mock_request({'crs': crs}) # noqa + req = mock_api_request({'crs': crs}) # noqa if crs == 'none': # Test for default bbox CRS - req = mock_request({}) # noqa + req = mock_api_request({}) # noqa crs = DEFAULT_CRS - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -173,8 +175,8 @@ def test_get_collection_items_crs(config, api_): assert test_geom.equals_exact(crs_geom, 1), f'coords not equal for CRS: {crs} {crs_geom} in COLL: {coll} {test_geom}' # noqa # Same for single Feature 'get' - req = mock_request({'crs': crs}) # noqa - rsp_headers, code, response = api_.get_collection_item(req, coll, feature_id) # noqa + req = mock_api_request({'crs': crs}) # noqa + rsp_headers, code, response = get_collection_item(api_, req, coll, feature_id) # noqa test_feature = json.loads(response) assert test_feature['id'] == feature_id @@ -195,13 +197,13 @@ def test_get_collection_items_crs(config, api_): # Test combining BBOX and BBOX-CRS for bbox_crs in CRS_BBOX_DICT: # Do for query (/items) - req = mock_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs], 'bbox-crs': bbox_crs}) # noqa + req = mock_api_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs], 'bbox-crs': bbox_crs}) # noqa if bbox_crs == 'none': # Test for default bbox CRS - req = mock_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs]}) # noqa + req = mock_api_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs]}) # noqa bbox_crs = DEFAULT_CRS - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 1 diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index 650a6096c..b027d0849 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -7,7 +7,7 @@ # Francesco Bartoli # # Copyright (c) 2019 Just van den Broecke -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Francesco Bartoli # @@ -47,7 +47,9 @@ from pygeofilter.parsers.ecql import parse from pygeoapi.api import API - +from pygeoapi.api.itemtypes import ( + get_collection_items, get_collection_item, post_collection_items +) from pygeoapi.provider.base import ( ProviderConnectionError, ProviderItemNotFoundError, @@ -59,7 +61,7 @@ from pygeoapi.util import (yaml_load, geojson_to_geom, get_transform_from_crs, get_crs_from_uri) -from .util import get_test_file_path, mock_request +from .util import get_test_file_path, mock_api_request PASSWORD = os.environ.get('POSTGRESQL_PASSWORD', 'postgres') DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' @@ -429,12 +431,12 @@ def test_get_collection_items_postgresql_cql(pg_api_): expected_ids = [80835474, 80835483] # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-text', 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -443,11 +445,11 @@ def test_get_collection_items_postgresql_cql(pg_api_): assert ids == expected_ids # Act, no filter-lang - req = mock_request({ + req = mock_api_request({ 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -467,12 +469,12 @@ def test_get_collection_items_postgresql_cql_invalid_filter_language(pg_api_): cql_query = 'osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL' # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json', # Only cql-text is valid for GET 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -494,11 +496,11 @@ def test_get_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): Test for bad cql """ # Act - req = mock_request({ + req = mock_api_request({ 'filter': bad_cql }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -524,11 +526,11 @@ def test_post_collection_items_postgresql_cql(pg_api_): expected_ids = [80835474, 80835483] # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json' }, data=cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -550,11 +552,11 @@ def test_post_collection_items_postgresql_cql_invalid_filter_language(pg_api_): headers = {'CONTENT_TYPE': 'application/query-cql-json'} # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-text' # Only cql-json is valid for POST }, data=cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -580,11 +582,11 @@ def test_post_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): headers = {'CONTENT_TYPE': 'application/query-cql-json'} # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json' }, data=bad_cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -601,10 +603,9 @@ def test_get_collection_items_postgresql_crs(pg_api_): crs_32735 = 'http://www.opengis.net/def/crs/EPSG/0/32735' # Without CRS query parameter -> no coordinates transformation - req = mock_request({'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + req = mock_api_request({'bbox': '29.0,-2.85,29.05,-2.8'}) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK @@ -613,10 +614,10 @@ def test_get_collection_items_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') - req = mock_request({'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + req = mock_api_request( + {'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -624,10 +625,9 @@ def test_get_collection_items_postgresql_crs(pg_api_): features_storage_crs = json.loads(response) # With CRS query parameter resulting in coordinates transformation - req = mock_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + req = mock_api_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_32735}>' @@ -688,10 +688,9 @@ def test_get_collection_item_postgresql_crs(pg_api_): ] for fid in fid_list: # Without CRS query parameter -> no coordinates transformation - req = mock_request({'f': 'json'}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{DEFAULT_CRS}>' @@ -701,10 +700,9 @@ def test_get_collection_item_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') - req = mock_request({'f': 'json', 'crs': storage_crs}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + req = mock_api_request({'f': 'json', 'crs': storage_crs}) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -716,10 +714,9 @@ def test_get_collection_item_postgresql_crs(pg_api_): assert feat_orig['geometry'] == feat_storage_crs['geometry'] # With CRS query parameter resulting in coordinates transformation - req = mock_request({'f': 'json', 'crs': crs_32735}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + req = mock_api_request({'f': 'json', 'crs': crs_32735}) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_32735}>' @@ -741,9 +738,9 @@ def test_get_collection_items_postgresql_automap_naming_conflicts(pg_api_): Test that PostgreSQLProvider can handle naming conflicts when automapping classes and relationships from database schema. """ - req = mock_request() - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'dummy_naming_conflicts') + req = mock_api_request() + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'dummy_naming_conflicts') assert code == HTTPStatus.OK features = json.loads(response).get('features') diff --git a/tests/util.py b/tests/util.py index 5de974531..64b0c37ae 100644 --- a/tests/util.py +++ b/tests/util.py @@ -40,6 +40,8 @@ from werkzeug.wrappers import Request from werkzeug.datastructures import ImmutableMultiDict +from pygeoapi.api import APIRequest + LOGGER = logging.getLogger(__name__) @@ -77,6 +79,25 @@ def mock_request(params: dict = None, data=None, **headers) -> Request: return request +def mock_api_request(params: dict | None = None, data=None, **headers + ) -> APIRequest: + """ + Mocks an APIRequest + + :param params: Optional query parameter dict for the request. + Will be set to {} if omitted. + :param data: Optional data/body to send with the request. + Can be text/bytes or a JSON dictionary. + :param headers: Optional request HTTP headers to set. + :returns: APIRequest instance + """ + return APIRequest.from_flask( + mock_request(params=params, data=data, **headers), + # NOTE: could also read supported_locales from test config + supported_locales=['en-US', 'fr-CA'], + ) + + @contextmanager def mock_flask(config_file: str = 'pygeoapi-test-config.yml', openapi_file: str = 'pygeoapi-test-openapi.yml',