Feature/8432 add search query explain

server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminApi.java

@@ -0,0 +1,120 @@
+package au.org.aodn.ogcapi.server.common;
+
+
+import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.enums.ParameterIn;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.Size;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.NotImplementedException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.*;
+
+@Slf4j
+@RestController("CommonRestAdminApi")
+@RequestMapping(value="/api/v1/ogc/admin")
+public class RestAdminApi {
+    @Autowired
+    protected RestAdminService restAdminService;
+
+    /**
+     * Explain the detail relevance score of a search query
+     * Internal debugging/troubleshooting usage only
+     * The parameters should be the same with /collections endpoint in RestApi
+     * */
+    @GetMapping(path="/explain")
+    public ResponseEntity<JsonNode> getExplainByParameters(
+            @Parameter(in = ParameterIn.QUERY, description = "Keyword search terms")
+            @Valid @RequestParam(value = "q", required = false) List<String> q,
+            @Parameter(in = ParameterIn.QUERY, description = "Filter expression")
+            @RequestParam(value = "filter", required = false) String filter,
+            @Parameter(in = ParameterIn.QUERY, description = "Properties used by the production search request")
+            @Valid @RequestParam(value = "properties", required = false) List<String> properties,
+            @Size(min = 1)
+            @Parameter(in = ParameterIn.QUERY, description = "Sort by a valid CQL property")
+            @Valid @RequestParam(value = "sortby", required = false, defaultValue = "-score,-rank") String sortBy,
+            @Parameter(in = ParameterIn.QUERY, description = "Coordinate system")
+            @RequestParam(value = "crs", required = false, defaultValue = "https://epsg.io/4326") String crs,
+            @Parameter(in = ParameterIn.QUERY, description = "Filter language")
+            @RequestParam(value = "filter-lang", required = false, defaultValue = "cql-text") String filterLang
+    ) throws Exception {
+        if (!restAdminService.isElasticsearchExplainEnabled()) {
+            //return 404 NotFound error if elasticsearch-explain-enabled is set as false
+            return ResponseEntity.notFound().build();
+        }
+
+        CQLCrsType convertedCrs = validateFilterAndCrs(filterLang, crs);
+        return ResponseEntity.ok(restAdminService.explainByParameters(
+                q,
+                filter,
+                properties,
+                sortBy,
+                convertedCrs));
+    }
+    /**
+     * Explain an uuid matches a search query or not
+     * Internal debugging/troubleshooting usage only
+     * The parameters should be the same with /collections endpoint in RestApi
+     * */
+    @GetMapping(path="/explain/{uuid}")
+    public ResponseEntity<JsonNode> getExplainByParametersUuid(
+            @Parameter(in = ParameterIn.PATH, description = "Elasticsearch document id")
+            @PathVariable String uuid,
+            @Parameter(in = ParameterIn.QUERY, description = "Keyword search terms")
+            @Valid @RequestParam(value = "q", required = false) List<String> q,
+            @Parameter(in = ParameterIn.QUERY, description = "Filter expression")
+            @RequestParam(value = "filter", required = false) String filter,
+            @Parameter(in = ParameterIn.QUERY, description = "Properties used by the production search request")
+            @Valid @RequestParam(value = "properties", required = false) List<String> properties,
+            @Size(min = 1)
+            @Parameter(in = ParameterIn.QUERY, description = "Sort by a valid CQL property")
+            @Valid @RequestParam(value = "sortby", required = false, defaultValue = "-score,-rank") String sortBy,
+            @Parameter(in = ParameterIn.QUERY, description = "Coordinate system")
+            @RequestParam(value = "crs", required = false, defaultValue = "https://epsg.io/4326") String crs,
+            @Parameter(in = ParameterIn.QUERY, description = "Filter language")
+            @RequestParam(value = "filter-lang", required = false, defaultValue = "cql-text") String filterLang
+    ) throws Exception {
+        if (!restAdminService.isElasticsearchExplainEnabled()) {
+            //return 404 NotFound error if elasticsearch-explain-enabled is set as false
+            return ResponseEntity.notFound().build();
+        }
+
+        CQLCrsType convertedCrs = validateFilterAndCrs(filterLang, crs);
+        return ResponseEntity.ok(restAdminService.explainByUuid(
+                uuid,
+                q,
+                filter,
+                properties,
+                sortBy,
+                convertedCrs));
+    }
+
+    protected CQLCrsType validateFilterAndCrs(String filterLang, String crs) {
+        boolean isCqlFilter = "cql-text".equals(filterLang);
+        CQLCrsType convertedCrs = CQLCrsType.convertFromUrl(crs);
+
+        if (!isCqlFilter || convertedCrs != CQLCrsType.EPSG4326) {
+            List<String> reasons = new ArrayList<>();
+
+            if (!isCqlFilter) {
+                reasons.add("Unknown filter language, support cql-text only");
+            }
+
+            if (convertedCrs != CQLCrsType.EPSG4326) {
+                reasons.add("Unknown crs, support EPSG4326 only");
+            }
+
+            throw new NotImplementedException(String.join(",", reasons));
+        }
+        return convertedCrs;
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminService.java

@@ -0,0 +1,50 @@
+package au.org.aodn.ogcapi.server.common;
+
+import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
+import au.org.aodn.ogcapi.server.core.service.Search;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+@Slf4j
+public class RestAdminService {
+    @Value("${ogcapi.debug.elasticsearch-explain-enabled:false}")
+    protected boolean elasticsearchExplainEnabled;
+
+    @Autowired
+    protected Search searchService;
+
+    /**
+     * Value defined in application-*.yml, for dev and test, this value is set as true,
+     * for other environments, use default false value.
+     */
+    public boolean isElasticsearchExplainEnabled() {
+        return elasticsearchExplainEnabled;
+    }
+
+    public JsonNode explainByParameters(
+            List<String> q,
+            String filter,
+            List<String> properties,
+            String sortBy,
+            CQLCrsType crs) throws Exception {
+        log.info("Explaining search query");
+        return searchService.explainByParameters(q, filter, properties, sortBy, crs);
+    }
+
+    public JsonNode explainByUuid(
+            String uuid,
+            List<String> q,
+            String filter,
+            List<String> properties,
+            String sortBy,
+            CQLCrsType crs) throws Exception {
+        log.info("Explaining search query for uuid {}", uuid);
+        return searchService.explainByUuid(uuid, q, filter, properties, sortBy, crs);
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -10,13 +10,13 @@
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.*;
 import co.elastic.clients.elasticsearch._types.query_dsl.*;
-import co.elastic.clients.json.JsonData;
 import co.elastic.clients.elasticsearch.core.SearchMvtRequest;
 import co.elastic.clients.elasticsearch.core.SearchRequest;
 import co.elastic.clients.elasticsearch.core.SearchResponse;
 import co.elastic.clients.elasticsearch.core.search.Hit;
 import co.elastic.clients.elasticsearch.core.search_mvt.GridType;
 import co.elastic.clients.transport.endpoints.BinaryResponse;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.geotools.filter.text.commons.CompilerUtil;
@@ -33,6 +33,7 @@
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.function.Supplier;
 
 import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.ELASTIC_SEARCH_UUID_ONLY;
 
@@ -198,8 +199,8 @@ protected ElasticSearchBase.SearchResult<StacCollectionModel> searchCollectionsB
 
         List<Query> queries = new ArrayList<>();
         queries.add(MatchQuery.of(m -> m
-                            .field(StacType.searchField)
-                            .query(StacType.Collection.value))._toQuery());
+                .field(StacType.searchField)
+                .query(StacType.Collection.value))._toQuery());
 
         if(isWithGeometry) {
             queries.add(ExistsQuery.of(m -> m
@@ -256,6 +257,124 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchAllCollections(
         return searchCollectionsByIds(null, Boolean.FALSE, sortBy);
     }
 
+
+    /**
+     * Build SearchRequest for searchByParameters and explainByParameters
+     * */
+    protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
+            List<String> keywords,
+            String cql,
+            List<String> properties,
+            String sortBy,
+            CQLCrsType coor) throws CQLException {
+
+        if ((keywords == null || keywords.isEmpty()) && cql == null) {
+            List<Query> queries = new ArrayList<>();
+            queries.add(MatchQuery.of(m -> m
+                    .field(StacType.searchField)
+                    .query(StacType.Collection.value))._toQuery());
+
+            return buildCollectionSearchRequestSupplier(
+                    queries,
+                    null,
+                    null,
+                    null,
+                    null,
+                    createSortOptions(sortBy, CQLFields.class),
+                    null,
+                    null);
+        }
+
+        List<Query> should = null;
+        if (keywords != null && !keywords.isEmpty()) {
+            should = new ArrayList<>();
+
+            for (String t : keywords) {
+                boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2;
+                String term = isExact ? t.substring(1, t.length() - 1) : t;
+
+                if (isExact) {
+                    should.add(CQLFields.title.getPropertyEqualToQuery(term));
+                    should.add(CQLFields.description.getPropertyEqualToQuery(term));
+                }
+                else {
+                    should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term));
+                    should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term));
+                }
+
+                should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term));
+                should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
+                should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
+                should.add(CQLFields.id.getPropertyEqualToQuery(term));
+                should.add(BoolQuery.of(b -> b
+                        .should(CQLFields.links_title_contains.getPropertyEqualToQuery(term))
+                        .boost(0.5f))._toQuery());
+                should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
+            }
+        }
+
+        List<Query> filters = new ArrayList<>();
+        CQLToElasticFilterFactory<CQLFields> factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class);
+
+        if (cql != null) {
+            Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory);
+            if (filter instanceof QueryHandler handler) {
+                if (handler.getErrors() == null || handler.getErrors().isEmpty()) {
+                    if (handler.getQuery() != null) {
+                        filters = List.of(handler.getQuery());
+                    }
+                }
+                else {
+                    throw new IllegalArgumentException("ECQL Parse Error");
+                }
+            }
+        }
+
+        Map<CQLElasticSetting, String> setting = factory.getQuerySetting();
+
+        Long maxSize = setting.get(CQLElasticSetting.page_size) == null
+                || setting.get(CQLElasticSetting.page_size).isBlank()
+                ? null
+                : Long.parseLong(setting.get(CQLElasticSetting.page_size));
+
+        Double score = setting.get(CQLElasticSetting.score) == null
+                || setting.get(CQLElasticSetting.score).isBlank()
+                ? null
+                : Double.parseDouble(setting.get(CQLElasticSetting.score));
+
+        List<FieldValue> searchAfter = null;
+        if (setting.get(CQLElasticSetting.search_after) != null
+                && !setting.get(CQLElasticSetting.search_after).isBlank()) {
+            searchAfter = Arrays.stream(setting.get(CQLElasticSetting.search_after).split(searchAfterSplitRegex))
+                    .filter(v -> !v.isBlank())
+                    .map(String::trim)
+                    .map(ElasticSearch::toFieldValue)
+                    .toList();
+        }
+
+        List<SortOptions> sortOptions = createSortOptions(sortBy, CQLFields.class);
+
+        if (factory.isParameterPrioritySort()) {
+            if (sortOptions == null) sortOptions = new ArrayList<>();
+            sortOptions.add(0, CQLFields.parameter_vocabs.getSortBuilder().apply(SortOrder.Desc).build());
+        }
+
+        if (factory.isPlatformPrioritySort()) {
+            if (sortOptions == null) sortOptions = new ArrayList<>();
+            sortOptions.add(0, CQLFields.platform_vocabs.getSortBuilder().apply(SortOrder.Desc).build());
+        }
+
+        return buildCollectionSearchRequestSupplier(
+                null,
+                should,
+                filters,
+                properties,
+                searchAfter,
+                sortOptions,
+                score,
+                maxSize);
+    }
+
     @Override
     public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(List<String> keywords, String cql, List<String> properties, String sortBy, CQLCrsType coor) throws CQLException {
 
@@ -341,7 +460,7 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
             Long maxSize = null;
             try {
                 if(setting.get(CQLElasticSetting.page_size) != null &&
-                    !setting.get(CQLElasticSetting.page_size).isBlank()) {
+                        !setting.get(CQLElasticSetting.page_size).isBlank()) {
                     maxSize = Long.parseLong(setting.get(CQLElasticSetting.page_size));
                 }
             }
@@ -370,7 +489,7 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
                     !setting.get(CQLElasticSetting.search_after).isBlank()) {
                 // Convert the regex separate string to List<FieldValue>
                 searchAfter = Arrays.stream(setting.get(CQLElasticSetting.search_after)
-                        .split(searchAfterSplitRegex))
+                                .split(searchAfterSplitRegex))
                         .filter(v -> !v.isBlank())
                         .map(String::trim)
                         .map(ElasticSearch::toFieldValue)
@@ -407,6 +526,19 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
         }
     }
 
+    @Override
+    public JsonNode explainByParameters(List<String> targets, String filter, List<String> properties, String sortBy, CQLCrsType coor) throws Exception {
+        return explainCollectionBy(
+                buildParameterSearchRequestSupplier(targets, filter, properties, sortBy, coor));
+    }
+
+    @Override
+    public JsonNode explainByUuid(String uuid, List<String> targets, String filter, List<String> properties, String sortBy, CQLCrsType coor) throws Exception {
+        return explainCollectionById(
+                uuid,
+                buildParameterSearchRequestSupplier(targets, filter, properties, sortBy, coor));
+    }
+
     @Override
     public BinaryResponse searchCollectionVectorTile(List<String> ids, Integer tileMatrix, Integer tileRow, Integer tileCol) throws IOException {