diff --git a/src/main/java/uk/org/llgc/annotation/store/adapters/StoreAdapter.java b/src/main/java/uk/org/llgc/annotation/store/adapters/StoreAdapter.java index 98d818ad..356f4edb 100644 --- a/src/main/java/uk/org/llgc/annotation/store/adapters/StoreAdapter.java +++ b/src/main/java/uk/org/llgc/annotation/store/adapters/StoreAdapter.java @@ -32,9 +32,15 @@ public interface StoreAdapter { // CRUD manifests public String indexManifest(final Manifest pManifest) throws IOException; + /** + * Check for any canvases that have been annotated before the Manifest was loaded + * Call after index Manifest + */ + public void linkupOrphanCanvas(final Manifest pManifest) throws IOException; + public List getManifests() throws IOException; public List getSkeletonManifests(final User pUser) throws IOException; - public String getManifestId(final String pShortId) throws IOException; + public String getManifestId(final String pShortId) throws IOException; public Manifest getManifest(final String pId) throws IOException; public Manifest getManifestForCanvas(final Canvas pCanvasId) throws IOException; diff --git a/src/main/java/uk/org/llgc/annotation/store/adapters/elastic/ElasticStore.java b/src/main/java/uk/org/llgc/annotation/store/adapters/elastic/ElasticStore.java index 315d8242..6693664c 100644 --- a/src/main/java/uk/org/llgc/annotation/store/adapters/elastic/ElasticStore.java +++ b/src/main/java/uk/org/llgc/annotation/store/adapters/elastic/ElasticStore.java @@ -267,6 +267,7 @@ protected void createIndex() throws IOException { // id, motivation, body, target, selector, within, data, short_id, label public Annotation addAnnotationSafe(final Annotation pAnno) throws IOException { + _logger.debug("addAnnotationSafe"); IndexRequest tIndex = new IndexRequest(_index); tIndex.id(pAnno.getId()); @@ -324,6 +325,7 @@ protected Map anno2json(final Annotation pAnno) { } public void deleteAnnotation(final String pAnnoId) throws IOException { + _logger.debug("deleteAnnotation"); DeleteRequest tDelete = new DeleteRequest(_index); tDelete.id(pAnnoId); @@ -332,6 +334,7 @@ public void deleteAnnotation(final String pAnnoId) throws IOException { } public AnnotationList getAnnotationsFromPage(final User pUser, final Canvas pPage) throws IOException { + _logger.debug("getAnnotationsFromPage"); BoolQueryBuilder tBuilder = QueryBuilders.boolQuery(); tBuilder.must(QueryBuilders.termQuery("target.id", pPage.getId())); @@ -353,6 +356,7 @@ public AnnotationList getAnnotationsFromPage(final User pUser, final Canvas pPag } public Annotation getAnnotation(final String pId) throws IOException { + _logger.debug("getAnnotation"); GetRequest tRequest = new GetRequest(_index, pId); GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT); if (tResponse != null && tResponse.getSource() != null) { @@ -364,6 +368,7 @@ public Annotation getAnnotation(final String pId) throws IOException { public AnnotationList getAllAnnotations() throws IOException { + _logger.debug("getAllAnnotations"); AnnotationList tList = new AnnotationList(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.termQuery("type", "oa:Annotation")); @@ -380,6 +385,7 @@ public AnnotationList getAllAnnotations() throws IOException { } public void linkupOrphanCanvas(final Manifest pManifest) throws IOException { + _logger.debug("linkupOrphanCanvas"); for (Canvas tCanvas : pManifest.getCanvases()) { BoolQueryBuilder tBuilder = QueryBuilders.boolQuery(); tBuilder.mustNot(QueryBuilders.existsQuery("target.within.id")); @@ -390,6 +396,7 @@ public void linkupOrphanCanvas(final Manifest pManifest) throws IOException { searchSourceBuilder.size(10000); SearchRequest searchRequest = new SearchRequest(_index); searchRequest.source(searchSourceBuilder); + _logger.debug("callingSearch"); SearchResponse searchResponse = _client.search(searchRequest, RequestOptions.DEFAULT); SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); @@ -414,22 +421,28 @@ public void linkupOrphanCanvas(final Manifest pManifest) throws IOException { } protected String indexManifestNoCheck(final String pShortId, final Manifest pManifest) throws IOException { + _logger.debug("indexManifestNoCheck"); pManifest.setShortId(pShortId); IndexRequest tIndex = new IndexRequest(_index); tIndex.id(pShortId); tIndex.source(manifest2Json(pManifest)); + RefreshPolicy tGeneralPolicy = _policy; + _policy = RefreshPolicy.NONE; - tIndex.setRefreshPolicy(_policy); - _client.index(tIndex, RequestOptions.DEFAULT); - this.linkupOrphanCanvas(pManifest); + //this.linkupOrphanCanvas(pManifest); for (Canvas tCanvas : pManifest.getCanvases()) { this.storeCanvas(tCanvas); } + + _policy = tGeneralPolicy; + tIndex.setRefreshPolicy(_policy); + _client.index(tIndex, RequestOptions.DEFAULT); return pShortId; } public String getManifestId(final String pShortId) throws IOException { + _logger.debug("getManifestId"); GetRequest tRequest = new GetRequest(_index, pShortId); GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT); @@ -442,6 +455,7 @@ public String getManifestId(final String pShortId) throws IOException { } public Manifest getManifest(final String pId) throws IOException { + _logger.debug("getManifest"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.termQuery("id", pId)); searchSourceBuilder.size(1); @@ -462,6 +476,7 @@ public Manifest getManifest(final String pId) throws IOException { } public Manifest getManifestForCanvas(final Canvas pCanvas) throws IOException { + _logger.debug("getManifestCanvas"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.termQuery("canvases.id", pCanvas.getId())); SearchRequest searchRequest = new SearchRequest(_index); @@ -477,6 +492,7 @@ public Manifest getManifestForCanvas(final Canvas pCanvas) throws IOException { } public List getManifests() throws IOException { + _logger.debug("getManifests"); List tManifests = new ArrayList(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(10000); @@ -495,6 +511,7 @@ public List getManifests() throws IOException { // TODO note this will return indexed manifests as well as non indexed.. public List getSkeletonManifests(final User pUser) throws IOException { + _logger.debug("getSkeletonManifests"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.aggregation(AggregationBuilders.terms("manifests").field("target.within.id").size(10000)); if (!pUser.isAdmin()) { @@ -516,6 +533,7 @@ public List getSkeletonManifests(final User pUser) throws IOException } public void storeCanvas(final Canvas pCanvas) throws IOException { + _logger.debug("storeCanvas: " + pCanvas.getId()); IndexRequest tIndex = new IndexRequest(_index); tIndex.id(pCanvas.getShortId()); Map tJson = pCanvas.toJson(); @@ -523,11 +541,14 @@ public void storeCanvas(final Canvas pCanvas) throws IOException { tIndex.source(tJson); tIndex.setRefreshPolicy(_policy); + _logger.debug("Index"); _client.index(tIndex, RequestOptions.DEFAULT); + _logger.debug("done"); } public Canvas resolveCanvas(final String pShortId) throws IOException { + _logger.debug("resolveCanvas"); GetRequest tRequest = new GetRequest(_index, pShortId); GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT); @@ -565,6 +586,7 @@ protected Manifest json2Manifest(final Map pJson) throws IOExcep } public IIIFSearchResults search(final SearchQuery pQuery) throws IOException { + _logger.debug("search"); BoolQueryBuilder tBuilder = QueryBuilders.boolQuery(); if (pQuery.getMotivations() != null && !pQuery.getMotivations().isEmpty()) { tBuilder.must(QueryBuilders.termsQuery("motivation", pQuery.getMotivations())); @@ -649,6 +671,7 @@ public IIIFSearchResults search(final SearchQuery pQuery) throws IOException { } public List listAnnoPages(final Manifest pManifest, final User pUser) throws IOException { + _logger.debug("listAnnoPages"); BoolQueryBuilder tBuilder = QueryBuilders.boolQuery(); tBuilder.must(QueryBuilders.termQuery("target.within.id", pManifest.getURI())); if (pUser != null) { @@ -669,7 +692,7 @@ public List listAnnoPages(final Manifest pManifest, final User pU tLabel = pManifest.getCanvas(tFacet.getKeyAsString()).getLabel(); } Canvas tCanvas = new Canvas(tFacet.getKeyAsString(), tLabel); - this.storeCanvas(tCanvas); + //this.storeCanvas(tCanvas); tAnnoPageCount.add(new PageAnnoCount(tCanvas, (int)tFacet.getDocCount(), pManifest)); } @@ -677,6 +700,7 @@ public List listAnnoPages(final Manifest pManifest, final User pU } public List getUsers() throws IOException { + _logger.debug("getUsers"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.termQuery("type", "User")); searchSourceBuilder.size(10000); @@ -694,6 +718,7 @@ public List getUsers() throws IOException { } public User getUser(final User pUser) throws IOException { + _logger.debug("getUser"); GetRequest tRequest = new GetRequest(_index, pUser.getId()); GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT); @@ -707,6 +732,7 @@ public User getUser(final User pUser) throws IOException { } } public User saveUser(final User pUser) throws IOException { + _logger.debug("saveUser"); User tSavedUser = getUser(pUser); if (tSavedUser != null) { // This is an update @@ -725,6 +751,7 @@ public User saveUser(final User pUser) throws IOException { } public User deleteUser(final User pUser) throws IOException { + _logger.debug("deleteUser"); DeleteRequest tDelete = new DeleteRequest(_index); tDelete.id(pUser.getId()); @@ -838,6 +865,7 @@ protected Collection collectionFromMap(final Map pJson) throws I } public Collection createCollection(final Collection pCollection) throws IOException { + _logger.debug("createCollection"); IndexRequest tIndex = new IndexRequest(_index); tIndex.id(pCollection.getId()); Map tJson = this.object2json(pCollection); @@ -850,6 +878,7 @@ public Collection createCollection(final Collection pCollection) throws IOExcept } public List getCollections(final User pUser) throws IOException { + _logger.debug("getCollections(pUser)"); BoolQueryBuilder tBuilder = QueryBuilders.boolQuery(); tBuilder.must(QueryBuilders.termQuery("type", "Collection")); tBuilder.must(QueryBuilders.termQuery("creator", pUser.getId())); @@ -874,6 +903,7 @@ public List getCollections(final User pUser) throws IOException { } public List getCollections(final SearchSourceBuilder tQuery) throws IOException { + _logger.debug("getCollections(SearchSourceBuilder)"); SearchRequest searchRequest = new SearchRequest(_index); searchRequest.source(tQuery); SearchResponse searchResponse = _client.search(searchRequest, RequestOptions.DEFAULT); @@ -892,6 +922,7 @@ public List getCollections(final SearchSourceBuilder tQuery) throws } public Collection getCollection(final String pId) throws IOException { + _logger.debug("getCollection(id)"); GetRequest tRequest = new GetRequest(_index, pId); GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT); @@ -903,6 +934,7 @@ public Collection getCollection(final String pId) throws IOException { } public void deleteCollection(final Collection pCollection) throws IOException { + _logger.debug("deleteCollection()"); DeleteRequest tDelete = new DeleteRequest(_index); tDelete.id(pCollection.getId()); @@ -911,6 +943,7 @@ public void deleteCollection(final Collection pCollection) throws IOException { } public int getTotalAnnotations(final User pUser) throws IOException { + _logger.debug("getTotalAnnotations()"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); if (pUser == null) { searchSourceBuilder.query(QueryBuilders.termQuery("type", "oa:Annotation")); @@ -928,6 +961,7 @@ public int getTotalAnnotations(final User pUser) throws IOException { } public int getTotalManifests(final User pUser) throws IOException { + _logger.debug("getTotalManifests()"); if (pUser == null) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.termQuery("type", "sc:Manifest")); @@ -952,6 +986,7 @@ public int getTotalManifests(final User pUser) throws IOException { } public int getTotalAnnoCanvases(final User pUser) throws IOException { + _logger.debug("getTotalAnnoCanvases()"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); if (pUser == null) { searchSourceBuilder.query(QueryBuilders.termQuery("type", "oa:Annotation")); diff --git a/src/main/java/uk/org/llgc/annotation/store/adapters/rdf/AbstractRDFStore.java b/src/main/java/uk/org/llgc/annotation/store/adapters/rdf/AbstractRDFStore.java index a3e784e9..3d163e9c 100644 --- a/src/main/java/uk/org/llgc/annotation/store/adapters/rdf/AbstractRDFStore.java +++ b/src/main/java/uk/org/llgc/annotation/store/adapters/rdf/AbstractRDFStore.java @@ -876,7 +876,12 @@ protected String indexManifestNoCheck(final String pShortId, Manifest pManifest) pManifest.setShortId(pShortId); pManifest.toJson().put(DC.identifier.getURI(), pShortId); String tShortId = this.indexManifestOnly(pShortId, pManifest.toJson()); - // Now update any annotations which don't contain a link to this manifest. + + return tShortId; + } + + public void linkupOrphanCanvas(final Manifest pManifest) throws IOException { + // Now update any annotations which don't contain a link to this manifest. String tQueryString = "PREFIX oa: " + "PREFIX sc: " + "PREFIX rdf: " + @@ -933,8 +938,7 @@ protected String indexManifestNoCheck(final String pShortId, Manifest pManifest) // found no annotations that weren't linked to this manifest } - return tShortId; - } + } public int getTotalAnnotations(final User pUser) { StringBuffer sparql = new StringBuffer("PREFIX rdf: \n"); diff --git a/src/main/java/uk/org/llgc/annotation/store/adapters/solr/SolrStore.java b/src/main/java/uk/org/llgc/annotation/store/adapters/solr/SolrStore.java index 729da2da..8de17593 100644 --- a/src/main/java/uk/org/llgc/annotation/store/adapters/solr/SolrStore.java +++ b/src/main/java/uk/org/llgc/annotation/store/adapters/solr/SolrStore.java @@ -204,7 +204,6 @@ public List getSkeletonManifests(final User pUser) throws IOException protected String indexManifestNoCheck(final String pShortId, final Manifest pManifest) throws IOException { pManifest.setShortId(pShortId); _manifestStore.indexManifestNoCheck(pManifest); - this.linkupOrphanCanvas(pManifest); return pManifest.getShortId(); } @@ -439,7 +438,7 @@ public List listAnnoPages(final Manifest pManifest, final User pU tLabel = pManifest.getCanvas(tFacetValue.getName()).getLabel(); } Canvas tCanvas = new Canvas(tFacetValue.getName(), tLabel); - this.storeCanvas(tCanvas); + //this.storeCanvas(tCanvas); tAnnoPageCount.add(new PageAnnoCount(tCanvas, (int)tFacetValue.getCount(), pManifest)); } return tAnnoPageCount; diff --git a/src/main/java/uk/org/llgc/annotation/store/controllers/StoreService.java b/src/main/java/uk/org/llgc/annotation/store/controllers/StoreService.java index 6d5c9145..2af40e04 100644 --- a/src/main/java/uk/org/llgc/annotation/store/controllers/StoreService.java +++ b/src/main/java/uk/org/llgc/annotation/store/controllers/StoreService.java @@ -302,6 +302,11 @@ public Manifest getManifestId(final String pURI) { public List getCollections(final HttpServletRequest pRequest) throws IOException { UserService tService = new UserService(pRequest); User tUser = tService.getUser(); + String tKey = "get_collections_from_request_" + tService.getUser().getShortId(); + if (this.isCached(tKey)) { + return (List)this.getCacheObject(tKey); + } + _logger.debug("getCollections(pRequest)"); List tCollections = _store.getCollections(tUser); // if empty create the default collection if (tCollections.isEmpty()) { @@ -314,7 +319,7 @@ public List getCollections(final HttpServletRequest pRequest) throws } Collections.sort(tCollections); - + this.putCacheObject(tKey, tCollections); return tCollections; } @@ -334,6 +339,7 @@ protected boolean isCached(final String pKey) { } public List getCollections(final User pUser) throws IOException { + _logger.debug("getCollections(pUser)"); String tKey = "collections_for_user_" + pUser.getShortId(); if (this.isCached(tKey)) { @@ -354,6 +360,7 @@ public List getCollections(final User pUser) throws IOException { } public Collection getCollection(final String pID, final HttpServletRequest pRequest) throws IOException { + String tCollectionKey = "collection_"; if (pID == null || pID.length() == 0) { UserService tService = new UserService(pRequest); User tUser = tService.getUser(); @@ -368,9 +375,17 @@ public Collection getCollection(final String pID, final HttpServletRequest pRequ Collection tDefaultCollection = new Collection(); tDefaultCollection.setUser(tUser); tDefaultCollection.createDefaultId(StoreConfig.getConfig().getBaseURI(pRequest)); - return _store.getCollection(tDefaultCollection.getId()); + Collection tResponse = _store.getCollection(tDefaultCollection.getId()); + this.putCacheObject(tCollectionKey + tResponse.getId(), tResponse); + return tResponse; } else { - return _store.getCollection(pID); + if (this.isCached(tCollectionKey + pID)) { + return (Collection)this.getCacheObject(tCollectionKey + pID); + } else { + Collection tResponse = _store.getCollection(pID); + this.putCacheObject(tCollectionKey + tResponse.getId(), tResponse); + return tResponse; + } } } } diff --git a/src/test/java/uk/org/llgc/annotation/store/test/TestSearch.java b/src/test/java/uk/org/llgc/annotation/store/test/TestSearch.java index 3b80f506..0cd3ab9f 100644 --- a/src/test/java/uk/org/llgc/annotation/store/test/TestSearch.java +++ b/src/test/java/uk/org/llgc/annotation/store/test/TestSearch.java @@ -112,8 +112,10 @@ public void loadManifest() throws IOException, IDConflictException, MalformedAnn List tLoadedManifests = _store.getManifests(); assertTrue("Store shouldn't have any manifests registered but answered " + tLoadedManifests, tLoadedManifests != null && tLoadedManifests.isEmpty()); - Map tManifest = (Map)JsonUtils.fromInputStream(new FileInputStream(getClass().getResource("/jsonld/testManifest.json").getFile())); - String tShortId = _store.indexManifest(new Manifest(tManifest)); + Map tManifestJson = (Map)JsonUtils.fromInputStream(new FileInputStream(getClass().getResource("/jsonld/testManifest.json").getFile())); + Manifest tManifest = new Manifest(tManifestJson); + String tShortId = _store.indexManifest(tManifest); + _store.linkupOrphanCanvas(tManifest); tLoadedManifests = _store.getManifests(); assertEquals("Store should have 1 manifests registered but answered " + tLoadedManifests, 1, tLoadedManifests.size()); @@ -320,8 +322,10 @@ public void testEndToEnd() throws IOException, IDConflictException, URISyntaxExc _store.addAnnotationList(new AnnotationList(tAnnotationListJSON)); // Upload Manifest - Map tManifest = (Map)JsonUtils.fromInputStream(new FileInputStream(getClass().getResource("/examples/Cambrian_1804-01-28.json").getFile())); //annotaiton list - String tShortId = _store.indexManifest(new Manifest(tManifest)); + Map tManifestJson = (Map)JsonUtils.fromInputStream(new FileInputStream(getClass().getResource("/examples/Cambrian_1804-01-28.json").getFile())); //annotaiton list + Manifest tManifest = new Manifest(tManifestJson); + String tShortId = _store.indexManifest(tManifest); + _store.linkupOrphanCanvas(tManifest); SearchQuery tQuery = new SearchQuery("chimney"); tQuery.setScope("http://dams.llgc.org.uk/iiif/newspaper/issue/3320640/manifest.json");