Skip to content

Speeding up index manifest #134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
glenrobson merged 3 commits into from
Jan 25, 2022
Merged

Speeding up index manifest #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
843e0ed
Speeding up index manifest
glenrobson Jan 25, 2022
09623b0
Moving linkupOrphanCanvas to a optional task
glenrobson Jan 25, 2022
fd79fc5
Removing unnecessary call.
glenrobson Jan 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/main/java/uk/org/llgc/annotation/store/adapters/StoreAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,15 @@ public interface StoreAdapter {

// CRUD manifests
public String indexManifest(final Manifest pManifest) throws IOException;
/**
* Check for any canvases that have been annotated before the Manifest was loaded
* Call after index Manifest
*/
public void linkupOrphanCanvas(final Manifest pManifest) throws IOException;

public List<Manifest> getManifests() throws IOException;
public List<Manifest> getSkeletonManifests(final User pUser) throws IOException;
public String getManifestId(final String pShortId) throws IOException;
public String getManifestId(final String pShortId) throws IOException;
public Manifest getManifest(final String pId) throws IOException;
public Manifest getManifestForCanvas(final Canvas pCanvasId) throws IOException;

Expand Down
43 changes: 39 additions & 4 deletions src/main/java/uk/org/llgc/annotation/store/adapters/elastic/ElasticStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ protected void createIndex() throws IOException {

// id, motivation, body, target, selector, within, data, short_id, label
public Annotation addAnnotationSafe(final Annotation pAnno) throws IOException {
_logger.debug("addAnnotationSafe");
IndexRequest tIndex = new IndexRequest(_index);
tIndex.id(pAnno.getId());

Expand Down Expand Up @@ -324,6 +325,7 @@ protected Map<String, Object> anno2json(final Annotation pAnno) {
}

public void deleteAnnotation(final String pAnnoId) throws IOException {
_logger.debug("deleteAnnotation");
DeleteRequest tDelete = new DeleteRequest(_index);
tDelete.id(pAnnoId);

Expand All @@ -332,6 +334,7 @@ public void deleteAnnotation(final String pAnnoId) throws IOException {
}

public AnnotationList getAnnotationsFromPage(final User pUser, final Canvas pPage) throws IOException {
_logger.debug("getAnnotationsFromPage");
BoolQueryBuilder tBuilder = QueryBuilders.boolQuery();
tBuilder.must(QueryBuilders.termQuery("target.id", pPage.getId()));

Expand All @@ -353,6 +356,7 @@ public AnnotationList getAnnotationsFromPage(final User pUser, final Canvas pPag
}

public Annotation getAnnotation(final String pId) throws IOException {
_logger.debug("getAnnotation");
GetRequest tRequest = new GetRequest(_index, pId);
GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT);
if (tResponse != null && tResponse.getSource() != null) {
Expand All @@ -364,6 +368,7 @@ public Annotation getAnnotation(final String pId) throws IOException {


public AnnotationList getAllAnnotations() throws IOException {
_logger.debug("getAllAnnotations");
AnnotationList tList = new AnnotationList();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("type", "oa:Annotation"));
Expand All @@ -380,6 +385,7 @@ public AnnotationList getAllAnnotations() throws IOException {
}

public void linkupOrphanCanvas(final Manifest pManifest) throws IOException {
_logger.debug("linkupOrphanCanvas");
for (Canvas tCanvas : pManifest.getCanvases()) {
BoolQueryBuilder tBuilder = QueryBuilders.boolQuery();
tBuilder.mustNot(QueryBuilders.existsQuery("target.within.id"));
Expand All @@ -390,6 +396,7 @@ public void linkupOrphanCanvas(final Manifest pManifest) throws IOException {
searchSourceBuilder.size(10000);
SearchRequest searchRequest = new SearchRequest(_index);
searchRequest.source(searchSourceBuilder);
_logger.debug("callingSearch");
SearchResponse searchResponse = _client.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = searchResponse.getHits();
SearchHit[] searchHits = hits.getHits();
Expand All @@ -414,22 +421,28 @@ public void linkupOrphanCanvas(final Manifest pManifest) throws IOException {
}

protected String indexManifestNoCheck(final String pShortId, final Manifest pManifest) throws IOException {
_logger.debug("indexManifestNoCheck");
pManifest.setShortId(pShortId);
IndexRequest tIndex = new IndexRequest(_index);
tIndex.id(pShortId);
tIndex.source(manifest2Json(pManifest));
RefreshPolicy tGeneralPolicy = _policy;
_policy = RefreshPolicy.NONE;

tIndex.setRefreshPolicy(_policy);
_client.index(tIndex, RequestOptions.DEFAULT);

this.linkupOrphanCanvas(pManifest);
//this.linkupOrphanCanvas(pManifest);
for (Canvas tCanvas : pManifest.getCanvases()) {
this.storeCanvas(tCanvas);
}

_policy = tGeneralPolicy;
tIndex.setRefreshPolicy(_policy);
_client.index(tIndex, RequestOptions.DEFAULT);
return pShortId;
}

public String getManifestId(final String pShortId) throws IOException {
_logger.debug("getManifestId");
GetRequest tRequest = new GetRequest(_index, pShortId);
GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT);

Expand All @@ -442,6 +455,7 @@ public String getManifestId(final String pShortId) throws IOException {
}

public Manifest getManifest(final String pId) throws IOException {
_logger.debug("getManifest");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("id", pId));
searchSourceBuilder.size(1);
Expand All @@ -462,6 +476,7 @@ public Manifest getManifest(final String pId) throws IOException {
}

public Manifest getManifestForCanvas(final Canvas pCanvas) throws IOException {
_logger.debug("getManifestCanvas");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("canvases.id", pCanvas.getId()));
SearchRequest searchRequest = new SearchRequest(_index);
Expand All @@ -477,6 +492,7 @@ public Manifest getManifestForCanvas(final Canvas pCanvas) throws IOException {
}

public List<Manifest> getManifests() throws IOException {
_logger.debug("getManifests");
List<Manifest> tManifests = new ArrayList<Manifest>();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(10000);
Expand All @@ -495,6 +511,7 @@ public List<Manifest> getManifests() throws IOException {

// TODO note this will return indexed manifests as well as non indexed..
public List<Manifest> getSkeletonManifests(final User pUser) throws IOException {
_logger.debug("getSkeletonManifests");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(AggregationBuilders.terms("manifests").field("target.within.id").size(10000));
if (!pUser.isAdmin()) {
Expand All @@ -516,18 +533,22 @@ public List<Manifest> getSkeletonManifests(final User pUser) throws IOException
}

public void storeCanvas(final Canvas pCanvas) throws IOException {
_logger.debug("storeCanvas: " + pCanvas.getId());
IndexRequest tIndex = new IndexRequest(_index);
tIndex.id(pCanvas.getShortId());
Map<String, Object> tJson = pCanvas.toJson();
tJson.put("short_id", tJson.remove("http://purl.org/dc/terms/identifier"));
tIndex.source(tJson);

tIndex.setRefreshPolicy(_policy);
_logger.debug("Index");
_client.index(tIndex, RequestOptions.DEFAULT);
_logger.debug("done");
}


public Canvas resolveCanvas(final String pShortId) throws IOException {
_logger.debug("resolveCanvas");
GetRequest tRequest = new GetRequest(_index, pShortId);
GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT);

Expand Down Expand Up @@ -565,6 +586,7 @@ protected Manifest json2Manifest(final Map<String, Object> pJson) throws IOExcep
}

public IIIFSearchResults search(final SearchQuery pQuery) throws IOException {
_logger.debug("search");
BoolQueryBuilder tBuilder = QueryBuilders.boolQuery();
if (pQuery.getMotivations() != null && !pQuery.getMotivations().isEmpty()) {
tBuilder.must(QueryBuilders.termsQuery("motivation", pQuery.getMotivations()));
Expand Down Expand Up @@ -649,6 +671,7 @@ public IIIFSearchResults search(final SearchQuery pQuery) throws IOException {
}

public List<PageAnnoCount> listAnnoPages(final Manifest pManifest, final User pUser) throws IOException {
_logger.debug("listAnnoPages");
BoolQueryBuilder tBuilder = QueryBuilders.boolQuery();
tBuilder.must(QueryBuilders.termQuery("target.within.id", pManifest.getURI()));
if (pUser != null) {
Expand All @@ -669,14 +692,15 @@ public List<PageAnnoCount> listAnnoPages(final Manifest pManifest, final User pU
tLabel = pManifest.getCanvas(tFacet.getKeyAsString()).getLabel();
}
Canvas tCanvas = new Canvas(tFacet.getKeyAsString(), tLabel);
this.storeCanvas(tCanvas);
//this.storeCanvas(tCanvas);
tAnnoPageCount.add(new PageAnnoCount(tCanvas, (int)tFacet.getDocCount(), pManifest));
}

return tAnnoPageCount;
}

public List<User> getUsers() throws IOException {
_logger.debug("getUsers");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("type", "User"));
searchSourceBuilder.size(10000);
Expand All @@ -694,6 +718,7 @@ public List<User> getUsers() throws IOException {
}

public User getUser(final User pUser) throws IOException {
_logger.debug("getUser");
GetRequest tRequest = new GetRequest(_index, pUser.getId());
GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT);

Expand All @@ -707,6 +732,7 @@ public User getUser(final User pUser) throws IOException {
}
}
public User saveUser(final User pUser) throws IOException {
_logger.debug("saveUser");
User tSavedUser = getUser(pUser);
if (tSavedUser != null) {
// This is an update
Expand All @@ -725,6 +751,7 @@ public User saveUser(final User pUser) throws IOException {
}

public User deleteUser(final User pUser) throws IOException {
_logger.debug("deleteUser");
DeleteRequest tDelete = new DeleteRequest(_index);
tDelete.id(pUser.getId());

Expand Down Expand Up @@ -838,6 +865,7 @@ protected Collection collectionFromMap(final Map<String, Object> pJson) throws I
}

public Collection createCollection(final Collection pCollection) throws IOException {
_logger.debug("createCollection");
IndexRequest tIndex = new IndexRequest(_index);
tIndex.id(pCollection.getId());
Map<String, Object> tJson = this.object2json(pCollection);
Expand All @@ -850,6 +878,7 @@ public Collection createCollection(final Collection pCollection) throws IOExcept
}

public List<Collection> getCollections(final User pUser) throws IOException {
_logger.debug("getCollections(pUser)");
BoolQueryBuilder tBuilder = QueryBuilders.boolQuery();
tBuilder.must(QueryBuilders.termQuery("type", "Collection"));
tBuilder.must(QueryBuilders.termQuery("creator", pUser.getId()));
Expand All @@ -874,6 +903,7 @@ public List<Collection> getCollections(final User pUser) throws IOException {
}

public List<Collection> getCollections(final SearchSourceBuilder tQuery) throws IOException {
_logger.debug("getCollections(SearchSourceBuilder)");
SearchRequest searchRequest = new SearchRequest(_index);
searchRequest.source(tQuery);
SearchResponse searchResponse = _client.search(searchRequest, RequestOptions.DEFAULT);
Expand All @@ -892,6 +922,7 @@ public List<Collection> getCollections(final SearchSourceBuilder tQuery) throws
}

public Collection getCollection(final String pId) throws IOException {
_logger.debug("getCollection(id)");
GetRequest tRequest = new GetRequest(_index, pId);
GetResponse tResponse = _client.get(tRequest, RequestOptions.DEFAULT);

Expand All @@ -903,6 +934,7 @@ public Collection getCollection(final String pId) throws IOException {
}

public void deleteCollection(final Collection pCollection) throws IOException {
_logger.debug("deleteCollection()");
DeleteRequest tDelete = new DeleteRequest(_index);
tDelete.id(pCollection.getId());

Expand All @@ -911,6 +943,7 @@ public void deleteCollection(final Collection pCollection) throws IOException {
}

public int getTotalAnnotations(final User pUser) throws IOException {
_logger.debug("getTotalAnnotations()");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (pUser == null) {
searchSourceBuilder.query(QueryBuilders.termQuery("type", "oa:Annotation"));
Expand All @@ -928,6 +961,7 @@ public int getTotalAnnotations(final User pUser) throws IOException {
}

public int getTotalManifests(final User pUser) throws IOException {
_logger.debug("getTotalManifests()");
if (pUser == null) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("type", "sc:Manifest"));
Expand All @@ -952,6 +986,7 @@ public int getTotalManifests(final User pUser) throws IOException {
}

public int getTotalAnnoCanvases(final User pUser) throws IOException {
_logger.debug("getTotalAnnoCanvases()");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (pUser == null) {
searchSourceBuilder.query(QueryBuilders.termQuery("type", "oa:Annotation"));
Expand Down
10 changes: 7 additions & 3 deletions src/main/java/uk/org/llgc/annotation/store/adapters/rdf/AbstractRDFStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -876,7 +876,12 @@ protected String indexManifestNoCheck(final String pShortId, Manifest pManifest)
pManifest.setShortId(pShortId);
pManifest.toJson().put(DC.identifier.getURI(), pShortId);
String tShortId = this.indexManifestOnly(pShortId, pManifest.toJson());
// Now update any annotations which don't contain a link to this manifest.

return tShortId;
}

public void linkupOrphanCanvas(final Manifest pManifest) throws IOException {
// Now update any annotations which don't contain a link to this manifest.
String tQueryString = "PREFIX oa: <http://www.w3.org/ns/oa#> " +
"PREFIX sc: <http://iiif.io/api/presentation/2#> " +
"PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>" +
Expand Down Expand Up @@ -933,8 +938,7 @@ protected String indexManifestNoCheck(final String pShortId, Manifest pManifest)
// found no annotations that weren't linked to this manifest
}

return tShortId;
}
}

public int getTotalAnnotations(final User pUser) {
StringBuffer sparql = new StringBuffer("PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n");
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/uk/org/llgc/annotation/store/adapters/solr/SolrStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ public List<Manifest> getSkeletonManifests(final User pUser) throws IOException
protected String indexManifestNoCheck(final String pShortId, final Manifest pManifest) throws IOException {
pManifest.setShortId(pShortId);
_manifestStore.indexManifestNoCheck(pManifest);
this.linkupOrphanCanvas(pManifest);
return pManifest.getShortId();
}

Expand Down Expand Up @@ -439,7 +438,7 @@ public List<PageAnnoCount> listAnnoPages(final Manifest pManifest, final User pU
tLabel = pManifest.getCanvas(tFacetValue.getName()).getLabel();
}
Canvas tCanvas = new Canvas(tFacetValue.getName(), tLabel);
this.storeCanvas(tCanvas);
//this.storeCanvas(tCanvas);
tAnnoPageCount.add(new PageAnnoCount(tCanvas, (int)tFacetValue.getCount(), pManifest));
}
return tAnnoPageCount;
Expand Down
21 changes: 18 additions & 3 deletions src/main/java/uk/org/llgc/annotation/store/controllers/StoreService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,11 @@ public Manifest getManifestId(final String pURI) {
public List<Collection> getCollections(final HttpServletRequest pRequest) throws IOException {
UserService tService = new UserService(pRequest);
User tUser = tService.getUser();
String tKey = "get_collections_from_request_" + tService.getUser().getShortId();
if (this.isCached(tKey)) {
return (List<Collection>)this.getCacheObject(tKey);
}
_logger.debug("getCollections(pRequest)");
List<Collection> tCollections = _store.getCollections(tUser);
// if empty create the default collection
if (tCollections.isEmpty()) {
Expand All @@ -314,7 +319,7 @@ public List<Collection> getCollections(final HttpServletRequest pRequest) throws
}

Collections.sort(tCollections);

this.putCacheObject(tKey, tCollections);
return tCollections;
}

Expand All @@ -334,6 +339,7 @@ protected boolean isCached(final String pKey) {
}

public List<Collection> getCollections(final User pUser) throws IOException {
_logger.debug("getCollections(pUser)");
String tKey = "collections_for_user_" + pUser.getShortId();

if (this.isCached(tKey)) {
Expand All @@ -354,6 +360,7 @@ public List<Collection> getCollections(final User pUser) throws IOException {
}

public Collection getCollection(final String pID, final HttpServletRequest pRequest) throws IOException {
String tCollectionKey = "collection_";
if (pID == null || pID.length() == 0) {
UserService tService = new UserService(pRequest);
User tUser = tService.getUser();
Expand All @@ -368,9 +375,17 @@ public Collection getCollection(final String pID, final HttpServletRequest pRequ
Collection tDefaultCollection = new Collection();
tDefaultCollection.setUser(tUser);
tDefaultCollection.createDefaultId(StoreConfig.getConfig().getBaseURI(pRequest));
return _store.getCollection(tDefaultCollection.getId());
Collection tResponse = _store.getCollection(tDefaultCollection.getId());
this.putCacheObject(tCollectionKey + tResponse.getId(), tResponse);
return tResponse;
} else {
return _store.getCollection(pID);
if (this.isCached(tCollectionKey + pID)) {
return (Collection)this.getCacheObject(tCollectionKey + pID);
} else {
Collection tResponse = _store.getCollection(pID);
this.putCacheObject(tCollectionKey + tResponse.getId(), tResponse);
return tResponse;
}
}
}
}
Loading