Adding filtering and pagination to collectionList APIs

doc/release-notes/12423-collection-list-search-and-pagination.md

@@ -0,0 +1,7 @@
+## Feature ##
+
+Updated the following APIs to add ability to filter using `searchTerm` and added optional pagination parameters `offset` and `limit` to limit the results with each GET.
+
+GET `/api/users/$USERNAME/allowedCollections/$PERMISSION?limit=10&offset=0&searchTerm=bio`
+
+GET `/api/mydata/retrieve/collectionList?userIdentifier=anotherUser&limit=10&offset=11&searchTerm=bio`

doc/sphinx-guides/source/api/native-api.rst

@@ -8719,13 +8719,20 @@ The ``$identifier`` is the username of the requested user.
 The ``$permission`` is the permission (tied to the roles) that gives the user access to the collection.
 Passing ``$permission`` as 'any' will return the collection as long as the user has any access/permission on the collection
 
+**For filtering and pagination these query parameters can be used:**
+
+- ``searchTerm``: To filter the results.
+- ``offset``: Starting row.
+- ``limit``: Number of items to limit the output.
+
 .. code-block:: bash
 
   export SERVER_URL=https://demo.dataverse.org
   export $USERNAME=jsmith
   export PERMISSION=PublishDataverse
 
   curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/$USERNAME/allowedCollections/$PERMISSION"
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/$USERNAME/allowedCollections/$PERMISSION?limit=10&offset=0&searchTerm=bio"
 
 Show Role Assignee
 ~~~~~~~~~~~~~~~~~~
@@ -9313,6 +9320,12 @@ MyData Collection List
 The MyData Collection List API is used to get a list of the collections an authenticated user can create a Dataset in.
 Param userIdentifier={userName} is used by a superuser to get the collections for a specific user.
 
+**For filtering and pagination these query parameters can be used:**
+
+- ``searchTerm``: To filter the results.
+- ``offset``: Starting row.
+- ``limit``: Number of items to limit the output.
+
 A curl example listing collections:
 
 .. code-block:: bash
@@ -9322,4 +9335,5 @@ A curl example listing collections:
 
   curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList"
   curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList?userIdentifier=anotherUser"
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList?userIdentifier=anotherUser&limit=10&offset=11&searchTerm=bio"
 

src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java

@@ -1,11 +1,10 @@
 package edu.harvard.iq.dataverse;
 
-import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
+import edu.harvard.iq.dataverse.api.util.Pagination;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv4Address;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv6Address;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
-import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.GuestUser;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.RoleAssignee;
@@ -16,6 +15,7 @@
 
 import java.util.*;
 import java.util.logging.Logger;
+
 import jakarta.ejb.EJB;
 import jakarta.ejb.Stateless;
 import jakarta.inject.Inject;
@@ -59,20 +59,11 @@
                     .filter(Permission::requiresAuthenticatedUser)
                     .collect(Collectors.toList()));
 
-    @EJB
-    BuiltinUserServiceBean userService;
-
-    @EJB
-    AuthenticationServiceBean authenticationService;
-
     @EJB
     DataverseRoleServiceBean roleService;
 
     @EJB
     RoleAssigneeServiceBean roleAssigneeService;
-
-    @EJB
-    DataverseServiceBean dataverseService;
 
     @EJB
     DvObjectServiceBean dvObjectServiceBean;
@@ -86,26 +77,23 @@
     @EJB
     GroupServiceBean groupService;
 
-    @Inject
-    DataverseSession session;
-
     @Inject
     DataverseRequestServiceBean dvRequestService;
 
     @Inject
     DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
 
     private static final String LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION = """
-        SELECT id, name, alias FROM DATAVERSE dv
+        SELECT id, name, alias, ROW_NUMBER() OVER (ORDER BY id) AS row_num FROM DATAVERSE dv
     """;
 
     private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """
             WITH grouplist AS (
                   SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser
                   WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID
                 )
-                        
-                SELECT * FROM DATAVERSE dv WHERE id IN (
+
+                SELECT *, ROW_NUMBER() OVER (ORDER BY id) AS row_num FROM DATAVERSE dv WHERE id IN (
                   SELECT definitionpoint_id
                   FROM roleassignment
                   WHERE roleassignment.assigneeidentifier IN (
@@ -179,6 +167,14 @@
                                                                      or (LOWER(dv.name) LIKE ?))))
                                                      """;
 
+    private static final String PAGE_PARAMS  = """
+                                               select * from ( @SQL@ )
+                                               where row_num BETWEEN @START AND @END
+                                               """;
+
+    private static final String TOTAL_COUNT  = """
+                                               select count(*) from ( @SQL@ )
+                                               """;
     /**
      * A request-level permission query (e.g includes IP ras).
      */
@@ -939,33 +935,52 @@
     }
 
     public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission) {
-        return findPermittedCollections(request, user, 1 << permission.ordinal(), "");
+        return findPermittedCollections(request, user, 1 << permission.ordinal(), "", null);
     }
 
     public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) {
-        return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm);
+        return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm, null);
     }
 
-    public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) {
-        return findPermittedCollections(request, user, permissionBit, "");
-    }
-
-
-    public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm) {
+    public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm, Pagination pagination) {
         if (user != null) {
+            List<Dataverse> dataverses = new ArrayList<>();
             var sqlCode = getBaseQueryForAllPermittedDataverses(request, user, permissionBit);
-            if (searchTerm == null || searchTerm.isEmpty()) {
-                return em.createNativeQuery(sqlCode, Dataverse.class).getResultList();
-            } else if (user.isSuperuser()) {
-                Query query = em.createNativeQuery(sqlCode.concat(WHERE).concat(SEARCH_PARAMS), Dataverse.class);
-                setSearchParamValues(searchTerm, query);
-                return query.getResultList();
+            if (searchTerm != null && !searchTerm.isEmpty()) {
+                if (user.isSuperuser()) {
+                    sqlCode = sqlCode.concat(WHERE).concat(SEARCH_PARAMS);
+                } else {
+                    sqlCode = sqlCode.concat(AND).concat(SEARCH_PARAMS);
+                }
             }
-            else {
-                Query query = em.createNativeQuery(sqlCode.concat(AND).concat(SEARCH_PARAMS), Dataverse.class);
+            if (pagination != null) {
+                if (pagination.getNumResults() < 0) {
+                    // Get the total count
+                    String sqlCodeCount = TOTAL_COUNT.replace("@SQL@", sqlCode);
+                    Query queryCount = em.createNativeQuery(sqlCodeCount);
+                    if (searchTerm != null && !searchTerm.isEmpty()) {
+                        setSearchParamValues(searchTerm, queryCount);
+                    }
+                    Number countResult = (Number) queryCount.getSingleResult();
+                    pagination.setNumResults(countResult.intValue());
+                }
+                // Add a pagination wrapper around the sqlCode
+                int pageStart = pagination.getOffset();
+                int pageEnd = pageStart + pagination.getLimitPerPage();
+                sqlCode = PAGE_PARAMS
+                        .replace("@START", String.valueOf(pageStart))
+                        .replace("@END", String.valueOf(pageEnd))
+                        .replace("@SQL@", sqlCode);
+            }
+
+            Query query = em.createNativeQuery(sqlCode, Dataverse.class);
+            if (searchTerm != null && !searchTerm.isEmpty()) {
                 setSearchParamValues(searchTerm, query);
-                return query.getResultList();
             }
+
+            List resultList = query.getResultList();
+            dataverses.addAll(resultList);
+            return dataverses;
         }
         return null;
     }

src/main/java/edu/harvard/iq/dataverse/api/Users.java

@@ -7,6 +7,7 @@
 
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.api.util.Pagination;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.GuestUser;
@@ -323,7 +324,11 @@ public Response getUserPermittedCollections(@Context ContainerRequestContext crc
             @Parameter(description = "Authenticated user identifier whose permitted collections are returned.", required = true)
             @PathParam("identifier") String identifier,
             @Parameter(description = "Permission name used to select permitted collections.", required = true)
-            @PathParam("permission") String permission) {
+            @PathParam("permission") String permission,
+            @Parameter(description = "Offset used to override the starting point of the list.")
+            @QueryParam("offset") Integer start,
+            @Parameter(description = "Page size to limit the number of items in the list.")
+            @QueryParam("limit") Integer pageSize) {
         AuthenticatedUser authenticatedUser = null;
         try {
             authenticatedUser = getRequestAuthenticatedUserOrDie(crc);
@@ -335,8 +340,9 @@ public Response getUserPermittedCollections(@Context ContainerRequestContext crc
         }
         try {
             AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier);
-            List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), userToQuery, permission));
-            return ok(JsonPrinter.jsonArray(collections));
+            Pagination pagination = (start != null || pageSize != null) ? new Pagination(pageSize, start) : null;
+            List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), userToQuery, permission, null, pagination));
+            return ok(JsonPrinter.jsonArray(collections, pagination));
         } catch (WrappedResponse ex) {
             return ex.getResponse();
         }

src/main/java/edu/harvard/iq/dataverse/api/util/Pagination.java

@@ -0,0 +1,31 @@
+package edu.harvard.iq.dataverse.api.util;
+
+public class Pagination {
+
+    private int numResults = -1;
+    private int offset;
+    private int limitPerPage;
+
+    private static int DEFAULT_PAGE_SIZE = 10;
+
+    public Pagination(Integer limitPerPage, Integer offset) {
+        this.offset = offset != null ? Math.max(offset, 0) : 0;
+        this.limitPerPage = limitPerPage != null ? Math.max(limitPerPage, 1) : DEFAULT_PAGE_SIZE;
+    }
+
+    public void setNumResults(int numResults) {
+        this.numResults = numResults;
+    }
+
+    public int getNumResults() {
+        return numResults;
+    }
+
+    public int getOffset() {
+        return offset;
+    }
+
+    public int getLimitPerPage() {
+        return limitPerPage;
+    }
+}

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java

@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.api.util.Pagination;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
@@ -40,12 +41,19 @@ public class GetUserPermittedCollectionsCommand extends AbstractCommand<List<Dat
     private final DataverseRequest request;
     private final AuthenticatedUser user;
     private final String permission;
+    private String searchTerm;
+    private final Pagination pagination;
 
     public GetUserPermittedCollectionsCommand(DataverseRequest request, AuthenticatedUser user, String permission) {
+        this(request, user, permission, null, null);
+    }
+    public GetUserPermittedCollectionsCommand(DataverseRequest request, AuthenticatedUser user, String permission, String searchTerm, Pagination pagination) {
         super(request, (DvObject) null);
         this.request = request;
         this.user = user;
         this.permission = permission;
+        this.searchTerm = searchTerm;
+        this.pagination = pagination;
     }
 
     @Override
@@ -59,6 +67,6 @@ public List<Dataverse> execute(CommandContext ctxt) throws CommandException {
         } catch (IllegalArgumentException e) {
             throw new InvalidCommandArgumentsException(BundleUtil.getStringFromBundle("getUserPermittedCollectionsCommand.errors.permissionNotValid"), this);
         }
-        return ctxt.permissions().findPermittedCollections(request, user, permissionBit);
+        return ctxt.permissions().findPermittedCollections(request, user, permissionBit, searchTerm, pagination);
     }
 }

src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java

@@ -5,6 +5,7 @@
 
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.api.util.Pagination;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.GuestUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
@@ -336,11 +337,19 @@ private JsonObjectBuilder myDataAsJson(String message, Pager pager, RoleTagRetri
     @Produces("application/json")
     @Operation(summary = "Lists collections for My Data",
             description = "Returns collections where the requester or selected user may add datasets.")
-    public Response retrieveMyCollectionList(@Context ContainerRequestContext crc, @Parameter(description = "User identifier filter.") @QueryParam("userIdentifier") String userIdentifier) {
+    public Response retrieveMyCollectionList(@Context ContainerRequestContext crc,
+                                             @Parameter(description = "User identifier filter.") @QueryParam("userIdentifier") String userIdentifier,
+                                             @Parameter(description = "Search term filter.") @QueryParam("searchTerm") String searchTerm,
+                                             @Parameter(description = "Offset used to override the starting point of the list.")
+                                             @QueryParam("offset") Integer start,
+                                             @Parameter(description = "Page size to limit the number of items in the list.")
+                                             @QueryParam("limit") Integer pageSize) {
         try {
             verifyAuth(crc, userIdentifier);
-            List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), searchUser, Permission.AddDataset.name()));
-            return ok(JsonPrinter.jsonArray(collections));
+            Pagination pagination = (start != null || pageSize != null) ? new Pagination(pageSize, start) : null;
+            List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), searchUser, Permission.AddDataset.name(),
+                    searchTerm, pagination));
+            return ok(JsonPrinter.jsonArray(collections, pagination));
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.api.Util;
+import edu.harvard.iq.dataverse.api.util.Pagination;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.RoleAssigneeDisplayInfo;
@@ -385,8 +386,14 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re
     }
 
     public static JsonObjectBuilder jsonArray(List<Dataverse> dataverses) {
+        return jsonArray(dataverses, null);
+    }
+    public static JsonObjectBuilder jsonArray(List<Dataverse> dataverses, Pagination pagination) {
         JsonObjectBuilder job = JsonUtil.createObjectBuilder();
         job.add("count", dataverses.size());
+        if (pagination != null) {
+            job.add("totalCount", pagination.getNumResults());
+        }
         JsonArrayBuilder jsonArrayBuilder = JsonUtil.createArrayBuilder();
         for (Dataverse dataverse : dataverses) {
             NullSafeJsonBuilder jsonObject = NullSafeJsonBuilder.jsonObjectBuilder();

src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java

@@ -167,12 +167,19 @@
         // Sleep for indexing
         Thread.sleep(4000);
 
-        // User1 gets the list of Dataverses/Collections it has access to
-        retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User1ApiToken, null);
+        // User1 gets the list of Dataverses/Collections it has access to (with pagination)
+        // Get the first page
+        int offset = 0;
+        retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User1ApiToken, null, null, 10);
+        retrieveMyCollectionListResponse.prettyPrint();
+        int count = retrieveMyCollectionListResponse.getBody().jsonPath().getInt("data.count");
+        int totalCount = retrieveMyCollectionListResponse.getBody().jsonPath().getInt("data.totalCount");
+        // get the second page
+        retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User1ApiToken, null, offset + count + 1, 10);
         retrieveMyCollectionListResponse.prettyPrint();
+        count = count + retrieveMyCollectionListResponse.getBody().jsonPath().getInt("data.count");
         // The count should show the list size to be User1's + Root Dataverse count
-        items = retrieveMyCollectionListResponse.getBody().jsonPath().getList("data.items");
-        assertEquals(rootCount + user1DataverseCount, items.size());
+        assertEquals(rootCount + user1DataverseCount, count);
 
         // User2 gets the list of Dataverses/Collections it has access to
         retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User2ApiToken, null);