Fix some DbCache-related test failures

study/api-src/org/labkey/api/specimen/SpecimenColumns.java → specimen/src/org/labkey/specimen/SpecimenColumns.java

@@ -1,9 +1,9 @@
-package org.labkey.api.specimen;
+package org.labkey.specimen;
 
-import org.labkey.api.specimen.importer.ImportTypes;
-import org.labkey.api.specimen.importer.ImportableColumn;
-import org.labkey.api.specimen.importer.SpecimenColumn;
-import org.labkey.api.specimen.importer.TargetTable;
+import org.labkey.specimen.importer.ImportTypes;
+import org.labkey.specimen.importer.ImportableColumn;
+import org.labkey.specimen.importer.SpecimenColumn;
+import org.labkey.specimen.importer.TargetTable;
 
 import java.util.Arrays;
 import java.util.Collection;

specimen/src/org/labkey/specimen/SpecimenManager.java

@@ -1,5 +1,6 @@
 package org.labkey.specimen;
 
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.audit.AuditLogService;
 import org.labkey.api.data.ColumnInfo;
@@ -8,6 +9,7 @@
 import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.Sort;
+import org.labkey.api.data.SqlExecutor;
 import org.labkey.api.data.SqlSelector;
 import org.labkey.api.data.Table;
 import org.labkey.api.data.TableInfo;
@@ -18,9 +20,10 @@
 import org.labkey.api.query.QueryService;
 import org.labkey.api.query.UserSchema;
 import org.labkey.api.security.User;
+import org.labkey.api.specimen.SpecimenEvent;
+import org.labkey.api.specimen.SpecimenEventManager;
 import org.labkey.api.specimen.SpecimenManagerNew;
 import org.labkey.api.specimen.SpecimenQuerySchema;
-import org.labkey.api.specimen.SpecimenRequestException;
 import org.labkey.api.specimen.SpecimenSchema;
 import org.labkey.api.specimen.Vial;
 import org.labkey.api.specimen.location.LocationImpl;
@@ -34,6 +37,7 @@
 import org.labkey.api.util.DateUtil;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.Path;
+import org.labkey.api.view.NotFoundException;
 import org.labkey.api.view.ViewContext;
 import org.labkey.specimen.model.AdditiveType;
 import org.labkey.specimen.model.DerivativeType;
@@ -347,15 +351,15 @@ public List<Vial> getVials(Container container, User user, String participantId,
         SimpleFilter filter = SimpleFilter.createContainerFilter(container);
         filter.addClause(new SimpleFilter.SQLClause("LOWER(ptid) = LOWER(?)", new Object[] {participantId}, FieldKey.fromParts("ptid")));
         filter.addCondition(FieldKey.fromParts("VisitValue"), visit);
-        return SpecimenManagerNew.get().getVials(container, user, filter);
+        return getVials(container, user, filter);
     }
 
     public List<Vial> getVials(Container container, User user, int[] vialsRowIds)
     {
         Set<Long> uniqueRowIds = new HashSet<>(vialsRowIds.length);
         for (int vialRowId : vialsRowIds)
             uniqueRowIds.add((long)vialRowId);
-        return SpecimenManagerNew.get().getVials(container, user, uniqueRowIds);
+        return getVials(container, user, uniqueRowIds);
     }
 
     public List<Vial> getVials(Container container, User user, String[] globalUniqueIds) throws SpecimenRequestException
@@ -365,7 +369,7 @@ public List<Vial> getVials(Container container, User user, String[] globalUnique
         Collections.addAll(uniqueRowIds, globalUniqueIds);
         List<String> ids = new ArrayList<>(uniqueRowIds);
         filter.addInClause(FieldKey.fromParts("GlobalUniqueId"), ids);
-        List<Vial> vials = SpecimenManagerNew.get().getVials(container, user, filter);
+        List<Vial> vials = getVials(container, user, filter);
         if (vials == null || vials.size() != ids.size())
             throw new SpecimenRequestException("Vial not found.");       // an id has no matching specimen, let caller determine what to report
         return vials;
@@ -377,7 +381,7 @@ public List<Vial> getVials(Container container, User user, String participantId,
         Calendar endCal = DateUtil.newCalendar(date.getTime());
         endCal.add(Calendar.DATE, 1);
         filter.addClause(new SimpleFilter.SQLClause("DrawTimestamp >= ? AND DrawTimestamp < ?", new Object[] {date, endCal.getTime()}));
-        return SpecimenManagerNew.get().getVials(container, user, filter);
+        return getVials(container, user, filter);
     }
 
     @Nullable
@@ -415,14 +419,14 @@ private List<DerivativeType> getDerivativeTypes(final Container container, @Null
 
     public boolean isSpecimensEmpty(Container container, User user)
     {
-        TableSelector selector = SpecimenManagerNew.get().getSpecimensSelector(container, user, null);
+        TableSelector selector = getSpecimensSelector(container, user, null);
         return !selector.exists();
     }
 
     public Vial getVial(Container container, User user, String globalUniqueId)
     {
         SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("LOWER(GlobalUniqueId) = LOWER(?)", new Object[] { globalUniqueId }));
-        List<Vial> matches = SpecimenManagerNew.get().getVials(container, user, filter);
+        List<Vial> matches = getVials(container, user, filter);
         if (matches == null || matches.isEmpty())
             return null;
         if (matches.size() > 1)
@@ -453,7 +457,7 @@ public List<Vial> getRequestableVials(Container container, User user, Set<Long>
     {
         SimpleFilter filter = SimpleFilter.createContainerFilter(container);
         filter.addInClause(FieldKey.fromParts("RowId"), vialRowIds).addCondition(FieldKey.fromString("available"), true);
-        return SpecimenManagerNew.get().getVials(container, user, filter);
+        return getVials(container, user, filter);
     }
 
     public Map<String,List<Vial>> getVialsForSpecimenHashes(Container container, User user, Collection<String> hashes, boolean onlyAvailable)
@@ -462,7 +466,7 @@ public Map<String,List<Vial>> getVialsForSpecimenHashes(Container container, Use
         filter.addInClause(FieldKey.fromParts("SpecimenHash"), hashes);
         if (onlyAvailable)
             filter.addCondition(FieldKey.fromParts("Available"), true);
-        List<Vial> vials = SpecimenManagerNew.get().getVials(container, user, filter);
+        List<Vial> vials = getVials(container, user, filter);
         Map<String, List<Vial>> map = new HashMap<>();
         for (Vial vial : vials)
         {
@@ -473,4 +477,117 @@ public Map<String,List<Vial>> getVialsForSpecimenHashes(Container container, Use
 
         return map;
     }
+
+    public List<Vial> getVials(Container container, User user, Set<Long> vialRowIds)
+    {
+        // Take a set to eliminate dups - issue 26940
+
+        SimpleFilter filter = SimpleFilter.createContainerFilter(container);
+        filter.addInClause(FieldKey.fromParts("RowId"), vialRowIds);
+        List<Vial> vials = getVials(container, user, filter);
+        if (vials.size() != vialRowIds.size())
+        {
+            List<Long> unmatchedRowIds = new ArrayList<>(vialRowIds);
+            for (Vial vial : vials)
+            {
+                unmatchedRowIds.remove(vial.getRowId());
+            }
+            throw new SpecimenRequestException("One or more specimen RowIds had no matching specimen: " + unmatchedRowIds);
+        }
+        return vials;
+    }
+
+    public List<Vial> getVials(final Container container, final User user, SimpleFilter filter)
+    {
+        // TODO: LinkedList?
+        final List<Vial> vials = new ArrayList<>();
+
+        getSpecimensSelector(container, user, filter)
+                .forEachMap(map -> vials.add(new Vial(container, map)));
+
+        return vials;
+    }
+
+    public TableSelector getSpecimensSelector(final Container container, final User user, SimpleFilter filter)
+    {
+        Study study = StudyService.get().getStudy(container);
+        if (study == null)
+        {
+            throw new NotFoundException("No study in container " + container.getPath());
+        }
+        UserSchema schema = SpecimenQuerySchema.get(study, user);
+        TableInfo specimenTable = schema.getTable(SpecimenQuerySchema.SPECIMEN_WRAP_TABLE_NAME);
+        return new TableSelector(specimenTable, filter, null);
+    }
+
+    public Vial getVial(Container container, User user, long rowId)
+    {
+        SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("RowId"), rowId);
+        List<Vial> vials = getVials(container, user, filter);
+        if (vials.isEmpty())
+            return null;
+        return vials.get(0);
+    }
+
+    public void deleteSpecimen(@NotNull Vial vial, boolean clearCaches)
+    {
+        Container container = vial.getContainer();
+        TableInfo tableInfoSpecimenEvent = SpecimenSchema.get().getTableInfoSpecimenEvent(container);
+        TableInfo tableInfoVial = SpecimenSchema.get().getTableInfoVial(container);
+        if (null == tableInfoSpecimenEvent || null == tableInfoVial)
+            return;
+
+        String tableInfoSpecimenEventSelectName = tableInfoSpecimenEvent.getSelectName();
+        String tableInfoVialSelectName = tableInfoVial.getSelectName();
+
+        SQLFragment sqlFragmentEvent = new SQLFragment("DELETE FROM ");
+        sqlFragmentEvent.append(tableInfoSpecimenEventSelectName).append(" WHERE VialId = ?");
+        sqlFragmentEvent.add(vial.getRowId());
+        new SqlExecutor(SpecimenSchema.get().getSchema()).execute(sqlFragmentEvent);
+
+        SQLFragment sqlFragment = new SQLFragment("DELETE FROM ");
+        sqlFragment.append(tableInfoVialSelectName).append(" WHERE RowId = ?");
+        sqlFragment.add(vial.getRowId());
+        new SqlExecutor(SpecimenSchema.get().getSchema()).execute(sqlFragment);
+
+        if (clearCaches)
+        {
+            SpecimenRequestManager.get().clearCaches(vial.getContainer());
+        }
+    }
+
+    public long getMaxExternalId(Container container)
+    {
+        TableInfo tableInfo = SpecimenSchema.get().getTableInfoSpecimenEvent(container);
+        SQLFragment sql = new SQLFragment("SELECT MAX(ExternalId) FROM ");
+        sql.append(tableInfo);
+        return new SqlSelector(tableInfo.getSchema(), sql).getArrayList(Long.class).get(0);
+    }
+
+    public String getFirstProcessedByInitials(List<SpecimenEvent> dateOrderedEvents)
+    {
+        SpecimenEvent firstEvent = SpecimenEventManager.get().getFirstEvent(dateOrderedEvents);
+        return firstEvent != null ? firstEvent.getProcessedByInitials() : null;
+    }
+
+    public List<SpecimenEvent> getSpecimenEvents(List<Vial> vials, boolean includeObsolete)
+    {
+        if (vials == null || vials.isEmpty())
+            return Collections.emptyList();
+        Collection<Long> vialIds = new HashSet<>();
+        Container container = null;
+        for (Vial vial : vials)
+        {
+            vialIds.add(vial.getRowId());
+            if (container == null)
+                container = vial.getContainer();
+            else if (!container.equals(vial.getContainer()))
+                throw new IllegalArgumentException("All specimens must be from the same container");
+        }
+        SimpleFilter filter = new SimpleFilter();
+        filter.addInClause(FieldKey.fromString("VialId"), vialIds);
+        if (!includeObsolete)
+            filter.addCondition(FieldKey.fromString("Obsolete"), false);
+        return SpecimenEventManager.get().getSpecimenEvents(container, filter);
+    }
 }

specimen/src/org/labkey/specimen/SpecimenModule.java

@@ -16,6 +16,7 @@
 
 package org.labkey.specimen;
 
+import jakarta.servlet.http.HttpServletResponse;
 import org.apache.commons.collections4.bag.HashBag;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.logging.log4j.Logger;
@@ -36,21 +37,24 @@
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.query.FieldKey;
-import org.labkey.api.query.QueryParam;
+import org.labkey.api.query.QuerySettings;
+import org.labkey.api.query.QueryUpdateService;
 import org.labkey.api.query.QueryView;
 import org.labkey.api.query.ValidationException;
+import org.labkey.api.reader.TabLoader;
 import org.labkey.api.security.User;
 import org.labkey.api.security.roles.RoleManager;
 import org.labkey.api.specimen.SpecimenMigrationService;
 import org.labkey.api.specimen.SpecimenQuerySchema;
 import org.labkey.api.specimen.SpecimenSchema;
 import org.labkey.api.specimen.SpecimensPage;
-import org.labkey.api.specimen.importer.SpecimenImporter;
 import org.labkey.api.specimen.settings.RepositorySettings;
 import org.labkey.api.specimen.settings.SettingsManager;
+import org.labkey.api.study.MapArrayExcelWriter;
 import org.labkey.api.study.SpecimenService;
 import org.labkey.api.study.StudyInternalService;
 import org.labkey.api.study.StudyService;
+import org.labkey.api.study.TimepointType;
 import org.labkey.api.study.importer.SimpleStudyImportContext;
 import org.labkey.api.study.importer.SimpleStudyImporterRegistry;
 import org.labkey.api.study.writer.SimpleStudyWriterRegistry;
@@ -59,17 +63,23 @@
 import org.labkey.api.util.logging.LogHelper;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.HttpView;
+import org.labkey.api.view.ViewContext;
 import org.labkey.api.view.WebPartFactory;
 import org.labkey.specimen.actions.SpecimenApiController;
 import org.labkey.specimen.actions.SpecimenController;
 import org.labkey.specimen.importer.AbstractSpecimenTask;
 import org.labkey.specimen.importer.DefaultSpecimenImportStrategyFactory;
+import org.labkey.specimen.importer.RequestabilityManager;
+import org.labkey.specimen.importer.SimpleSpecimenImporter;
+import org.labkey.specimen.importer.SpecimenImporter;
 import org.labkey.specimen.importer.SpecimenSchemaImporter;
 import org.labkey.specimen.importer.SpecimenSettingsImporter;
 import org.labkey.specimen.model.SpecimenRequestEventType;
 import org.labkey.specimen.pipeline.SampleMindedTransform;
 import org.labkey.specimen.pipeline.SampleMindedTransformTask;
 import org.labkey.specimen.pipeline.SpecimenPipeline;
+import org.labkey.specimen.query.SpecimenQueryView;
+import org.labkey.specimen.query.SpecimenUpdateService;
 import org.labkey.specimen.security.roles.SpecimenCoordinatorRole;
 import org.labkey.specimen.security.roles.SpecimenRequesterRole;
 import org.labkey.specimen.view.ManageSpecimenView;
@@ -82,6 +92,7 @@
 import org.labkey.specimen.writer.SpecimenSettingsWriter;
 import org.labkey.specimen.writer.SpecimenWriter;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
@@ -163,47 +174,63 @@ public ActionURL getSpecimensURL(Container c)
             }
 
             @Override
-            public ActionURL getSpecimenEventsURL(Container c, ActionURL returnURL)
+            public void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge, boolean syncParticipantVisit) throws PipelineJobException, ValidationException
             {
-                return new ActionURL(SpecimenController.SpecimenEventsAction.class, c).addReturnURL(returnURL);
+                AbstractSpecimenTask.doImport(inputFile, job, ctx, merge, syncParticipantVisit);
             }
 
             @Override
-            public ActionURL getInsertSpecimenQueryRowURL(Container c, String schemaName, TableInfo table)
+            public void clearRequestCaches(Container c)
             {
-                ActionURL url = new ActionURL(SpecimenController.InsertSpecimenQueryRowAction.class, c);
-                url.addParameter("schemaName", schemaName);
-                url.addParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName, table.getName());
+                SpecimenRequestManager.get().clearCaches(c);
+            }
+
+            @Override
+            public @Nullable QueryUpdateService getSpecimenQueryUpdateService(Container c, TableInfo queryTable)
+            {
+                return SettingsManager.get().getRepositorySettings(c).isSpecimenDataEditable() ? new SpecimenUpdateService(queryTable) : null;
+            }
 
-                return url;
+            @Override
+            public void importSpecimens(Container container, User user, List<Map<String, Object>> specimens) throws ValidationException, IOException
+            {
+                SimpleSpecimenImporter importer = new SimpleSpecimenImporter(container, user);
+                importer.process(specimens, false);
             }
 
             @Override
-            public ActionURL getUpdateSpecimenQueryRowURL(Container c, String schemaName, TableInfo table)
+            public void exportSpecimens(Container container, User user, List<Map<String, Object>> specimens, TimepointType timepointType, String participantIdLabel, HttpServletResponse response)
             {
-                ActionURL url = new ActionURL(SpecimenController.UpdateSpecimenQueryRowAction.class, c);
-                url.addParameter("schemaName", schemaName);
-                url.addParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName, table.getName());
+                SimpleSpecimenImporter importer = new SimpleSpecimenImporter(container, user, timepointType, participantIdLabel);
+                MapArrayExcelWriter xlWriter = new MapArrayExcelWriter(specimens, importer.getSimpleSpecimenColumns());
+                // Note: I don't think this is having any effect on the output because ExcelColumn.renderCaption() uses
+                // the DisplayColumn's caption, not its own caption. That seems wrong...
+                xlWriter.setColumnModifier(col -> col.setCaption(importer.label(col.getName())));
+                xlWriter.renderWorkbook(response);
+            }
 
-                return url;
+            @Override
+            public Map<String, String> getColumnLabelMap(Container container, User user)
+            {
+                return new SimpleSpecimenImporter(container, user).getColumnLabels();
             }
 
             @Override
-            public void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge, boolean syncParticipantVisit) throws PipelineJobException, ValidationException
+            public void fixupSpecimenColumns(Container container, User user, TabLoader loader) throws IOException
             {
-                AbstractSpecimenTask.doImport(inputFile, job, ctx, merge, syncParticipantVisit);
+                new SimpleSpecimenImporter(container, user).fixupSpecimenColumns(loader);
             }
 
             @Override
-            public void clearRequestCaches(Container c)
+            public QueryView getSpecimenQueryView(ViewContext context, QuerySettings settings)
             {
-                SpecimenRequestManager.get().clearCaches(c);
+                return SpecimenQueryView.createView(context, settings, SpecimenQueryView.ViewType.VIALS);
             }
 
             @Override
-            public void updateVialCounts(Container container, User user)
+            public void setDefaultRequestabilityRules(Container container, User user)
             {
-                SpecimenRequestManager.get().updateVialCounts(container, user);
+                RequestabilityManager.getInstance().setDefaultRules(container, user);
             }
         });
      }

study/api-src/org/labkey/api/specimen/SpecimenRequestException.java → specimen/src/org/labkey/specimen/SpecimenRequestException.java

@@ -1,4 +1,4 @@
-package org.labkey.api.specimen;
+package org.labkey.specimen;
 
 public class SpecimenRequestException extends RuntimeException
 {