diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql index c117ff514..1dcdba771 100644 --- a/mcc/resources/queries/mcc/genomicDatasetsSource.sql +++ b/mcc/resources/queries/mcc/genomicDatasetsSource.sql @@ -7,4 +7,4 @@ SELECT FROM sequenceanalysis.sequence_readsets r -WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC_%' \ No newline at end of file +WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC[_]%' \ No newline at end of file diff --git a/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml b/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml index b97e7ed08..77d9a2808 100644 --- a/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml +++ b/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml @@ -13,7 +13,7 @@ query.queryName=Weight& query.id~eq=${id} - 0.## + 0.#### Current Weight (g) diff --git a/mcc/resources/queries/study/demographicsMostRecentWeight.sql b/mcc/resources/queries/study/demographicsMostRecentWeight.sql index 283c33561..3497dc940 100644 --- a/mcc/resources/queries/study/demographicsMostRecentWeight.sql +++ b/mcc/resources/queries/study/demographicsMostRecentWeight.sql @@ -12,7 +12,7 @@ timestampdiff('SQL_TSI_DAY', w.MostRecentWeightDate, now()) AS DaysSinceWeight, null as weightField, --NOTE: we need to be careful in case duplicate weights are entered on the same time cast(( - SELECT round(cast(AVG(w2.weight) as double), 2) AS _expr + SELECT round(cast(AVG(w2.weight) as double), 4) AS _expr FROM study.weight w2 WHERE w.id=w2.id AND w.MostRecentWeightDate=w2.date ) as double) AS MostRecentWeight, diff --git a/mcc/resources/queries/study/weight.query.xml b/mcc/resources/queries/study/weight.query.xml index 29c16ce25..aa161af35 100644 --- a/mcc/resources/queries/study/weight.query.xml +++ b/mcc/resources/queries/study/weight.query.xml @@ -38,8 +38,8 @@ - Weight (kg) - 0.### + Weight (kg) + 0.#### diff --git a/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java b/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java index 308256d28..87a9f7f3c 100644 --- a/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java +++ b/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -133,7 +134,7 @@ private void processFile(PipelineJob job) throws PipelineJobException String objectId = new TableSelector(clinicalObs, PageFlowUtil.set("objectid"), filter, null).getObject(String.class); if (observation == null && objectId != null) { - toDelete.add(row); + toDelete.add(new CaseInsensitiveHashMap<>(Map.of("objectid", objectId))); continue; } diff --git a/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java b/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java index 7e2ab8bb1..0efdead3d 100644 --- a/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java +++ b/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java @@ -14,8 +14,6 @@ import org.labkey.mcc.MccManager; import java.util.Date; -import java.util.HashMap; -import java.util.Map; /** * User: bimber @@ -81,6 +79,7 @@ public String getMessageBodyHTML(Container c, User u) doU24AssignedCheck(mccData, u , msg); doMissingIdCheck(mccData, u, msg); doZeroWeightCheck(mccData, u, msg); + doDuplicationCheck(mccData, u, msg); //since we dont want to trigger an email if there's no alerts, conditionally append the title if (msg.length() > 0) @@ -174,4 +173,37 @@ protected void doParentSexCheck(final Container c, User u, final StringBuilder m msg.append("
\n\n"); } } + + protected void doDuplicationCheck(final Container c, User u, final StringBuilder msg) + { + TableInfo ti = getUserSchemaByName(c, u, "mcc").getTable("duplicateDemographics"); + TableSelector ts = new TableSelector(ti); + long count = ts.getRowCount(); + if (count > 0) + { + msg.append("WARNING: There are ").append(count).append(" demographics records with duplicated MCC IDs\n"); + msg.append("

Click here to view them
\n\n"); + msg.append("


\n\n"); + } + + ti = getUserSchemaByName(c, u, "mcc").getTable("duplicatedAggregatedDemographics"); + ts = new TableSelector(ti); + count = ts.getRowCount(); + if (count > 0) + { + msg.append("WARNING: There are ").append(count).append(" aggregated demographics records with duplicated MCC IDs\n"); + msg.append("

Click here to view them
\n\n"); + msg.append("


\n\n"); + } + + ti = getUserSchemaByName(c, u, "mcc").getTable("duplicatedAggregatedDemographicsParents"); + ts = new TableSelector(ti); + count = ts.getRowCount(); + if (count > 0) + { + msg.append("WARNING: There are ").append(count).append(" aggregated demographics parent records with duplicated MCC IDs\n"); + msg.append("

Click here to view them
\n\n"); + msg.append("


\n\n"); + } + } } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 9dba77ced..e0140293f 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -98,8 +98,8 @@ public Integer getMaxRequestCpus(PipelineJob job) if (isSequenceNormalizationTask(job)) { - job.getLogger().debug("setting max CPUs to 8"); - return 8; + job.getLogger().debug("setting max CPUs to 4"); + return 4; } if (isLuceneIndexJob(job)) @@ -125,14 +125,14 @@ public Integer getMaxRequestCpus(PipelineJob job) } else if (totalFileSize < 20e9) { - job.getLogger().debug("file size less than 20gb, lowering CPUs to 16"); + job.getLogger().debug("file size less than 20gb, lowering CPUs to 12"); - return 16; + return 12; } - job.getLogger().debug("file size greater than 20gb, using 24 CPUs"); + job.getLogger().debug("file size greater than 20gb, using 12 CPUs"); - return 24; + return 12; } return null; @@ -155,8 +155,8 @@ public Integer getMaxRequestMemory(PipelineJob job) if (isSequenceNormalizationTask(job)) { - job.getLogger().debug("setting memory to 48"); - return 48; + job.getLogger().debug("setting memory to 18"); + return 18; } if (isGeneticsTask(job)) @@ -167,8 +167,8 @@ public Integer getMaxRequestMemory(PipelineJob job) if (isCacheAlignerIndexesTask(job)) { - job.getLogger().debug("setting memory to 48"); - return 48; + job.getLogger().debug("setting memory to 12"); + return 12; } if (isLuceneIndexJob(job)) @@ -314,7 +314,7 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng } @Override - public Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine) + public @NotNull Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine) { Map ret = new HashMap<>(); diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index c970f16a7..1bea4bf76 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -512,6 +512,12 @@ else if (discordantBarcodes.contains(barcode)) for (File f : Arrays.asList(consensusFastaGD, allFastaGD)) { _log.info("processing G/D FASTA: " + f.getPath()); + if (!SequencePipelineService.get().hasMinLineCount(f, 2)) + { + _log.info("too few lines, skipping FASTA:" + f.getPath()); + continue; + } + try (FastaDataLoader loader = new FastaDataLoader(f, false)) { loader.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter());