diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
index c117ff514..1dcdba771 100644
--- a/mcc/resources/queries/mcc/genomicDatasetsSource.sql
+++ b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
@@ -7,4 +7,4 @@ SELECT
FROM sequenceanalysis.sequence_readsets r
-WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC_%'
\ No newline at end of file
+WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC[_]%'
\ No newline at end of file
diff --git a/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml b/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml
index b97e7ed08..77d9a2808 100644
--- a/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml
+++ b/mcc/resources/queries/study/demographicsMostRecentWeight.query.xml
@@ -13,7 +13,7 @@
query.queryName=Weight&
query.id~eq=${id}
- 0.##
+ 0.####
Current Weight (g)
diff --git a/mcc/resources/queries/study/demographicsMostRecentWeight.sql b/mcc/resources/queries/study/demographicsMostRecentWeight.sql
index 283c33561..3497dc940 100644
--- a/mcc/resources/queries/study/demographicsMostRecentWeight.sql
+++ b/mcc/resources/queries/study/demographicsMostRecentWeight.sql
@@ -12,7 +12,7 @@ timestampdiff('SQL_TSI_DAY', w.MostRecentWeightDate, now()) AS DaysSinceWeight,
null as weightField,
--NOTE: we need to be careful in case duplicate weights are entered on the same time
cast((
- SELECT round(cast(AVG(w2.weight) as double), 2) AS _expr
+ SELECT round(cast(AVG(w2.weight) as double), 4) AS _expr
FROM study.weight w2
WHERE w.id=w2.id AND w.MostRecentWeightDate=w2.date
) as double) AS MostRecentWeight,
diff --git a/mcc/resources/queries/study/weight.query.xml b/mcc/resources/queries/study/weight.query.xml
index 29c16ce25..aa161af35 100644
--- a/mcc/resources/queries/study/weight.query.xml
+++ b/mcc/resources/queries/study/weight.query.xml
@@ -38,8 +38,8 @@
- Weight (kg)
- 0.###
+ Weight (kg)
+ 0.####
diff --git a/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java b/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java
index 308256d28..87a9f7f3c 100644
--- a/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java
+++ b/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java
@@ -36,6 +36,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -133,7 +134,7 @@ private void processFile(PipelineJob job) throws PipelineJobException
String objectId = new TableSelector(clinicalObs, PageFlowUtil.set("objectid"), filter, null).getObject(String.class);
if (observation == null && objectId != null)
{
- toDelete.add(row);
+ toDelete.add(new CaseInsensitiveHashMap<>(Map.of("objectid", objectId)));
continue;
}
diff --git a/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java b/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java
index 7e2ab8bb1..0efdead3d 100644
--- a/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java
+++ b/mcc/src/org/labkey/mcc/notification/MCCDataNotification.java
@@ -14,8 +14,6 @@
import org.labkey.mcc.MccManager;
import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
/**
* User: bimber
@@ -81,6 +79,7 @@ public String getMessageBodyHTML(Container c, User u)
doU24AssignedCheck(mccData, u , msg);
doMissingIdCheck(mccData, u, msg);
doZeroWeightCheck(mccData, u, msg);
+ doDuplicationCheck(mccData, u, msg);
//since we dont want to trigger an email if there's no alerts, conditionally append the title
if (msg.length() > 0)
@@ -174,4 +173,37 @@ protected void doParentSexCheck(final Container c, User u, final StringBuilder m
msg.append("
\n\n");
}
}
+
+ protected void doDuplicationCheck(final Container c, User u, final StringBuilder msg)
+ {
+ TableInfo ti = getUserSchemaByName(c, u, "mcc").getTable("duplicateDemographics");
+ TableSelector ts = new TableSelector(ti);
+ long count = ts.getRowCount();
+ if (count > 0)
+ {
+ msg.append("WARNING: There are ").append(count).append(" demographics records with duplicated MCC IDs\n");
+ msg.append("Click here to view them
\n\n");
+ msg.append("
\n\n");
+ }
+
+ ti = getUserSchemaByName(c, u, "mcc").getTable("duplicatedAggregatedDemographics");
+ ts = new TableSelector(ti);
+ count = ts.getRowCount();
+ if (count > 0)
+ {
+ msg.append("WARNING: There are ").append(count).append(" aggregated demographics records with duplicated MCC IDs\n");
+ msg.append("Click here to view them
\n\n");
+ msg.append("
\n\n");
+ }
+
+ ti = getUserSchemaByName(c, u, "mcc").getTable("duplicatedAggregatedDemographicsParents");
+ ts = new TableSelector(ti);
+ count = ts.getRowCount();
+ if (count > 0)
+ {
+ msg.append("WARNING: There are ").append(count).append(" aggregated demographics parent records with duplicated MCC IDs\n");
+ msg.append("Click here to view them
\n\n");
+ msg.append("
\n\n");
+ }
+ }
}
diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
index 9dba77ced..e0140293f 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
@@ -98,8 +98,8 @@ public Integer getMaxRequestCpus(PipelineJob job)
if (isSequenceNormalizationTask(job))
{
- job.getLogger().debug("setting max CPUs to 8");
- return 8;
+ job.getLogger().debug("setting max CPUs to 4");
+ return 4;
}
if (isLuceneIndexJob(job))
@@ -125,14 +125,14 @@ public Integer getMaxRequestCpus(PipelineJob job)
}
else if (totalFileSize < 20e9)
{
- job.getLogger().debug("file size less than 20gb, lowering CPUs to 16");
+ job.getLogger().debug("file size less than 20gb, lowering CPUs to 12");
- return 16;
+ return 12;
}
- job.getLogger().debug("file size greater than 20gb, using 24 CPUs");
+ job.getLogger().debug("file size greater than 20gb, using 12 CPUs");
- return 24;
+ return 12;
}
return null;
@@ -155,8 +155,8 @@ public Integer getMaxRequestMemory(PipelineJob job)
if (isSequenceNormalizationTask(job))
{
- job.getLogger().debug("setting memory to 48");
- return 48;
+ job.getLogger().debug("setting memory to 18");
+ return 18;
}
if (isGeneticsTask(job))
@@ -167,8 +167,8 @@ public Integer getMaxRequestMemory(PipelineJob job)
if (isCacheAlignerIndexesTask(job))
{
- job.getLogger().debug("setting memory to 48");
- return 48;
+ job.getLogger().debug("setting memory to 12");
+ return 12;
}
if (isLuceneIndexJob(job))
@@ -314,7 +314,7 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng
}
@Override
- public Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine)
+ public @NotNull Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine)
{
Map ret = new HashMap<>();
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java
index c970f16a7..1bea4bf76 100644
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java
+++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java
@@ -512,6 +512,12 @@ else if (discordantBarcodes.contains(barcode))
for (File f : Arrays.asList(consensusFastaGD, allFastaGD))
{
_log.info("processing G/D FASTA: " + f.getPath());
+ if (!SequencePipelineService.get().hasMinLineCount(f, 2))
+ {
+ _log.info("too few lines, skipping FASTA:" + f.getPath());
+ continue;
+ }
+
try (FastaDataLoader loader = new FastaDataLoader(f, false))
{
loader.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter());