diff --git a/snprc_ehr/resources/queries/study/GenDemoCustomizer.sql b/snprc_ehr/resources/queries/study/GenDemoCustomizer.sql index bb9c6979d..04fc0d3f2 100644 --- a/snprc_ehr/resources/queries/study/GenDemoCustomizer.sql +++ b/snprc_ehr/resources/queries/study/GenDemoCustomizer.sql @@ -16,10 +16,13 @@ Select d.Id, d.species, d.species.arc_species_code as ARC_species, */ - IFNULL(g.HasGeneExpressionData,0) as HasGeneExpressionData, - IFNULL(s.HasSNPData,0) as HasSNPData, - IFNULL(m.HasMicrosatellitesData,0) as HasMicrosatellitesData, - IFNULL(p.HasphenotypesData,0) as HasPhenotypeData + -- Cast boolean → integer first so the IFNULL result type is INTEGER, not BOOLEAN. + -- Both args matching as BOOLEAN triggers SQL Server's BIT→BOOLEAN CASE-wrap (error 4145); + -- mixing BOOLEAN with integer 0 fails on Postgres ("COALESCE types boolean and integer cannot be matched"). + IFNULL(CAST(g.HasGeneExpressionData AS INTEGER), 0) as HasGeneExpressionData, + IFNULL(CAST(s.HasSNPData AS INTEGER), 0) as HasSNPData, + IFNULL(CAST(m.HasMicrosatellitesData AS INTEGER), 0) as HasMicrosatellitesData, + IFNULL(CAST(p.HasphenotypesData AS INTEGER), 0) as HasPhenotypeData From study.demographics d LEFT OUTER JOIN study.GenFlagSNP s diff --git a/snprc_ehr/resources/queries/study/GenDemoHasData.sql b/snprc_ehr/resources/queries/study/GenDemoHasData.sql index ccbf954af..75ff62f44 100644 --- a/snprc_ehr/resources/queries/study/GenDemoHasData.sql +++ b/snprc_ehr/resources/queries/study/GenDemoHasData.sql @@ -7,10 +7,10 @@ Select d.Id, d.gender, d.species, d.species.arc_species_code as ARC_species, - IFNULL(g.HasGeneExpressionData,0) as HasGeneExpressionData, - IFNULL(s.HasSNPData,0) as HasSNPData, - IFNULL(m.HasMicrosatellitesData,0) as HasMicrosatellitesData, - IFNULL(p.HasphenotypesData,0) as HasPhenotypeData + IFNULL(CAST(g.HasGeneExpressionData AS INTEGER), 0) as HasGeneExpressionData, + IFNULL(CAST(s.HasSNPData AS INTEGER), 0) as HasSNPData, + IFNULL(CAST(m.HasMicrosatellitesData AS INTEGER), 0) as HasMicrosatellitesData, + IFNULL(CAST(p.HasphenotypesData AS INTEGER), 0) as HasPhenotypeData From study.demographics d LEFT OUTER JOIN study.GenFlagSNP s diff --git a/snprc_ehr/resources/queries/study/GenHasMicrosatellites.query.xml b/snprc_ehr/resources/queries/study/GenHasMicrosatellites.query.xml index ca31f55e4..d0c10c07a 100644 --- a/snprc_ehr/resources/queries/study/GenHasMicrosatellites.query.xml +++ b/snprc_ehr/resources/queries/study/GenHasMicrosatellites.query.xml @@ -11,7 +11,6 @@ animal Id - diff --git a/snprc_ehr/resources/queries/study/GenHasPhenotype.query.xml b/snprc_ehr/resources/queries/study/GenHasPhenotype.query.xml index 34bdcf847..4214f751b 100644 --- a/snprc_ehr/resources/queries/study/GenHasPhenotype.query.xml +++ b/snprc_ehr/resources/queries/study/GenHasPhenotype.query.xml @@ -11,7 +11,6 @@ animal Id - diff --git a/snprc_ehr/resources/queries/study/GenHasSNP.query.xml b/snprc_ehr/resources/queries/study/GenHasSNP.query.xml index 4abc4519b..d572850ec 100644 --- a/snprc_ehr/resources/queries/study/GenHasSNP.query.xml +++ b/snprc_ehr/resources/queries/study/GenHasSNP.query.xml @@ -11,7 +11,6 @@ animal Id - diff --git a/snprc_ehr/resources/queries/study/LabworkOvaParasite.sql b/snprc_ehr/resources/queries/study/LabworkOvaParasite.sql deleted file mode 100644 index a4f3f0c87..000000000 --- a/snprc_ehr/resources/queries/study/LabworkOvaParasite.sql +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -SELECT ServiceId.Dataset, ServiceId.ServiceName, * -FROM snprc_ehr.labwork_panels -WHERE ServiceId.Dataset='Parasitology' and ServiceId.ServiceName in ('OVA & PARASITES' , 'OVA & PARASITES, URINE' ) -SELECT - l.id, - l.date, - l.project, - l.testid, - l.resultOORIndicator, - l.value_type, - lt.testName, - l.result, - l.qualresult, - l.units, - l.refRange, - l.abnormal_flags, - l.remark, - l.description, - l.runid, - l.taskid, - l.method, - l.objectid, - l.modified, - l.modifiedby, - l.created, - l.createdby -FROM labworkResults as l - inner join snprc_ehr.labwork_panels as lt - on l.serviceTestid = lt.rowId and lt.TestId in (77, 201, 727) \ No newline at end of file diff --git a/snprc_ehr/resources/queries/study/ReportTcruziNewPositives.query.xml b/snprc_ehr/resources/queries/study/ReportTcruziNewPositives.query.xml index 771b551b4..4b9306e69 100644 --- a/snprc_ehr/resources/queries/study/ReportTcruziNewPositives.query.xml +++ b/snprc_ehr/resources/queries/study/ReportTcruziNewPositives.query.xml @@ -27,9 +27,6 @@ First Date - - - diff --git a/snprc_ehr/resources/queries/study/ReportTcruziSummaryAll.query.xml b/snprc_ehr/resources/queries/study/ReportTcruziSummaryAll.query.xml index d42e2b85f..4c85c4717 100644 --- a/snprc_ehr/resources/queries/study/ReportTcruziSummaryAll.query.xml +++ b/snprc_ehr/resources/queries/study/ReportTcruziSummaryAll.query.xml @@ -31,9 +31,6 @@ Test Count - - - diff --git a/snprc_ehr/resources/queries/study/chemMisc.sql b/snprc_ehr/resources/queries/study/chemMisc.sql index 6833c8381..6b1e228dd 100644 --- a/snprc_ehr/resources/queries/study/chemMisc.sql +++ b/snprc_ehr/resources/queries/study/chemMisc.sql @@ -7,7 +7,7 @@ SELECT b.id, b.date, b.testId, - b.testId.Name, + lt.testName, b.resultOORIndicator, b.result, @@ -17,6 +17,9 @@ SELECT b.taskid, b.runId FROM study.labworkResults b - -WHERE b.testId.Type = 'Biochemistry' AND (b.testId.includeInPanel = false OR b.testId.includeInPanel IS NULL) AND b.qcstate.publicdata = true +INNER JOIN snprc_ehr.labwork_panels AS lt + ON b.serviceTestid = lt.rowId + AND lt.ServiceId.Dataset='Biochemistry' + AND (b.serviceTestid.includeInPanel = false OR b.serviceTestid.includeInPanel IS NULL) + AND b.qcstate.publicdata = true diff --git a/snprc_ehr/resources/queries/study/colonyPopulationChange.sql b/snprc_ehr/resources/queries/study/colonyPopulationChange.sql new file mode 100644 index 000000000..7744967b3 --- /dev/null +++ b/snprc_ehr/resources/queries/study/colonyPopulationChange.sql @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +PARAMETERS(StartDate TIMESTAMP, EndDate TIMESTAMP) + +SELECT + T1.id, + T1.id.dataset.demographics.species as species, + 'Births' AS Category, + T1.date, + convert(year(T1.date), integer) AS Year + +FROM study.Birth T1 +WHERE T1.date IS NOT NULL +and cast(COALESCE(STARTDATE, '1900-01-01') as date) <= T1.date and cast(COALESCE(ENDDATE, curdate()) as date) >= cast(T1.date as date) + +UNION ALL + +SELECT + T2.id, + T2.id.dataset.demographics.species, + 'Arrivals' AS Category, + T2.date, + convert(year(T2.date), INTEGER) AS Year + +FROM study.Arrival T2 +WHERE T2.date IS NOT NULL +AND T2.qcstate.publicdata = true +and cast(COALESCE(STARTDATE, '1900-01-01') as date) <= T2.date and cast(COALESCE(ENDDATE, curdate()) as date) >= cast(T2.date as date) + +UNION ALL + +SELECT + T3.id, + T3.id.dataset.demographics.species, + 'Departures' AS Category, + T3.date, + convert(year(T3.date), INTEGER) AS Year + +FROM study.Departure T3 +WHERE T3.date IS NOT NULL +AND T3.qcstate.publicdata = true +and cast(COALESCE(STARTDATE, '1900-01-01') as date) <= T3.date and cast(COALESCE(ENDDATE, curdate()) as date) >= cast(T3.date as date) + +UNION ALL + +SELECT + T4.id, + T4.id.dataset.demographics.species, + 'Deaths' AS Category, + T4.date, + convert(year(T4.date), INTEGER) AS Year + +FROM study.Deaths T4 +WHERE T4.date IS NOT NULL +and cast(COALESCE(STARTDATE, '1900-01-01') as date) <= T4.date and cast(COALESCE(ENDDATE, curdate()) as date) >= cast(T4.date as date) diff --git a/snprc_ehr/resources/queries/study/hematologyMisc.sql b/snprc_ehr/resources/queries/study/hematologyMisc.sql index c9f8fa4d9..81c4c9e07 100644 --- a/snprc_ehr/resources/queries/study/hematologyMisc.sql +++ b/snprc_ehr/resources/queries/study/hematologyMisc.sql @@ -15,5 +15,8 @@ SELECT b.taskid, b.runId FROM study.labworkResults b - -WHERE (testId.includeInPanel = FALSE OR b.testid.includeInPanel IS NULL) and b.qcstate.publicdata = true + INNER JOIN snprc_ehr.labwork_panels AS lt + ON b.serviceTestid = lt.rowId + AND lt.ServiceId.Dataset='Hematology' + AND (b.serviceTestid.includeInPanel = false OR b.serviceTestid.includeInPanel IS NULL) + AND b.qcstate.publicdata = true diff --git a/snprc_ehr/resources/queries/study/labworkResultsAll.query.xml b/snprc_ehr/resources/queries/study/labworkResultsAll.query.xml index ade016a6a..9168b0ca5 100644 --- a/snprc_ehr/resources/queries/study/labworkResultsAll.query.xml +++ b/snprc_ehr/resources/queries/study/labworkResultsAll.query.xml @@ -107,7 +107,7 @@ UserId Users core - ` + true diff --git a/snprc_ehr/test/src/org/labkey/test/tests/snprc_ehr/SNPRC_EHRTest.java b/snprc_ehr/test/src/org/labkey/test/tests/snprc_ehr/SNPRC_EHRTest.java index 5b106e7c3..9c881d5c6 100644 --- a/snprc_ehr/test/src/org/labkey/test/tests/snprc_ehr/SNPRC_EHRTest.java +++ b/snprc_ehr/test/src/org/labkey/test/tests/snprc_ehr/SNPRC_EHRTest.java @@ -26,6 +26,8 @@ import org.labkey.remoteapi.CommandException; import org.labkey.remoteapi.Connection; import org.labkey.remoteapi.query.InsertRowsCommand; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; import org.labkey.remoteapi.query.TruncateTableCommand; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; @@ -40,6 +42,7 @@ import org.labkey.test.components.ext4.widgets.SearchPanel; import org.labkey.test.pages.ehr.AnimalHistoryPage; import org.labkey.test.pages.ehr.ColonyOverviewPage; +import org.labkey.test.pages.ehr.EHRAdminPage; import org.labkey.test.pages.ehr.ParticipantViewPage; import org.labkey.test.pages.snprc_ehr.SNPRCAnimalHistoryPage; import org.labkey.test.tests.ehr.AbstractGenericEHRTest; @@ -100,6 +103,16 @@ public class SNPRC_EHRTest extends AbstractGenericEHRTest implements SqlserverOn private static final String ANIMAL_HISTORY_URL = "/" + PROJECT_NAME + "/ehr-animalHistory.view"; private static final String SNPRC_ROOM_ID = "S824778"; private static final String SNPRC_ROOM_ID2 = "S043365"; + private static final List SND_CATEGORIES = Arrays.asList( + "Alopecia", + "BCS", + "Cumulative Blood", + "Vitals Temperature", + "TB", + "Vitals", + "Weight"); + private static final int SND_PKG_ID_START = 901; + private static final int SND_SUPER_PKG_ID_START = 1900; private static int _pipelineJobCount = 0; @@ -168,29 +181,87 @@ private void doSetup() throws Exception new RReportHelper(this).ensureRConfig(); initProject("SNPRC EHR"); goToProjectHome(); - _containerHelper.enableModules(Arrays.asList("SND")); createTestSubjects(); initGenetics(); - initSND(); + completeSnprcPostStudyImportSetup(); + } + + @Override + protected void initCreatedProject() throws Exception + { + setFormatStrings(); + setEHRModuleProperties(); + createUsersandPermissions(); + + doExtraPreStudyImportSetup(); + + defineQCStates(); + populateInitialData(); + + prepareSnprcPreStudyImportSetup(); + + goToProjectHome(); + + // Populate hard tables (e.g. snprc_ehr.animal_groups) before importing the study so that + // saved queries that pivot over those tables resolve their dynamic columns during import-time + // query validation. Otherwise queries like baboonAssignedColonyUsage / colonyUsageQuery fail + // with "Unknown field [col.pc_SPF::colonytotal]" because the pivot subquery returns no rows. + populateHardTableRecords(); + + importStudy(); + disableMiniProfiler(); + defineQCStates(); + setupStudyPermissions(); + primeCaches(); + } + + private void prepareSnprcPreStudyImportSetup() throws Exception + { goToProjectHome(); + _containerHelper.enableModules(Arrays.asList("SND")); + initSND(); + createSNDCategories(); + createSNDPackages(); + + addExtensibleCols(); + clickFolder(GENETICSFOLDER); _assayHelper = new APIAssayHelper(this); _assayHelper.uploadXarFileAsAssayDesign(ASSAY_GENE_EXPRESSION_XAR, 1); _assayHelper.uploadXarFileAsAssayDesign(ASSAY_MICROSATELLITES_XAR, 2); _assayHelper.uploadXarFileAsAssayDesign(ASSAY_PHENOTYPES_XAR, 3); _assayHelper.uploadXarFileAsAssayDesign(ASSAY_SNPS_XAR, 4); + + _assayHelper.importAssay(ASSAY_GENE_EXPRESSION, ASSAY_GENE_EXPRESSION_TSV, getGeneticsPath()); + _assayHelper.importAssay(ASSAY_MICROSATELLITES, ASSAY_MICROSATELLITES_TSV, getGeneticsPath()); + _assayHelper.importAssay(ASSAY_PHENOTYPES, ASSAY_PHENOTYPES_TSV, getGeneticsPath()); + _assayHelper.importAssay(ASSAY_SNPS, ASSAY_SNPS_TSV, getGeneticsPath()); + } + + private void addExtensibleCols() + { + log("Setup the EHR table definitions"); + EHRAdminPage.beginAt(this, getContainerPath()); + clickAndWait(Locator.linkWithText("Add EHR extensible columns")); + + log("Load EHR table definitions"); + click(Locator.linkWithText("Generate EHR Extensible Columns")); + waitForElement(Locator.tagWithClass("span", "x4-window-header-text").withText("Success")); + assertExt4MsgBox("successfully created columns.", "OK"); + } + + private void completeSnprcPostStudyImportSetup() + { + goToProjectHome(); clickFolder(GENETICSFOLDER); _listHelper.importListArchive(LOOKUP_LIST_ARCHIVE); + clickProject(PROJECT_NAME); PortalHelper portalHelper = new PortalHelper(this); portalHelper.addWebPart("EHR Datasets"); + clickFolder(GENETICSFOLDER); portalHelper.addWebPart("Assay List"); - _assayHelper = new APIAssayHelper(this); - _assayHelper.importAssay(ASSAY_GENE_EXPRESSION, ASSAY_GENE_EXPRESSION_TSV, getGeneticsPath()); - _assayHelper.importAssay(ASSAY_MICROSATELLITES, ASSAY_MICROSATELLITES_TSV, getGeneticsPath()); - _assayHelper.importAssay(ASSAY_PHENOTYPES, ASSAY_PHENOTYPES_TSV, getGeneticsPath()); - _assayHelper.importAssay(ASSAY_SNPS, ASSAY_SNPS_TSV, getGeneticsPath()); } @Override @@ -230,7 +301,7 @@ protected String getExpectedAnimalIDCasing(String id) @Override protected boolean skipStudyImportQueryValidation() { - return true; + return false; } @Override @@ -287,6 +358,141 @@ protected void initSND() goToProjectHome(); } + protected void createSNDCategories() throws CommandException, IOException + { + InsertRowsCommand command = new InsertRowsCommand("snd", "PkgCategories"); + for (String category : SND_CATEGORIES) + { + command.addRow(new HashMap<>(Maps.of( + "Description", category, + "Active", true + ))); + } + + command.execute(createDefaultConnection(), getProjectName()); + } + + protected void createSNDPackages() throws Exception + { + goToProjectHome(); + + Map categoryIds = getSndCategoryIds(); + saveSndPackage(SND_PKG_ID_START, SND_SUPER_PKG_ID_START, "Alopecia", categoryIds.get("Alopecia"), + "Alopecia score: {alopeciaScore}", + Arrays.asList( + sndAttribute("alopeciaScore", "Alopecia Score", "string", false), + sndAttribute("scorer", "Scorer", "string", false))); + saveSndPackage(SND_PKG_ID_START + 1, SND_SUPER_PKG_ID_START + 10, "BCS", categoryIds.get("BCS"), + "BCS: {bcs}", + List.of(sndAttribute("bcs", "BCS", "int", false))); + saveSndPackage(SND_PKG_ID_START + 2, SND_SUPER_PKG_ID_START + 20, "Cumulative Blood", categoryIds.get("Cumulative Blood"), + "Blood volume: {BLOOD_Volume}", + Arrays.asList( + sndAttribute("reason", "Reason", "string", false), + sndAttribute("BLOOD_Volume", "Blood Volume", "double", false))); + saveSndPackage(SND_PKG_ID_START + 3, SND_SUPER_PKG_ID_START + 30, "Vitals Temperature", categoryIds.get("Vitals Temperature"), + "Temperature: {temp}", + List.of(sndAttribute("temp", "Temperature", "double", false))); + saveSndPackage(SND_PKG_ID_START + 4, SND_SUPER_PKG_ID_START + 40, "TB", categoryIds.get("TB"), + "TB result: {tb_result}", + Arrays.asList( + sndAttribute("tb_site", "TB Site", "string", false), + sndAttribute("tb_result", "TB Result", "string", false))); + saveSndPackage(SND_PKG_ID_START + 5, SND_SUPER_PKG_ID_START + 50, "Vitals", categoryIds.get("Vitals"), + "Vitals: {HR}/{RR}/{temp}", + Arrays.asList( + sndAttribute("HR", "Heart Rate", "double", false), + sndAttribute("RR", "Respiratory Rate", "double", false), + sndAttribute("temp", "Temperature", "double", false))); + saveSndPackage(SND_PKG_ID_START + 6, SND_SUPER_PKG_ID_START + 60, "Weight", categoryIds.get("Weight"), + "Weight: {weight} kg", + List.of(sndAttribute("weight", "Weight", "double", false))); + } + + private Map getSndCategoryIds() throws IOException, CommandException + { + SelectRowsCommand command = new SelectRowsCommand("snd", "PkgCategories"); + command.setColumns(Arrays.asList("Description", "CategoryId")); + SelectRowsResponse response = command.execute(createDefaultConnection(), getProjectName()); + + Map categoryIds = new HashMap<>(); + for (Map row : response.getRows()) + { + Object description = row.get("Description"); + Object categoryId = row.get("CategoryId"); + if (description != null && categoryId != null) + { + categoryIds.put(String.valueOf(description), ((Number) categoryId).intValue()); + } + } + + for (String category : SND_CATEGORIES) + { + assertTrue("Missing SND category ID for " + category, categoryIds.containsKey(category)); + } + + return categoryIds; + } + + private Map sndAttribute(String name, String label, String rangeUri, boolean required) + { + return new HashMap<>(Maps.of( + "name", name, + "label", label, + "rangeURI", rangeUri, + "required", required + )); + } + + private void saveSndPackage(int pkgId, int superPkgIdStart, String description, int categoryId, String narrative, List> attributes) + { + String result = (String) executeAsyncScript(buildSaveSndPackageScript(pkgId, superPkgIdStart, description, categoryId, narrative, attributes)); + assertEquals("JavaScript API failure while saving SND package " + description, "Success!", result); + } + + private String buildSaveSndPackageScript(int pkgId, int superPkgIdStart, String description, int categoryId, String narrative, List> attributes) + { + return "LABKEY.Ajax.request({\n" + + " method: 'POST',\n" + + " url: LABKEY.ActionURL.buildURL('snd', 'savePackage.api'),\n" + + " success: function(){ callback('Success!'); },\n" + + " failure: function(e){ callback(e.exception || e.responseText || 'Failed'); },\n" + + " jsonData: {\n" + + " 'id': " + pkgId + ",\n" + + " 'testIdNumberStart': " + superPkgIdStart + ",\n" + + " 'description': '" + jsEscape(description) + "',\n" + + " 'active': true,\n" + + " 'repeatable': true,\n" + + " 'narrative': '" + jsEscape(narrative) + "',\n" + + " 'categories': [" + categoryId + "],\n" + + " 'subPackages': [],\n" + + " 'extraFields': [],\n" + + " 'attributes': " + attributesToJson(attributes) + "\n" + + " }\n" + + "});"; + } + + private String attributesToJson(List> attributes) + { + List values = new ArrayList<>(); + for (Map attribute : attributes) + { + values.add("{" + + "'name': '" + jsEscape(String.valueOf(attribute.get("name"))) + "', " + + "'label': '" + jsEscape(String.valueOf(attribute.get("label"))) + "', " + + "'rangeURI': '" + jsEscape(String.valueOf(attribute.get("rangeURI"))) + "', " + + "'required': " + attribute.get("required") + + "}"); + } + + return "[" + String.join(", ", values) + "]"; + } + + private String jsEscape(String value) + { + return value.replace("\\", "\\\\").replace("'", "\\'"); + } + protected void initGenetics() { beginAt(WebTestHelper.buildURL("ehr", getProjectName(), "doGeneticCalculations"));