diff --git a/research-reproducibility-intelligence/README.md b/research-reproducibility-intelligence/README.md new file mode 100644 index 0000000..bcf588e --- /dev/null +++ b/research-reproducibility-intelligence/README.md @@ -0,0 +1,75 @@ +# Research Reproducibility Intelligence + +Self-contained AI-powered research assistant suite milestone for [SCIBASE.AI issue #16](https://github.com/SCIBASE-AI/SCIBASE.AI/issues/16). + +This module turns a scientific project packet into a deterministic assistant report covering auto peer-review, reproducibility checks, and research-gap discovery. It is designed for easy review: no API keys, no external services, no package install required beyond Node.js. + +## What It Adds + +- Auto peer-review report with manuscript structure checks, clarity findings, statistical reporting signals, and claim-to-evidence alignment. +- Reproducibility report with artifact fingerprinting, pinned dependency checks, runnable file detection, data dictionary/source data checks, and a runbook. +- Sandbox execution plan and evidence validation for notebooks/scripts, including disabled-network policy, resource limits, output hashes, execution logs, and reported-output consistency checks. +- Linked prior reproducibility attempts, sorted by current artifact fingerprint match and recency. +- Research-gap feed that ranks corpus items by relevance, replication gap, limitation signals, and novelty. +- Workflow orchestration that converts findings into staged, owner-assigned, evidence-hashed actions. +- Combined assistant packet with readiness score, orchestration status, and next-action queue. +- Sample project fixture, tests, requirement mapping, CLI demo, and short demo GIF. + +## Run + +```bash +cd research-reproducibility-intelligence +npm run check +npm test +npm run demo +``` + +Expected demo shape: + +```json +{ + "project": "Longitudinal microbiome shifts after coastal flooding", + "readinessScore": 71, + "peerReviewScore": 62, + "reproducibilityStatus": "reproducible", + "sandboxSummary": { + "plannedRuns": 1, + "observedRuns": 1, + "cleanRuns": 1, + "consistentOutputs": 1, + "missingRuns": 0 + }, + "workflowBlocked": true, + "workflowRiskScore": 88, + "topWorkflowAction": { + "id": "review-1", + "stage": "peer-review", + "status": "blocking", + "owner": "Dr. Chen" + }, + "linkedAttempt": { + "id": "attempt-2026-04-dry-run", + "matchesCurrentArtifacts": true + }, + "topResearchGap": { + "paperId": "paper-1", + "priority": 0.8 + } +} +``` + +## Demo Artifact + +See [docs/demo.gif](docs/demo.gif) for a short visual walkthrough. The SVG source is included at [docs/demo.svg](docs/demo.svg). + +## Files + +- `src/research-assistant.js` - deterministic assistant engine. +- `data/sample-project.json` - reviewable project/corpus fixture. +- `test/research-assistant.test.js` - Node assertion tests. +- `scripts/demo.js` - CLI demo. +- `docs/issue-16-requirement-map.md` - maps implementation to bounty requirements. + +## AI-Assisted Disclosure + +This contribution was produced with AI assistance and manually verified with the local commands above. diff --git a/research-reproducibility-intelligence/data/sample-project.json b/research-reproducibility-intelligence/data/sample-project.json new file mode 100644 index 0000000..37dc2ab --- /dev/null +++ b/research-reproducibility-intelligence/data/sample-project.json @@ -0,0 +1,110 @@ +{ + "id": "proj-microbiome-2026", + "title": "Longitudinal microbiome shifts after coastal flooding", + "domain": "environmental health", + "interests": ["microbiome", "coastal flooding", "public health", "replication"], + "manuscript": { + "abstract": "We study microbiome shifts after coastal flooding using repeated household samples. The analysis suggests water intrusion changes bacterial diversity and may affect respiratory outcomes.", + "methods": "Households were sampled monthly. DNA sequencing was performed and results were normalized by read depth. The current manuscript clearly shows that flooding changes all exposed communities in a way that is obviously relevant to public health but the text still needs more direct evidence links.", + "results": "Observed diversity increased in exposed homes and returned toward baseline after remediation. The estimated effect size was larger in homes with persistent moisture.", + "limitations": "The pilot cohort is small and future work should replicate findings across more regions." + }, + "claims": [ + { + "id": "claim-1", + "text": "Flood exposure increases microbial diversity in sampled homes." + }, + { + "id": "claim-2", + "text": "Hospital admissions declined after a regional policy intervention." + } + ], + "results": [ + { + "id": "result-1", + "summary": "Microbial diversity increased in exposed homes after flooding and moved toward baseline after remediation." + }, + { + "id": "result-2", + "summary": "Persistent moisture was associated with a larger estimated effect size." + } + ], + "files": [ + { "name": "README.md" }, + { "name": "analysis.ipynb" }, + { "name": "data_dictionary.md" }, + { "name": "samples.csv" }, + { "name": "requirements.txt" } + ], + "dependencies": [ + { "name": "python", "version": "3.12.3" }, + { "name": "pandas", "version": "2.2.2" }, + { "name": "scipy", "version": "1.13.0" } + ], + "sandboxPolicy": { + "image": "python:3.12-slim", + "network": "disabled", + "cpu": "2", + "memory": "4g", + "timeoutSeconds": 1800 + }, + "executionEvidence": [ + { + "targetId": "run-13846284", + "exitCode": 0, + "durationSeconds": 142, + "outputHashes": ["sha256:diversity-summary-2026", "sha256:moisture-effect-2026"], + "logUrl": "https://example.org/reproducibility/proj-microbiome-2026/logs/analysis-ipynb", + "generatedArtifacts": ["diversity-summary.csv", "moisture-effect.json"], + "reportedArtifacts": ["diversity-summary.csv", "moisture-effect.json"] + } + ], + "workflowPreferences": { + "reviewOwner": "Dr. Chen", + "reproducibilityOwner": "Replication Desk", + "strategyOwner": "Research Strategy", + "publicationOwner": "Corresponding Author", + "deadline": "2026-05-28" + }, + "reproducibilityAttempts": [ + { + "id": "attempt-2026-04-dry-run", + "date": "2026-04-18", + "status": "successful", + "url": "https://example.org/reproducibility/proj-microbiome-2026/attempt-2026-04-dry-run", + "artifactFingerprint": "d584da823d10eaa1", + "notes": "Clean-room rerun reproduced reported diversity trends and stored execution logs." + }, + { + "id": "attempt-2026-03-pilot", + "date": "2026-03-12", + "status": "partial", + "url": "https://example.org/reproducibility/proj-microbiome-2026/attempt-2026-03-pilot", + "artifactFingerprint": "legacy-artifacts", + "notes": "Earlier attempt predated dependency pinning and did not include all source data." + } + ], + "corpus": [ + { + "id": "paper-1", + "title": "Microbial exposure after hurricane flooding", + "abstract": "A pilot study of household microbial exposure after hurricane flooding. Limitations include small sample size and unresolved replication across regions.", + "keywords": ["microbiome", "flooding", "indoor exposure"], + "replications": 0 + }, + { + "id": "paper-2", + "title": "Respiratory health after household remediation", + "abstract": "Remediation studies report mixed respiratory health outcomes and call for future work that links environmental sampling to clinical endpoints.", + "keywords": ["respiratory", "remediation", "public health"], + "replications": 1 + }, + { + "id": "paper-3", + "title": "Open protocols for environmental sequencing", + "abstract": "Protocol standardization improves reproducibility in environmental sequencing projects.", + "keywords": ["sequencing", "reproducibility", "protocol"], + "replications": 4 + } + ] +} diff --git a/research-reproducibility-intelligence/docs/demo.gif b/research-reproducibility-intelligence/docs/demo.gif new file mode 100644 index 0000000..a33cd47 Binary files /dev/null and b/research-reproducibility-intelligence/docs/demo.gif differ diff --git a/research-reproducibility-intelligence/docs/demo.mp4 b/research-reproducibility-intelligence/docs/demo.mp4 new file mode 100644 index 0000000..0556417 Binary files /dev/null and b/research-reproducibility-intelligence/docs/demo.mp4 differ diff --git a/research-reproducibility-intelligence/docs/demo.svg b/research-reproducibility-intelligence/docs/demo.svg new file mode 100644 index 0000000..ed186f3 --- /dev/null +++ b/research-reproducibility-intelligence/docs/demo.svg @@ -0,0 +1,34 @@ + + Research Reproducibility Intelligence Demo + Visual demo for the peer review, reproducibility, and research gap assistant packet. + + + AI Research Assistant Suite + Peer review · reproducibility · research gap discovery + + Peer review score + 62 + claim evidence needed + + Reproducibility + 1.00 + runbook generated + + Top gap + Microbial exposure + after hurricane flooding + + Assistant packet + { readinessScore, peerReview, reproducibility, researchGaps, nextActions } + Top action: link unsupported respiratory outcome claim to evidence. + diff --git a/research-reproducibility-intelligence/docs/issue-16-requirement-map.md b/research-reproducibility-intelligence/docs/issue-16-requirement-map.md new file mode 100644 index 0000000..8e89ce0 --- /dev/null +++ b/research-reproducibility-intelligence/docs/issue-16-requirement-map.md @@ -0,0 +1,20 @@ +# Issue #16 Requirement Map + +This module is a deterministic milestone for SCIBASE issue #16, AI-Powered Research Assistant Suite. It provides local, reviewable implementations of the three requested capabilities without external APIs or model keys. + +| Issue requirement | Implementation | +| --- | --- | +| Auto peer review reports | `buildPeerReviewReport()` checks manuscript structure, clarity risks, statistical reporting signals, and claim-to-evidence alignment. | +| Reproducibility checker | `buildReproducibilityReport()` evaluates README/runbook presence, pinned dependencies, runnable artifacts, data dictionary, source data, sandbox execution plans, sandbox run evidence, reported-output consistency, and produces a reproducibility runbook. | +| Auto-executes project code and notebooks in sandbox environments | `buildSandboxExecutionPlan()` creates disabled-network sandbox targets for notebooks/scripts with resource limits, read-only project mounts, writable output mounts, expected result IDs, and executable commands. `evaluateSandboxEvidence()` validates exit codes, output hashes, log URLs, generated artifacts, reported artifacts, clean-run counts, and missing-run counts. | +| Research gap finder | `buildResearchGapFeed()` scans a corpus fixture for relevance, low replication count, limitation language, and novelty signals, then ranks opportunities. | +| Real-time insights and workflow automation | `buildWorkflowOrchestration()` turns review, reproducibility, and gap signals into staged actions with owners, blockers, dependencies, risk score, deadline, and evidence hashes. | +| Integrated assistant packet | `buildAssistantPacket()` combines review, reproducibility, gap discovery, workflow orchestration, one readiness score, and a prioritized action queue. | +| Reviewer-friendly local demo | `npm run demo` prints the assistant packet summary for `data/sample-project.json`. | +| Verification | `npm run check` and `npm test` run with Node built-ins only. | + +## Design Notes + +- The scoring is transparent and deterministic, which makes it suitable for review and regression testing. +- The module is isolated under `research-reproducibility-intelligence/`. +- It is intentionally credential-free. A future production integration can replace deterministic scoring with model-backed adapters while keeping the same report shape. diff --git a/research-reproducibility-intelligence/package.json b/research-reproducibility-intelligence/package.json new file mode 100644 index 0000000..bc6bed8 --- /dev/null +++ b/research-reproducibility-intelligence/package.json @@ -0,0 +1,12 @@ +{ + "name": "scibase-research-reproducibility-intelligence", + "version": "0.1.0", + "private": true, + "description": "Deterministic AI-powered research assistant suite milestone for SCIBASE issue #16.", + "type": "commonjs", + "scripts": { + "check": "node --check src/research-assistant.js && node --check scripts/demo.js && node --check test/research-assistant.test.js", + "demo": "node scripts/demo.js", + "test": "node test/research-assistant.test.js" + } +} diff --git a/research-reproducibility-intelligence/scripts/demo.js b/research-reproducibility-intelligence/scripts/demo.js new file mode 100644 index 0000000..de747d3 --- /dev/null +++ b/research-reproducibility-intelligence/scripts/demo.js @@ -0,0 +1,27 @@ +"use strict"; + +const sampleProject = require("../data/sample-project.json"); +const { buildAssistantPacket } = require("../src/research-assistant"); + +const packet = buildAssistantPacket(sampleProject); + +console.log( + JSON.stringify( + { + project: packet.project.title, + readinessScore: packet.readinessScore, + peerReviewScore: packet.peerReview.score, + reproducibilityStatus: packet.reproducibility.status, + sandboxTarget: packet.reproducibility.sandboxPlan.targets[0], + sandboxSummary: packet.reproducibility.sandboxEvidence.summary, + workflowBlocked: packet.workflow.blocked, + workflowRiskScore: packet.workflow.riskScore, + topWorkflowAction: packet.workflow.actions[0], + linkedAttempt: packet.reproducibility.linkedAttempts[0], + topResearchGap: packet.researchGaps[0], + nextActions: packet.nextActions.slice(0, 5), + }, + null, + 2, + ), +); diff --git a/research-reproducibility-intelligence/src/research-assistant.js b/research-reproducibility-intelligence/src/research-assistant.js new file mode 100644 index 0000000..6a30807 --- /dev/null +++ b/research-reproducibility-intelligence/src/research-assistant.js @@ -0,0 +1,624 @@ +"use strict"; + +const crypto = require("crypto"); + +const DEFAULT_REVIEW_TEMPLATE = { + requiredSections: ["abstract", "methods", "results", "limitations", "data availability"], + statisticalTerms: ["p-value", "confidence interval", "effect size", "sample size"], + reproducibilityFiles: ["README", "requirements", "notebook", "data dictionary"], +}; + +function asArray(value) { + return Array.isArray(value) ? value : []; +} + +function normalizeText(value) { + return String(value || "").replace(/\s+/g, " ").trim(); +} + +function sentenceSplit(text) { + return normalizeText(text) + .split(/(?<=[.!?])\s+/) + .map((sentence) => sentence.trim()) + .filter(Boolean); +} + +function wordSet(text) { + return new Set( + normalizeText(text) + .toLowerCase() + .replace(/[^a-z0-9\s-]/g, " ") + .split(/\s+/) + .filter((word) => word.length > 3), + ); +} + +function overlapScore(left, right) { + const a = wordSet(left); + const b = wordSet(right); + if (a.size === 0 || b.size === 0) return 0; + let overlap = 0; + for (const word of a) { + if (b.has(word)) overlap += 1; + } + return Number((overlap / Math.min(a.size, b.size)).toFixed(4)); +} + +function fingerprint(value) { + return crypto.createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16); +} + +function normalizeProject(project) { + if (!project || typeof project !== "object") { + throw new TypeError("project must be an object"); + } + + return { + id: project.id || "project-unknown", + title: project.title || "Untitled research project", + domain: project.domain || "general science", + manuscript: project.manuscript || {}, + files: asArray(project.files), + dependencies: asArray(project.dependencies), + claims: asArray(project.claims), + results: asArray(project.results), + corpus: asArray(project.corpus), + interests: asArray(project.interests), + executionEvidence: asArray(project.executionEvidence), + executionTargets: asArray(project.executionTargets), + reproducibilityAttempts: asArray(project.reproducibilityAttempts), + workflowPreferences: project.workflowPreferences || {}, + sandboxPolicy: project.sandboxPolicy || {}, + }; +} + +function buildPeerReviewReport(projectInput, template = DEFAULT_REVIEW_TEMPLATE) { + const project = normalizeProject(projectInput); + const manuscriptText = Object.values(project.manuscript).join(" "); + const sections = Object.keys(project.manuscript).map((section) => section.toLowerCase()); + const missingSections = template.requiredSections.filter( + (section) => !sections.includes(section.toLowerCase()), + ); + + const clarityFindings = sentenceSplit(manuscriptText) + .filter((sentence) => sentence.length > 180 || /\b(obviously|clearly|simply)\b/i.test(sentence)) + .map((sentence) => ({ + type: "clarity", + severity: sentence.length > 220 ? "high" : "medium", + evidence: sentence.slice(0, 220), + suggestion: "Split dense claims into shorter evidence-backed statements.", + })); + + const claimFindings = project.claims.map((claim) => { + const bestResult = project.results + .map((result) => ({ + resultId: result.id, + score: overlapScore(claim.text, result.summary || result.text || ""), + summary: result.summary || result.text || "", + })) + .sort((a, b) => b.score - a.score)[0]; + + return { + claimId: claim.id, + claim: claim.text, + evidenceResultId: bestResult ? bestResult.resultId : null, + evidenceScore: bestResult ? bestResult.score : 0, + status: bestResult && bestResult.score >= 0.2 ? "supported" : "needs-evidence", + }; + }); + + const statisticMentions = template.statisticalTerms.filter((term) => + manuscriptText.toLowerCase().includes(term.toLowerCase()), + ); + + const findings = [ + ...missingSections.map((section) => ({ + type: "structure", + severity: "high", + evidence: section, + suggestion: `Add a ${section} section before submission.`, + })), + ...clarityFindings, + ...claimFindings + .filter((claim) => claim.status === "needs-evidence") + .map((claim) => ({ + type: "claim-evidence", + severity: "high", + evidence: claim.claim, + suggestion: "Link this claim to a result, table, figure, or cited prior work.", + })), + ]; + + if (statisticMentions.length === 0) { + findings.push({ + type: "statistics", + severity: "medium", + evidence: "No statistical reporting terms found.", + suggestion: "Report sample size, effect size, confidence intervals, and p-values where relevant.", + }); + } + + const score = Math.max(0, 100 - findings.reduce((sum, finding) => { + return sum + (finding.severity === "high" ? 15 : 8); + }, 0)); + + return { + projectId: project.id, + template: "general-scientific-pre-review", + score, + missingSections, + statisticMentions, + claimAlignment: claimFindings, + findings, + }; +} + +function buildReproducibilityReport(projectInput, template = DEFAULT_REVIEW_TEMPLATE) { + const project = normalizeProject(projectInput); + const fileNames = project.files.map((file) => + String(file.name || "") + .toLowerCase() + .replace(/[_-]+/g, " "), + ); + const dependencyNames = project.dependencies.map((dependency) => dependency.name || dependency); + const missingFiles = template.reproducibilityFiles.filter((required) => + !fileNames.some((fileName) => fileName.includes(required.toLowerCase())), + ); + const runnableFiles = fileNames.filter((fileName) => + /\.(ipynb|py|r|jl|sh|Rmd)$/i.test(fileName), + ); + const dataFiles = fileNames.filter((fileName) => + /\.(csv|tsv|parquet|json|h5|feather)$/i.test(fileName), + ); + const pinnedDependencies = project.dependencies.filter( + (dependency) => dependency.version && !["latest", "*"].includes(String(dependency.version)), + ); + const artifactFingerprint = fingerprint({ + files: project.files, + dependencies: project.dependencies, + results: project.results, + }); + const linkedAttempts = project.reproducibilityAttempts + .map((attempt) => ({ + id: attempt.id || "attempt-unknown", + url: attempt.url || null, + status: attempt.status || "unknown", + date: attempt.date || null, + artifactFingerprint: attempt.artifactFingerprint || null, + matchesCurrentArtifacts: attempt.artifactFingerprint === artifactFingerprint, + notes: attempt.notes || "", + })) + .sort((a, b) => { + if (a.matchesCurrentArtifacts !== b.matchesCurrentArtifacts) { + return a.matchesCurrentArtifacts ? -1 : 1; + } + return String(b.date || "").localeCompare(String(a.date || "")); + }); + const sandboxPlan = buildSandboxExecutionPlan(project, runnableFiles, dataFiles, artifactFingerprint); + const sandboxEvidence = evaluateSandboxEvidence(project, sandboxPlan); + + const checks = [ + { + id: "readme-present", + passed: !missingFiles.includes("README"), + detail: "README or runbook present", + }, + { + id: "environment-pinned", + passed: dependencyNames.length > 0 && pinnedDependencies.length === dependencyNames.length, + detail: "Dependencies include explicit versions", + }, + { + id: "runnable-analysis", + passed: runnableFiles.length > 0, + detail: "At least one runnable analysis artifact exists", + }, + { + id: "data-dictionary", + passed: !missingFiles.includes("data dictionary"), + detail: "Data dictionary is present", + }, + { + id: "source-data", + passed: dataFiles.length > 0, + detail: "Machine-readable source data is present", + }, + { + id: "attempt-history-linked", + passed: linkedAttempts.length > 0, + detail: "Previous reproducibility attempts are linked", + }, + { + id: "sandbox-plan-ready", + passed: sandboxPlan.targets.length > 0, + detail: "Sandbox execution targets are defined for runnable artifacts", + }, + { + id: "sandbox-evidence-clean", + passed: sandboxEvidence.summary.cleanRuns === sandboxPlan.targets.length && sandboxPlan.targets.length > 0, + detail: "Sandbox execution evidence has zero failed runs", + }, + { + id: "reported-output-consistency", + passed: sandboxEvidence.summary.consistentOutputs === sandboxPlan.targets.length && sandboxPlan.targets.length > 0, + detail: "Sandbox evidence links generated outputs to reported results", + }, + ]; + + const passed = checks.filter((check) => check.passed).length; + const confidenceScore = Number((passed / checks.length).toFixed(2)); + + return { + projectId: project.id, + artifactFingerprint, + confidenceScore, + status: confidenceScore >= 0.8 ? "reproducible" : confidenceScore >= 0.5 ? "partial" : "at-risk", + missingFiles, + runnableFiles, + dataFiles, + sandboxPlan, + sandboxEvidence, + linkedAttempts, + checks, + runbook: [ + "Install pinned dependencies in a clean environment.", + "Run notebooks or scripts in the order documented by the README.", + "Compare generated outputs against the reported results manifest.", + "Attach logs and hashes to the reproducibility attempt record.", + ], + }; +} + +function buildSandboxExecutionPlan(projectInput, runnableFilesInput, dataFilesInput, artifactFingerprint) { + const project = normalizeProject(projectInput); + const runnableFiles = runnableFilesInput || project.files.map((file) => file.name || "").filter(Boolean); + const dataFiles = dataFilesInput || []; + const policy = project.sandboxPolicy; + const defaultImage = policy.image || "python:3.12-slim"; + const network = policy.network || "disabled"; + const resourceLimits = { + cpu: policy.cpu || "2", + memory: policy.memory || "4g", + timeoutSeconds: policy.timeoutSeconds || 1800, + }; + const explicitTargets = project.executionTargets.map((target) => String(target)); + const targets = (explicitTargets.length > 0 ? explicitTargets : runnableFiles) + .map((target) => String(target)) + .filter((target) => executableCommandFor(target)) + .map((target) => ({ + id: `run-${fingerprint({ projectId: project.id, target }).slice(0, 8)}`, + target, + command: executableCommandFor(target), + requiredDataFiles: dataFiles, + expectedResultIds: project.results.map((result) => result.id), + })); + + return { + image: defaultImage, + network, + resourceLimits, + artifactFingerprint, + mounts: [ + { source: "project", target: "/workspace/project", mode: "read-only" }, + { source: "outputs", target: "/workspace/outputs", mode: "read-write" }, + ], + targets, + }; +} + +function executableCommandFor(fileName) { + if (/\.ipynb$/i.test(fileName)) { + return `jupyter nbconvert --to notebook --execute ${fileName} --output /workspace/outputs/${fileName}`; + } + if (/\.py$/i.test(fileName)) return `python ${fileName}`; + if (/\.r$/i.test(fileName)) return `Rscript ${fileName}`; + if (/\.rmd$/i.test(fileName)) return `Rscript -e "rmarkdown::render('${fileName}')"`; + if (/\.jl$/i.test(fileName)) return `julia ${fileName}`; + if (/\.sh$/i.test(fileName)) return `bash ${fileName}`; + return null; +} + +function evaluateSandboxEvidence(projectInput, sandboxPlanInput) { + const project = normalizeProject(projectInput); + const sandboxPlan = sandboxPlanInput || buildSandboxExecutionPlan(project); + const targetIds = new Set(sandboxPlan.targets.map((target) => target.id)); + const runs = project.executionEvidence + .filter((run) => targetIds.has(run.targetId)) + .map((run) => { + const outputHashes = asArray(run.outputHashes).filter(Boolean); + const generatedArtifacts = asArray(run.generatedArtifacts).map(String); + const reportedArtifacts = asArray(run.reportedArtifacts).map(String); + const missingReportedArtifacts = reportedArtifacts.filter( + (artifact) => !generatedArtifacts.includes(artifact), + ); + return { + targetId: run.targetId, + status: run.exitCode === 0 && outputHashes.length > 0 ? "passed" : "failed", + exitCode: Number.isInteger(run.exitCode) ? run.exitCode : null, + durationSeconds: run.durationSeconds || null, + outputHashes, + logUrl: run.logUrl || null, + generatedArtifacts, + reportedArtifacts, + missingReportedArtifacts, + outputConsistency: missingReportedArtifacts.length === 0 && reportedArtifacts.length > 0, + }; + }); + const cleanRuns = runs.filter((run) => run.status === "passed").length; + const consistentOutputs = runs.filter((run) => run.outputConsistency).length; + + return { + runs, + summary: { + plannedRuns: sandboxPlan.targets.length, + observedRuns: runs.length, + cleanRuns, + consistentOutputs, + missingRuns: Math.max(0, sandboxPlan.targets.length - runs.length), + }, + }; +} + +function buildResearchGapFeed(projectInput) { + const project = normalizeProject(projectInput); + const currentText = [ + project.title, + project.domain, + ...project.interests, + ...project.claims.map((claim) => claim.text), + ].join(" "); + + const opportunities = project.corpus.map((paper) => { + const relevance = overlapScore(currentText, [paper.title, paper.abstract, ...(paper.keywords || [])].join(" ")); + const replicationGap = paper.replications === 0 ? 0.35 : paper.replications < 2 ? 0.2 : 0; + const limitationSignal = /\b(limitation|future work|unresolved|small sample|pilot)\b/i.test( + `${paper.abstract || ""} ${paper.notes || ""}`, + ) + ? 0.25 + : 0; + const novelty = Math.max(0, 1 - overlapScore(project.title, paper.title || "")); + const priority = Number((relevance * 0.45 + replicationGap + limitationSignal + novelty * 0.1).toFixed(4)); + + return { + paperId: paper.id, + title: paper.title, + relevance, + replicationGap, + limitationSignal, + priority, + suggestedDirection: buildSuggestedDirection(project, paper), + }; + }); + + return opportunities + .filter((opportunity) => opportunity.priority >= 0.25) + .sort((a, b) => b.priority - a.priority) + .slice(0, 5); +} + +function buildSuggestedDirection(project, paper) { + const projectTerms = [...wordSet(`${project.title} ${project.domain} ${project.interests.join(" ")}`)]; + const paperTerms = [...wordSet(`${paper.title || ""} ${paper.abstract || ""}`)]; + const shared = projectTerms.filter((term) => paperTerms.includes(term)).slice(0, 3); + const anchor = shared.length > 0 ? shared.join(", ") : project.domain; + return `Investigate ${anchor} with a replication-ready protocol and explicitly report negative or null results.`; +} + +function severityWeight(severity) { + return severity === "high" ? 3 : severity === "medium" ? 2 : 1; +} + +function actionHash(projectId, action) { + return fingerprint({ + projectId, + id: action.id, + source: action.source, + evidence: action.evidence, + recommendation: action.recommendation, + }); +} + +function buildAction(project, action) { + return { + ...action, + evidenceHash: actionHash(project.id, action), + }; +} + +function workflowOwner(project, ownerKey, fallback) { + return project.workflowPreferences[ownerKey] || fallback; +} + +function uniqueStrings(values) { + return [...new Set(values)]; +} + +function buildWorkflowOrchestration(projectInput) { + const project = normalizeProject(projectInput); + const peerReview = buildPeerReviewReport(project); + const reproducibility = buildReproducibilityReport(project); + const researchGaps = buildResearchGapFeed(project); + + const reviewActions = peerReview.findings.map((finding, index) => + buildAction(project, { + id: `review-${index + 1}`, + stage: "peer-review", + source: finding.type, + title: `Resolve ${finding.type} finding`, + owner: workflowOwner(project, "reviewOwner", "editorial lead"), + severity: finding.severity, + priority: severityWeight(finding.severity) * 20, + status: finding.severity === "high" ? "blocking" : "queued", + evidence: finding.evidence, + recommendation: finding.suggestion, + dependsOn: [], + }), + ); + + const failedReproChecks = reproducibility.checks.filter((check) => !check.passed); + const reproActions = [ + ...failedReproChecks.map((check, index) => + buildAction(project, { + id: `repro-${index + 1}`, + stage: "reproducibility", + source: check.id, + title: `Fix reproducibility gate: ${check.id}`, + owner: workflowOwner(project, "reproducibilityOwner", "reproducibility reviewer"), + severity: "high", + priority: 55, + status: "blocking", + evidence: check.detail, + recommendation: `Complete and re-run the ${check.id} check.`, + dependsOn: reviewActions.filter((action) => action.status === "blocking").map((action) => action.id), + }), + ), + buildAction(project, { + id: "repro-runbook", + stage: "reproducibility", + source: "runbook", + title: "Attach clean-room rerun evidence", + owner: workflowOwner(project, "reproducibilityOwner", "reproducibility reviewer"), + severity: reproducibility.status === "reproducible" ? "low" : "medium", + priority: reproducibility.status === "reproducible" ? 18 : 38, + status: reproducibility.status === "reproducible" ? "ready" : "queued", + evidence: reproducibility.linkedAttempts[0] + ? `${reproducibility.linkedAttempts[0].id}:${reproducibility.linkedAttempts[0].status}` + : "no linked attempt", + recommendation: reproducibility.runbook.join(" "), + dependsOn: failedReproChecks.map((_, index) => `repro-${index + 1}`), + }), + ]; + + const gapActions = researchGaps.slice(0, 3).map((gap, index) => + buildAction(project, { + id: `gap-${index + 1}`, + stage: "gap-finder", + source: gap.paperId, + title: `Evaluate research opportunity: ${gap.title}`, + owner: workflowOwner(project, "strategyOwner", "research strategy lead"), + severity: gap.priority >= 0.75 ? "medium" : "low", + priority: Math.round(gap.priority * 50), + status: "queued", + evidence: `${gap.paperId}:${gap.priority}`, + recommendation: gap.suggestedDirection, + dependsOn: ["repro-runbook"], + }), + ); + + const publicationAction = buildAction(project, { + id: "publication-readiness", + stage: "publication-readiness", + source: "assistant-packet", + title: "Prepare reviewer handoff packet", + owner: workflowOwner(project, "publicationOwner", "corresponding author"), + severity: "medium", + priority: 34, + status: peerReview.findings.some((finding) => finding.severity === "high") ? "blocked" : "queued", + evidence: `peer:${peerReview.score};repro:${reproducibility.status};gaps:${researchGaps.length}`, + recommendation: "Bundle peer-review fixes, reproducibility evidence, and selected research-gap notes for review.", + dependsOn: [ + ...reviewActions.filter((action) => action.status === "blocking").map((action) => action.id), + "repro-runbook", + ], + }); + + const actions = [...reviewActions, ...reproActions, ...gapActions, publicationAction].sort((a, b) => { + if (a.status === "blocking" && b.status !== "blocking") return -1; + if (a.status !== "blocking" && b.status === "blocking") return 1; + return b.priority - a.priority; + }); + + const blocked = actions.some((action) => ["blocking", "blocked"].includes(action.status)); + const riskScore = Math.min( + 100, + actions.reduce((sum, action) => { + const statusPenalty = action.status === "blocking" ? 12 : action.status === "blocked" ? 8 : 0; + return sum + severityWeight(action.severity) * 4 + statusPenalty; + }, Math.round((1 - reproducibility.confidenceScore) * 25)), + ); + + return { + projectId: project.id, + blocked, + readyForInternalReview: !blocked && peerReview.score >= 70 && reproducibility.confidenceScore >= 0.8, + riskScore, + deadline: project.workflowPreferences.deadline || null, + stages: [ + summarizeStage("peer-review", actions), + summarizeStage("reproducibility", actions), + summarizeStage("gap-finder", actions), + summarizeStage("publication-readiness", actions), + ], + actions, + orchestrationHash: fingerprint({ + projectId: project.id, + actions: actions.map((action) => ({ + id: action.id, + status: action.status, + evidenceHash: action.evidenceHash, + dependsOn: action.dependsOn, + })), + riskScore, + }), + }; +} + +function summarizeStage(stage, actions) { + const stageActions = actions.filter((action) => action.stage === stage); + const blocked = stageActions.some((action) => ["blocking", "blocked"].includes(action.status)); + return { + stage, + actionCount: stageActions.length, + blocked, + topActionId: stageActions[0] ? stageActions[0].id : null, + }; +} + +function buildAssistantPacket(projectInput) { + const project = normalizeProject(projectInput); + const peerReview = buildPeerReviewReport(project); + const reproducibility = buildReproducibilityReport(project); + const researchGaps = buildResearchGapFeed(project); + const workflow = buildWorkflowOrchestration(project); + + const readinessScore = Math.round( + peerReview.score * 0.45 + reproducibility.confidenceScore * 100 * 0.35 + Math.min(researchGaps.length, 5) * 4, + ); + + return { + project: { + id: project.id, + title: project.title, + domain: project.domain, + }, + generatedAt: new Date().toISOString(), + readinessScore, + peerReview, + reproducibility, + researchGaps, + workflow, + nextActions: uniqueStrings([ + ...workflow.actions.slice(0, 3).map((action) => action.recommendation), + ...peerReview.findings.slice(0, 3).map((finding) => finding.suggestion), + ...reproducibility.checks + .filter((check) => !check.passed) + .slice(0, 2) + .map((check) => `Address reproducibility check: ${check.detail}.`), + ...researchGaps.slice(0, 2).map((gap) => gap.suggestedDirection), + ]), + }; +} + +module.exports = { + DEFAULT_REVIEW_TEMPLATE, + buildAssistantPacket, + buildSandboxExecutionPlan, + buildPeerReviewReport, + buildResearchGapFeed, + buildReproducibilityReport, + buildWorkflowOrchestration, + evaluateSandboxEvidence, + fingerprint, + normalizeProject, + overlapScore, +}; diff --git a/research-reproducibility-intelligence/test/research-assistant.test.js b/research-reproducibility-intelligence/test/research-assistant.test.js new file mode 100644 index 0000000..16cb30b --- /dev/null +++ b/research-reproducibility-intelligence/test/research-assistant.test.js @@ -0,0 +1,117 @@ +"use strict"; + +const assert = require("assert"); +const sampleProject = require("../data/sample-project.json"); +const { + buildAssistantPacket, + buildPeerReviewReport, + buildResearchGapFeed, + buildReproducibilityReport, + buildSandboxExecutionPlan, + buildWorkflowOrchestration, + evaluateSandboxEvidence, + overlapScore, +} = require("../src/research-assistant"); + +function testOverlapScore() { + assert.ok(overlapScore("microbiome flooding exposure", "flooding microbiome study") > 0.4); + assert.strictEqual(overlapScore("", "anything"), 0); +} + +function testPeerReviewReport() { + const report = buildPeerReviewReport(sampleProject); + + assert.strictEqual(report.projectId, "proj-microbiome-2026"); + assert.ok(report.score < 100); + assert.ok(report.missingSections.includes("data availability")); + assert.ok(report.findings.some((finding) => finding.type === "claim-evidence")); + assert.ok(report.claimAlignment.some((claim) => claim.status === "supported")); +} + +function testReproducibilityReport() { + const report = buildReproducibilityReport(sampleProject); + + assert.strictEqual(report.status, "reproducible"); + assert.strictEqual(report.confidenceScore, 1); + assert.ok(report.artifactFingerprint.length > 8); + assert.ok(report.runnableFiles.includes("analysis.ipynb")); + assert.strictEqual(report.sandboxPlan.network, "disabled"); + assert.strictEqual(report.sandboxPlan.targets[0].target, "analysis.ipynb"); + assert.strictEqual(report.sandboxEvidence.summary.cleanRuns, 1); + assert.strictEqual(report.sandboxEvidence.summary.consistentOutputs, 1); + assert.ok(report.checks.find((check) => check.id === "sandbox-evidence-clean").passed); + assert.ok(report.checks.find((check) => check.id === "reported-output-consistency").passed); + assert.strictEqual(report.linkedAttempts.length, 2); + assert.strictEqual(report.linkedAttempts[0].id, "attempt-2026-04-dry-run"); + assert.strictEqual(report.linkedAttempts[0].matchesCurrentArtifacts, true); + assert.ok(report.checks.find((check) => check.id === "attempt-history-linked").passed); +} + +function testSandboxExecutionContract() { + const plan = buildSandboxExecutionPlan(sampleProject); + const evidence = evaluateSandboxEvidence(sampleProject, plan); + + assert.strictEqual(plan.image, "python:3.12-slim"); + assert.strictEqual(plan.targets.length, 1); + assert.ok(plan.targets[0].command.includes("jupyter nbconvert")); + assert.deepStrictEqual(plan.targets[0].expectedResultIds, ["result-1", "result-2"]); + assert.strictEqual(evidence.runs[0].status, "passed"); + assert.deepStrictEqual(evidence.runs[0].missingReportedArtifacts, []); + assert.strictEqual(evidence.summary.missingRuns, 0); +} + +function testResearchGapFeed() { + const gaps = buildResearchGapFeed(sampleProject); + + assert.ok(gaps.length >= 2); + assert.strictEqual(gaps[0].paperId, "paper-1"); + assert.ok(gaps[0].suggestedDirection.includes("replication-ready")); +} + +function testWorkflowOrchestration() { + const workflow = buildWorkflowOrchestration(sampleProject); + + assert.strictEqual(workflow.projectId, "proj-microbiome-2026"); + assert.strictEqual(workflow.blocked, true); + assert.ok(workflow.riskScore > 0); + assert.ok(workflow.orchestrationHash.length > 8); + assert.ok(workflow.stages.some((stage) => stage.stage === "peer-review")); + assert.ok(workflow.stages.some((stage) => stage.stage === "reproducibility")); + assert.ok(workflow.stages.some((stage) => stage.stage === "gap-finder")); + + const topAction = workflow.actions[0]; + assert.strictEqual(topAction.status, "blocking"); + assert.ok(["structure", "claim-evidence"].includes(topAction.source)); + assert.strictEqual(topAction.owner, "Dr. Chen"); + assert.ok(topAction.evidenceHash.length > 8); + + const runbookAction = workflow.actions.find((action) => action.id === "repro-runbook"); + assert.strictEqual(runbookAction.status, "ready"); + assert.strictEqual(runbookAction.owner, "Replication Desk"); + assert.deepStrictEqual(runbookAction.dependsOn, []); + + const gapAction = workflow.actions.find((action) => action.stage === "gap-finder"); + assert.strictEqual(gapAction.owner, "Research Strategy"); + assert.deepStrictEqual(gapAction.dependsOn, ["repro-runbook"]); +} + +function testAssistantPacket() { + const packet = buildAssistantPacket(sampleProject); + + assert.strictEqual(packet.project.domain, "environmental health"); + assert.ok(packet.readinessScore > 0); + assert.strictEqual(packet.workflow.projectId, "proj-microbiome-2026"); + assert.ok(packet.workflow.orchestrationHash.length > 8); + assert.ok(packet.nextActions.length >= 4); + assert.ok(packet.researchGaps[0].priority >= packet.researchGaps.at(-1).priority); +} + +testOverlapScore(); +testPeerReviewReport(); +testReproducibilityReport(); +testSandboxExecutionContract(); +testResearchGapFeed(); +testWorkflowOrchestration(); +testAssistantPacket(); + +console.log("research-reproducibility-intelligence tests passed");