Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .dev.vars.example
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ BETTER_AUTH_URL="http://localhost:3000"
# Autumn usage analytics. When unset, usage tracking is skipped.
# AUTUMN_SECRET_KEY=""

# Web search and URL/PDF extraction (Firecrawl).
# Web search and URL extraction (Firecrawl).
# FIRECRAWL_API_KEY=""

# Agent observability export (The Context Company).
# TCC_API_KEY=""

# Document extraction (LlamaParse).
# LLAMA_CLOUD_API_KEY=""
# Hosted document extraction (FileRouter).
# FILEROUTER_API_KEY=""

# Direct workspace-upload completion claims. Use a separate random 32+ byte value.
# WORKSPACE_UPLOAD_TOKEN_SECRET=""
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ ThinkEx is a full-stack TypeScript app hosted on Cloudflare. The frontend is Rea
### AI, data, and product systems

- **AI SDK**, **Cloudflare Think**, **Cloudflare Sandbox**, **Cloudflare Shell**, and **Cloudflare Codemode**.
- **Better Auth**, **Drizzle ORM**, **Zod**, **PostHog**, **The Context Company**, **Autumn**, **Firecrawl**, and **LlamaCloud** integrations.
- **Better Auth**, **Drizzle ORM**, **Zod**, **PostHog**, **The Context Company**, **Autumn**, **Firecrawl**, and **FileRouter** integrations.
- **Streamdown**, **KaTeX**, **Shiki**, **PapaParse**, **dnd-kit**, **react-resizable-panels**, and **Zustand** for workspace interactions.

See [`package.json`](package.json), [`wrangler.jsonc`](wrangler.jsonc), and [`docs/ENVIRONMENT.md`](docs/ENVIRONMENT.md) for deeper implementation details.
Expand Down
147 changes: 0 additions & 147 deletions containers/liteparse/package-lock.json

This file was deleted.

8 changes: 0 additions & 8 deletions containers/liteparse/package.json

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ RUN apt-get update \

WORKDIR /app

COPY package.json package-lock.json ./
RUN npm ci --omit=dev && npm cache clean --force

COPY server.mjs ./

USER node
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
import { LiteParse } from "@llamaindex/liteparse";
import { execFile } from "node:child_process";
import { once } from "node:events";
import { createWriteStream } from "node:fs";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { createServer } from "node:http";
Expand All @@ -11,21 +9,12 @@ import { Transform } from "node:stream";
import { promisify } from "node:util";

const port = 8080;
const parser = new LiteParse({
extractLinks: true,
imageMode: "placeholder",
ocrEnabled: false,
outputFormat: "markdown",
quiet: true,
});
const parseTimeoutMs = 90_000;
const maxInputBytes = 100 * 1024 * 1024;
const execFileAsync = promisify(execFile);

createServer(async (request, response) => {
const startedAt = Date.now();
let inputBytes = 0;
let pageCount = 0;
let status = 500;
let errorType = null;
let errorMessage = null;
Expand All @@ -48,46 +37,26 @@ createServer(async (request, response) => {
return response.end(preview.bytes);
}

if (request.method !== "POST" || request.url !== "/parse/pdf") {
status = 404;
return sendJson(response, status, { error: "Not found." });
}

const bytes = await readPdfRequestBytes(request);
inputBytes = bytes.byteLength;
const result = await withTimeout(parser.parse(bytes), parseTimeoutMs);
pageCount = result.pages.length;
status = 200;
response.writeHead(status, { "content-type": "application/x-ndjson; charset=utf-8" });
for (const page of result.pages) {
if (
!response.write(
`${JSON.stringify({ markdown: page.markdown, pageNumber: page.pageNum })}\n`,
)
) {
await once(response, "drain");
}
}
return response.end();
status = 404;
return sendJson(response, status, { error: "Not found." });
} catch (error) {
errorType = error instanceof Error ? error.name : "UnknownError";
errorMessage = error instanceof Error ? error.message : String(error);
if (error instanceof PdfValidationError) {
status = error.status;
return sendJson(response, status, { code: error.code, error: error.message });
}
return sendJson(response, status, { error: "PDF parsing failed." });
return sendJson(response, status, { error: "File preview failed." });
} finally {
console.info(
JSON.stringify({
duration_ms: Date.now() - startedAt,
error_type: errorType,
error_message: errorMessage,
event: "liteparse_request",
event: "workspace_file_preview_request",
input_bytes: inputBytes,
method: request.method,
outcome: status < 400 ? "success" : "error",
page_count: pageCount,
path: request.url,
status,
}),
Expand All @@ -113,14 +82,6 @@ async function validatePdfFile(filePath) {
}
}

async function readPdfRequestBytes(request) {
return withRequestFile(
request,
"thinkex-parse-",
async ({ filePath }) => new Uint8Array(await readFile(filePath)),
);
}

async function generatePreviewRequest(request, kind) {
return withRequestFile(request, "thinkex-preview-", async ({ filePath, sizeBytes, tempDir }) => {
if (kind === "pdf") {
Expand Down Expand Up @@ -202,16 +163,3 @@ function sendJson(response, status, body) {
response.writeHead(status, { "content-type": "application/json" });
response.end(JSON.stringify(body));
}

function withTimeout(promise, timeoutMs) {
let timeout;
const timeoutPromise = new Promise((_, reject) => {
timeout = setTimeout(() => {
reject(new Error(`LiteParse parsing timed out after ${timeoutMs}ms.`));
}, timeoutMs);
});

return Promise.race([promise, timeoutPromise]).finally(() => {
clearTimeout(timeout);
});
}
4 changes: 2 additions & 2 deletions docs/ENVIRONMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ The dev server boots and lets you sign in with just the two variables below. Eve
| `GOOGLE_CLIENT_ID` / `GOOGLE_CLIENT_SECRET` | Google sign-in | Google sign-in fails; use **Continue as guest** (shown in local dev) |
| `AI_GATEWAY_API_KEY` | AI chat / title generation (Vercel AI Gateway) | AI calls error when invoked |
| `AUTUMN_SECRET_KEY` | Autumn usage analytics for completed AI chat messages | Usage tracking is skipped |
| `FIRECRAWL_API_KEY` | Web search + URL/PDF extraction (Firecrawl) | Those calls error when invoked |
| `FIRECRAWL_API_KEY` | Web search and URL extraction (Firecrawl) | Those calls error when invoked |
| `TCC_API_KEY` | Agent observability export (The Context Company) | AI chat still works; TCC export is skipped |
| `LLAMA_CLOUD_API_KEY` | Document extraction (LlamaParse) | Those calls error when invoked |
| `FILEROUTER_API_KEY` | Hosted document extraction (FileRouter) | File extraction errors when invoked |

## Signing in without Google

Expand Down
2 changes: 1 addition & 1 deletion docs/concepts/items.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,4 @@ Upload limits are currently 50 files or 100 MB per selection.

## Extraction and Previews

PDFs and images are routed through extraction and preview pipelines so the AI can read them and the UI can render usable previews. The current stack uses Cloudflare Workflows, R2, Workers AI, LlamaParse, and conversion containers depending on the file type and configured secrets.
PDFs and images are routed through extraction and preview pipelines so the AI can read them and the UI can render usable previews. The current stack uses FileRouter, Cloudflare Workflows, R2, Workers AI, and conversion containers depending on the file type and configured secrets.
4 changes: 2 additions & 2 deletions docs/configuration/architecture.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ The public landing page and product app are currently served by the same Worker.
| --- | --- |
| Better Auth | Sessions, Google sign-in, local guest sign-in |
| AI Gateway / AI SDK | Chat models and title generation |
| Firecrawl | Public web search and URL/PDF extraction |
| LlamaCloud | Document extraction |
| Firecrawl | Public web search and URL extraction |
| FileRouter | Hosted document extraction with LiteParse and LlamaParse |
| The Context Company | Agent observability export |
| Autumn | AI usage analytics |
| PostHog | Product analytics, feedback, and diagnostics |
Expand Down
4 changes: 2 additions & 2 deletions docs/configuration/environment.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ ThinkEx reads runtime configuration from Cloudflare bindings, environment variab
| `GOOGLE_CLIENT_ID` / `GOOGLE_CLIENT_SECRET` | Google sign-in |
| `AI_GATEWAY_API_KEY` | AI chat and title generation |
| `AUTUMN_SECRET_KEY` | AI usage analytics |
| `FIRECRAWL_API_KEY` | Web search and URL/PDF extraction |
| `FIRECRAWL_API_KEY` | Web search and URL extraction |
| `TCC_API_KEY` | Agent observability export |
| `LLAMA_CLOUD_API_KEY` | Document extraction |
| `FILEROUTER_API_KEY` | Hosted document extraction |

Declared but unset secrets do not stop local startup. The matching feature fails or stays disabled when invoked.

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
"@embedpdf/plugin-tiling": "^2.14.4",
"@embedpdf/plugin-viewport": "^2.14.4",
"@embedpdf/plugin-zoom": "^2.14.4",
"@file_router/sdk": "0.5.0",
"@fontsource-variable/geist": "5.3.0",
"@modelcontextprotocol/sdk": "^1.29.0",
"@posthog/ai": "^8.4.0",
Expand Down
Loading