From 3a6fd221870f1a6ee3d61dfb8017c831de76a855 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Sun, 26 Apr 2026 02:07:49 -0400 Subject: [PATCH 1/4] Self-hosting and Zero startup flow --- .env.example | 97 ++++++++------ CONTRIBUTING.md | 4 + README.md | 126 +++++------------- SELF_HOSTING.md | 111 +++++++++++++++ package.json | 6 +- scripts/dev-zero.mjs | 110 +++++++++++++++ setup.sh | 111 ++++++++++++--- src/app/api/audio/process/route.ts | 18 ++- src/app/api/delete-file/route.ts | 35 ++++- src/app/api/ocr/start/route.ts | 16 ++- .../office-conversion/convert-to-pdf/route.ts | 16 ++- src/app/api/pdf/ocr/route.ts | 19 ++- src/app/api/upload-file/route.ts | 34 ++++- src/app/api/upload-url/route.ts | 16 ++- src/components/chat/ChatProvider.tsx | 4 +- src/hooks/workspace/use-workspace-presence.ts | 12 ++ src/lib/self-host-config.ts | 42 ++++++ src/lib/supabase-client.ts | 13 +- src/lib/uploads/client-upload.ts | 29 ++-- src/lib/zero/client.ts | 6 + src/lib/zero/provider.tsx | 15 +++ 21 files changed, 655 insertions(+), 185 deletions(-) create mode 100644 SELF_HOSTING.md create mode 100644 scripts/dev-zero.mjs create mode 100644 src/lib/self-host-config.ts diff --git a/.env.example b/.env.example index 0c8872d0..65fdfc40 100644 --- a/.env.example +++ b/.env.example @@ -1,64 +1,81 @@ -# Application +# Core self-host runtime NEXT_PUBLIC_APP_URL=http://localhost:3000 - -# Better Auth (Authentication) -BETTER_AUTH_SECRET=your-better-auth-secret BETTER_AUTH_URL=http://localhost:3000 +NEXT_PUBLIC_BETTER_AUTH_URL=http://localhost:3000 +BETTER_AUTH_SECRET=your-better-auth-secret -# Google OAuth (for Better Auth) -GOOGLE_CLIENT_ID=... -GOOGLE_CLIENT_SECRET=... - -# Database Configuration +# Database # For Docker PostgreSQL (recommended): # DATABASE_URL=postgresql://thinkex:thinkex_password_change_me@localhost:5432/thinkex # For local PostgreSQL: # DATABASE_URL=postgresql://user:password@localhost:5432/thinkex +DATABASE_URL=postgresql://thinkex:thinkex_password_change_me@localhost:5432/thinkex -# PostgreSQL Docker Configuration (used by docker-compose.yml) +# PostgreSQL Docker defaults used by docker-compose.yml and setup.sh POSTGRES_USER=thinkex POSTGRES_PASSWORD=thinkex_password_change_me POSTGRES_DB=thinkex POSTGRES_PORT=5432 -# File storage (Supabase Storage — required for uploads) -# Create a public bucket named `file-upload` in the Supabase dashboard. +# Zero cache (required for workspace sync in local development) +NEXT_PUBLIC_ZERO_SERVER=http://localhost:4848 +ZERO_UPSTREAM_DB=postgresql://thinkex:thinkex_password_change_me@localhost:5432/thinkex +ZERO_QUERY_URL=http://localhost:3000/api/zero/query +ZERO_MUTATE_URL=http://localhost:3000/api/zero/mutate +ZERO_MUTATE_FORWARD_COOKIES=true +ZERO_QUERY_FORWARD_COOKIES=true +ZERO_APP_PUBLICATIONS=zero_pub +ZERO_ADMIN_PASSWORD=change-me-before-sharing +ZERO_APP_ID=zero_local_dev +ZERO_COOKIE_DOMAIN=localhost + +# Storage +# Core self-host mode uses local files. This supports uploads/viewing inside +# ThinkEx, but media processing features that call third-party providers still +# require provider-reachable object storage. +STORAGE_TYPE=local +UPLOADS_DIR=./uploads + +# Optional provider-reachable storage (required for OCR/audio/doc conversion) NEXT_PUBLIC_SUPABASE_URL=https://your-project.supabase.co NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJ... SUPABASE_SERVICE_ROLE_KEY=eyJ... -# Google AI +# AI backend +# Chat and autogen use the AI backend configured for your deployment. In local +# setups that use Google directly, set GOOGLE_GENERATIVE_AI_API_KEY. GOOGLE_GENERATIVE_AI_API_KEY=AIza... -# AssemblyAI (audio transcription) -ASSEMBLYAI_API_KEY=your-assemblyai-api-key - -# E2B Code Execution Sandbox -E2B_API_KEY=e2b_... - -# Firecrawl Web Scraping (Optional) -# Get your API key from firecrawl.dev -FIRECRAWL_API_KEY=fc_... - -# FastAPI Service (optional - file conversion, doc-to-markdown, audio/video analysis) -# FASTAPI_BASE_URL=https://your-fastapi-service.com -# FASTAPI_API_KEY=your-service-api-key +# Optional OAuth and export integrations +GOOGLE_CLIENT_ID=... +GOOGLE_CLIENT_SECRET=... +NEXT_PUBLIC_GOOGLE_CLIENT_ID=... -# Mistral OCR +# Optional media and retrieval integrations MISTRAL_API_KEY=your-mistral-api-key +MISTRAL_OCR_ENDPOINT=https://api.mistral.ai/v1/ocr +MISTRAL_OCR_MODEL=mistral-ocr-latest +ASSEMBLYAI_API_KEY=your-assemblyai-api-key +FASTAPI_BASE_URL=https://your-fastapi-service.com +FASTAPI_API_KEY=your-service-api-key +FIRECRAWL_API_KEY=fc_... +YOUTUBE_API_KEY=... -# Supermemory (memory layer for chat — https://console.supermemory.ai) -# Optional: when unset, the memory toggle is a no-op server-side. +# Optional chat integrations +E2B_API_KEY=e2b_... SUPERMEMORY_API_KEY=sm_... - -# Kill switch for the Supermemory wrapper. Must be exactly "true" to enable. -# `@supermemory/tools@1.4.3+` contains the Proxy-based fix for AI SDK v6 -# (https://github.com/supermemoryai/supermemory/pull/854). If you downgrade -# the dep or upstream regresses, set this to "false" to neutralize the -# wrapper without ripping out any UI/transport plumbing. SUPERMEMORY_ENABLED=true -# Optional: MISTRAL_OCR_ENDPOINT=https://api.mistral.ai/v1/ocr -# Optional: MISTRAL_OCR_MODEL=mistral-ocr-latest -# Zero Sync -NEXT_PUBLIC_ZERO_SERVER=http://localhost:4848 -ZERO_COOKIE_DOMAIN=localhost + +# Optional observability and email +NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com +NEXT_PUBLIC_POSTHOG_TOKEN=phc_... +POSTHOG_HOST=https://us.i.posthog.com +POSTHOG_PROJECT_TOKEN=phc_... +POSTHOG_PERSONAL_API_KEY=phx_... +POSTHOG_PROJECT_ID=... +POSTHOG_RELEASE_NAME=thinkex +POSTHOG_RELEASE_VERSION= +RESEND_API_KEY=re_... + +# Optional platform-specific settings +CHROME_EXTENSION_ID= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 087a9f38..558ea7e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,7 @@ Contributions should prioritize: - Node.js **v20+** - pnpm - PostgreSQL (local or Docker) +- Zero cache server configuration from [SELF_HOSTING.md](SELF_HOSTING.md) ### Quick Setup (Recommended) @@ -31,6 +32,7 @@ Run the interactive setup script: git clone https://github.com/ThinkEx-OSS/thinkex.git cd thinkex ./setup.sh +pnpm dev ``` ### Manual Setup @@ -42,3 +44,5 @@ cp .env.example .env pnpm db:push pnpm dev ``` + +Use [SELF_HOSTING.md](SELF_HOSTING.md) as the source of truth for environment variables, Zero startup, local storage, and optional media integrations. diff --git a/README.md b/README.md index 8dee3482..14ef03c3 100644 --- a/README.md +++ b/README.md @@ -58,112 +58,48 @@ Nothing disappears into a black box. You see what AI sees and control what it wo ## Self-Hosting -ThinkEx can be self hosted for local development. The setup uses Docker for PostgreSQL (recommended) while running the Next.js app locally. +ThinkEx supports a **core self-host** path for local development: -### Quick Start - -#### Prerequisites +- PostgreSQL +- Better Auth secret + app URL +- Zero cache server +- Local filesystem storage -* [Node.js](https://nodejs.org/) (v20+) -* [pnpm](https://pnpm.io/) (will be installed automatically if missing) -* [Docker](https://docs.docker.com/get-docker/) (recommended for PostgreSQL) OR [PostgreSQL](https://www.postgresql.org/download/) (v12+) installed locally -* **Required API Keys:** - * **Google AI**: API key from [Google AI Studio](https://aistudio.google.com/app/apikey) - * `GOOGLE_GENERATIVE_AI_API_KEY` - * **Supabase** (file uploads): Project URL and keys from [Supabase](https://supabase.com) - * `NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY` -* **Optional API Keys:** - * **Google OAuth**: Get credentials from [Google Cloud Console](https://console.cloud.google.com/apis/credentials) (for OAuth login) - * `GOOGLE_CLIENT_ID` - * `GOOGLE_CLIENT_SECRET` - * **Other optional keys** (web scraping, OCR, external conversion service): see [`.env.example`](.env.example) (`FIRECRAWL_API_KEY`, `MISTRAL_API_KEY`, `FASTAPI_*`, `SCRAPING_MODE`, etc.) +`pnpm dev` is the supported one-command entrypoint for this setup. It starts: -#### Automated Setup +- Next.js +- the AI SDK devtools +- Zero -Run the interactive setup script: +### Quick Start ```bash git clone https://github.com/ThinkEx-OSS/thinkex.git cd thinkex ./setup.sh +pnpm dev ``` -The script will: -- Check prerequisites (Node.js, pnpm, Docker) -- Create `.env` file from template -- Generate `BETTER_AUTH_SECRET` automatically -- Start PostgreSQL in Docker (or use local PostgreSQL if Docker is not available) -- Configure database connection -- Install dependencies -- Initialize the database schema -- Start the development server automatically - -Access ThinkEx at [http://localhost:3000](http://localhost:3000) - -**PostgreSQL Docker Commands:** -- Stop PostgreSQL: `docker-compose down` -- Start PostgreSQL: `docker-compose up -d` -- View logs: `docker-compose logs -f postgres` - -#### Manual Setup - -1. **Clone the repository** - ```bash - git clone https://github.com/ThinkEx-OSS/thinkex.git - cd thinkex - ``` - -2. **Start PostgreSQL (Docker)** - ```bash - docker-compose up -d postgres - ``` - - Or use your local PostgreSQL installation. - -3. **Install dependencies** - ```bash - pnpm install - ``` - -4. **Configure environment variables** - ```bash - cp .env.example .env - ``` - - Edit `.env` and configure: - - * **Database**: Set `DATABASE_URL` to your PostgreSQL connection string - ```bash - # For Docker PostgreSQL: - DATABASE_URL=postgresql://thinkex:thinkex_password_change_me@localhost:5432/thinkex - - # For local PostgreSQL: - DATABASE_URL=postgresql://user:password@localhost:5432/thinkex - ``` - * **Better Auth**: Generate `BETTER_AUTH_SECRET` with `openssl rand -base64 32`. For a public URL (not localhost), set `BETTER_AUTH_URL` and `NEXT_PUBLIC_APP_URL` to that origin. - * **Google OAuth**: Get credentials from [Google Cloud Console](https://console.cloud.google.com/apis/credentials) - * **Supabase**: Your Supabase project URL and keys (required for file uploads) - * **Google AI**: API key from [Google AI Studio](https://aistudio.google.com/app/apikey) - -5. **Initialize the database** - ```bash - pnpm db:push - ``` - -6. **Start the development server** - ```bash - pnpm dev - ``` - -7. **Access the application** - Open [http://localhost:3000](http://localhost:3000) in your browser. - -#### File storage (Supabase) - -Uploads go to **Supabase Storage**. Configure: - -- `NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY` -- A storage bucket named `file-upload` set to **Public** +Access ThinkEx at [http://localhost:3000](http://localhost:3000). + +### What Core Self-Host Includes + +- Local uploads, viewing, and deletion using `STORAGE_TYPE=local` +- Workspace sync through Zero +- The main app shell and workspace flows + +### What Core Self-Host Does Not Include + +OCR, audio transcription, and office document conversion require provider-reachable object storage plus the relevant provider credentials. Those features are intentionally not supported in local-file core mode and will fail with explicit messages. + +### Full Setup Guide + +See [SELF_HOSTING.md](SELF_HOSTING.md) for: + +- required local environment variables +- Zero configuration +- local storage behavior +- optional integrations such as Supabase, OCR, audio, analytics, and email ## Contributing diff --git a/SELF_HOSTING.md b/SELF_HOSTING.md new file mode 100644 index 00000000..48c8250a --- /dev/null +++ b/SELF_HOSTING.md @@ -0,0 +1,111 @@ +# Self-Hosting ThinkEx + +This repo currently supports a **core self-host** profile for local development: + +- PostgreSQL +- Better Auth secret + app URL +- Zero cache server +- Local filesystem storage + +`pnpm dev` is the supported one-command entrypoint. It starts: + +- Next.js +- the AI SDK devtools +- Zero + +## Quick Start + +```bash +git clone https://github.com/ThinkEx-OSS/thinkex.git +cd thinkex +./setup.sh +pnpm dev +``` + +If you prefer to bootstrap manually: + +```bash +pnpm install +cp .env.example .env +docker-compose up -d postgres +pnpm db:push +pnpm dev +``` + +## Core Self-Host Environment + +The minimum local profile needs these groups configured: + +- App/auth: + - `NEXT_PUBLIC_APP_URL` + - `BETTER_AUTH_URL` + - `NEXT_PUBLIC_BETTER_AUTH_URL` + - `BETTER_AUTH_SECRET` +- Database: + - `DATABASE_URL` +- Zero: + - `NEXT_PUBLIC_ZERO_SERVER` + - `ZERO_UPSTREAM_DB` + - `ZERO_QUERY_URL` + - `ZERO_MUTATE_URL` + - `ZERO_MUTATE_FORWARD_COOKIES` + - `ZERO_QUERY_FORWARD_COOKIES` + - `ZERO_APP_PUBLICATIONS` + - `ZERO_ADMIN_PASSWORD` + - `ZERO_APP_ID` + - `ZERO_COOKIE_DOMAIN` +- Storage: + - `STORAGE_TYPE=local` + - `UPLOADS_DIR` + +`./setup.sh` populates sensible local defaults for the core profile and creates the `zero_pub` publication used by the local Zero process. + +## Zero + +Zero is required for workspace sync in local development. ThinkEx does not currently support a non-Zero workspace mode. + +Use: + +```bash +pnpm dev +``` + +The repo also includes `pnpm run dev:zero` as a debugging hook, but `pnpm dev` is the supported workflow. + +## Local File Storage + +ThinkEx uses `STORAGE_TYPE=local` by default for the core self-host profile. That supports: + +- local uploads +- viewing uploaded files inside the app +- deleting uploaded files + +It does **not** make files reachable to third-party providers. + +## Unsupported in Core Local Mode + +These features require provider-reachable object storage and the relevant API keys: + +- OCR +- audio transcription +- office document conversion + +In local-file core mode, those flows are intentionally blocked with explicit capability errors. + +## Optional Integrations + +You can still configure optional integrations from `.env.example`, including: + +- Supabase storage for provider-reachable uploads +- Google OAuth +- Mistral OCR +- AssemblyAI +- FastAPI conversion +- Firecrawl +- E2B +- PostHog +- Resend +- Supermemory +- YouTube metadata + +Those are not required to boot the core self-host profile. diff --git a/package.json b/package.json index 30deb2f6..61d3f016 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,11 @@ "version": "0.0.1", "private": true, "scripts": { - "dev": "concurrently \"next dev\" \"npx @ai-sdk/devtools\"", + "dev": "concurrently -k -s first -n web,zero \"pnpm dev:web\" \"pnpm dev:zero\"", + "dev:web": "concurrently -k -s first -n next,ai \"pnpm dev:next\" \"pnpm dev:ai\"", + "dev:next": "next dev", + "dev:ai": "pnpm exec devtools", + "dev:zero": "node scripts/dev-zero.mjs", "build": "next build", "start": "next start", "lint": "oxlint", diff --git a/scripts/dev-zero.mjs b/scripts/dev-zero.mjs new file mode 100644 index 00000000..532657ab --- /dev/null +++ b/scripts/dev-zero.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { spawn } from "node:child_process"; + +const ENV_FILES = [ + ".env.development.local", + ".env.local", + ".env.development", + ".env", +]; + +const REQUIRED_ENV_KEYS = [ + "ZERO_UPSTREAM_DB", + "ZERO_QUERY_URL", + "ZERO_MUTATE_URL", + "ZERO_MUTATE_FORWARD_COOKIES", + "ZERO_QUERY_FORWARD_COOKIES", + "ZERO_APP_PUBLICATIONS", + "ZERO_ADMIN_PASSWORD", + "ZERO_APP_ID", + "NEXT_PUBLIC_ZERO_SERVER", +]; + +function parseEnvValue(rawValue) { + const trimmed = rawValue.trim(); + + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1); + } + + return trimmed; +} + +function loadEnvFiles() { + for (const file of ENV_FILES) { + const fullPath = resolve(process.cwd(), file); + if (!existsSync(fullPath)) continue; + + const content = readFileSync(fullPath, "utf8"); + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + + const eqIndex = trimmed.indexOf("="); + if (eqIndex <= 0) continue; + + const key = trimmed.slice(0, eqIndex).trim(); + if (!key || process.env[key] !== undefined) continue; + + const value = parseEnvValue(trimmed.slice(eqIndex + 1)); + process.env[key] = value; + } + } +} + +function failMissingEnv(keys) { + console.error( + [ + "Zero startup configuration is incomplete.", + "", + "Missing required environment variables:", + ...keys.map((key) => `- ${key}`), + "", + "Copy the values from `.env.example` or run `./setup.sh`, then retry `pnpm dev`.", + ].join("\n"), + ); + process.exit(1); +} + +function spawnZero() { + const pnpmCmd = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; + const forwardedArgs = [...process.argv.slice(2)]; + if (forwardedArgs[0] === "--") { + forwardedArgs.shift(); + } + const args = ["exec", "zero-cache", ...forwardedArgs]; + const child = spawn(pnpmCmd, args, { + stdio: "inherit", + env: process.env, + }); + + child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 0); + }); +} + +const helpRequested = process.argv + .slice(2) + .filter((arg) => arg !== "--") + .some((arg) => arg === "--help" || arg === "-h"); + +loadEnvFiles(); + +if (!helpRequested) { + const missingKeys = REQUIRED_ENV_KEYS.filter((key) => !process.env[key]?.trim()); + if (missingKeys.length > 0) { + failMissingEnv(missingKeys); + } +} + +spawnZero(); diff --git a/setup.sh b/setup.sh index 1c922d92..8bd2b1b0 100755 --- a/setup.sh +++ b/setup.sh @@ -33,6 +33,37 @@ command_exists() { command -v "$1" >/dev/null 2>&1 } +get_env_value() { + local key=$1 + grep -E "^${key}=" .env 2>/dev/null | head -n1 | cut -d'=' -f2- +} + +set_env_value() { + local key=$1 + local value=$2 + + if grep -q "^${key}=" .env 2>/dev/null; then + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "s|^${key}=.*|${key}=${value}|" .env + else + sed -i "s|^${key}=.*|${key}=${value}|" .env + fi + else + echo "${key}=${value}" >> .env + fi +} + +ensure_env_value() { + local key=$1 + local fallback=$2 + local current + current=$(get_env_value "$key") + + if [ -z "$current" ]; then + set_env_value "$key" "$fallback" + fi +} + # Function to check PostgreSQL connection check_postgres() { local db_url=$1 @@ -68,13 +99,42 @@ setup_env_file() { if ! grep -q "BETTER_AUTH_SECRET=.*[^=]$" .env 2>/dev/null || grep -q "BETTER_AUTH_SECRET=your-better-auth-secret" .env 2>/dev/null || grep -q "BETTER_AUTH_SECRET=your-better-auth-secret-change-this" .env 2>/dev/null; then echo -e "${YELLOW}Generating BETTER_AUTH_SECRET...${RESET}" SECRET=$(openssl rand -base64 32) - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "s|BETTER_AUTH_SECRET=.*|BETTER_AUTH_SECRET=$SECRET|" .env - else - sed -i "s|BETTER_AUTH_SECRET=.*|BETTER_AUTH_SECRET=$SECRET|" .env - fi + set_env_value "BETTER_AUTH_SECRET" "$SECRET" echo -e "${GREEN}Generated and set BETTER_AUTH_SECRET${RESET}" fi + + ensure_env_value "NEXT_PUBLIC_APP_URL" "http://localhost:3000" + ensure_env_value "BETTER_AUTH_URL" "$(get_env_value NEXT_PUBLIC_APP_URL)" + ensure_env_value "NEXT_PUBLIC_BETTER_AUTH_URL" "$(get_env_value NEXT_PUBLIC_APP_URL)" + ensure_env_value "STORAGE_TYPE" "local" + ensure_env_value "UPLOADS_DIR" "./uploads" + ensure_env_value "NEXT_PUBLIC_ZERO_SERVER" "http://localhost:4848" + ensure_env_value "ZERO_COOKIE_DOMAIN" "localhost" +} + +configure_zero_env() { + local db_url=$1 + local app_url + local zero_admin_password + + app_url=$(get_env_value "NEXT_PUBLIC_APP_URL") + if [ -z "$app_url" ]; then + app_url="http://localhost:3000" + set_env_value "NEXT_PUBLIC_APP_URL" "$app_url" + fi + + set_env_value "ZERO_UPSTREAM_DB" "$db_url" + set_env_value "ZERO_QUERY_URL" "${app_url}/api/zero/query" + set_env_value "ZERO_MUTATE_URL" "${app_url}/api/zero/mutate" + set_env_value "ZERO_MUTATE_FORWARD_COOKIES" "true" + set_env_value "ZERO_QUERY_FORWARD_COOKIES" "true" + ensure_env_value "ZERO_APP_PUBLICATIONS" "zero_pub" + ensure_env_value "ZERO_APP_ID" "zero_local_dev" + + zero_admin_password=$(get_env_value "ZERO_ADMIN_PASSWORD") + if [ -z "$zero_admin_password" ] || [ "$zero_admin_password" = "change-me-before-sharing" ] || [ "$zero_admin_password" = "your-local-dev-secret" ]; then + set_env_value "ZERO_ADMIN_PASSWORD" "$(openssl rand -hex 16)" + fi } # Main setup function @@ -258,6 +318,8 @@ else fi fi +configure_zero_env "$DB_URL" + # Create database if it doesn't exist (for local PostgreSQL) if [ "$USE_DOCKER_POSTGRES" = false ]; then echo "" @@ -299,6 +361,20 @@ else psql "$DB_URL" -c "CREATE OR REPLACE FUNCTION auth.jwt() RETURNS jsonb LANGUAGE sql STABLE AS \$func\$ SELECT NULL::jsonb; \$func\$;" 2>/dev/null || true fi +echo "" +echo -e "${YELLOW}Creating Zero publication for local sync...${RESET}" +ZERO_APP_PUBLICATIONS=$(get_env_value "ZERO_APP_PUBLICATIONS") +ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_pub} +if [ "$USE_DOCKER_POSTGRES" = true ]; then + docker-compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "CREATE PUBLICATION ${ZERO_APP_PUBLICATIONS} FOR TABLES IN SCHEMA public;" 2>/dev/null || docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "CREATE PUBLICATION ${ZERO_APP_PUBLICATIONS} FOR TABLES IN SCHEMA public;" 2>/dev/null || { + echo "Publication '${ZERO_APP_PUBLICATIONS}' may already exist, continuing..." + } +else + psql "$DB_URL" -c "CREATE PUBLICATION ${ZERO_APP_PUBLICATIONS} FOR TABLES IN SCHEMA public;" 2>/dev/null || { + echo "Publication '${ZERO_APP_PUBLICATIONS}' may already exist, continuing..." + } +fi + # Install dependencies echo "" echo -e "${YELLOW}Installing dependencies...${RESET}" @@ -331,19 +407,17 @@ pnpm drizzle-kit push --force || { } echo "" -echo -e "${GREEN}✓ Setup complete!${RESET}" +echo -e "${GREEN}✓ Core self-host bootstrap complete!${RESET}" echo "" -echo -e "${YELLOW}IMPORTANT: Please edit .env and configure:${RESET}" +echo -e "${YELLOW}Configured for core self-host:${RESET}" +echo " - PostgreSQL connection" +echo " - Better Auth local URLs + secret" +echo " - Zero local defaults and publication (${ZERO_APP_PUBLICATIONS})" +echo " - Local filesystem storage (STORAGE_TYPE=local)" echo "" -echo -e "${RED}Required API Keys:${RESET}" -echo " - GOOGLE_GENERATIVE_AI_API_KEY (from Google AI Studio)" -echo "" -echo -e "${YELLOW}Optional API Keys:${RESET}" -echo " - GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET (for OAuth login)" -echo " - Supabase credentials (if using Supabase storage):" -echo " * NEXT_PUBLIC_SUPABASE_URL" -echo " * NEXT_PUBLIC_SUPABASE_ANON_KEY" -echo " * SUPABASE_SERVICE_ROLE_KEY" +echo -e "${YELLOW}Optional next steps:${RESET}" +echo " - Add your AI backend credentials for chat/autogen features" +echo " - Add provider-reachable storage + media keys if you need OCR, audio transcription, or office conversion" echo "" if [ "$USE_DOCKER_POSTGRES" = true ]; then echo "PostgreSQL is running in Docker. Useful commands:" @@ -353,7 +427,6 @@ if [ "$USE_DOCKER_POSTGRES" = true ]; then echo "" fi echo "" -echo -e "${GREEN}Starting development server...${RESET}" +echo -e "${GREEN}Start ThinkEx with:${RESET} pnpm dev" +echo "That command starts Next.js, the AI SDK devtools, and Zero together." echo "Access ThinkEx at: http://localhost:3000" -echo "" -pnpm dev diff --git a/src/app/api/audio/process/route.ts b/src/app/api/audio/process/route.ts index e21e2572..9ec8e1c0 100644 --- a/src/app/api/audio/process/route.ts +++ b/src/app/api/audio/process/route.ts @@ -6,6 +6,10 @@ import { verifyWorkspaceAccess, withErrorHandling, } from "@/lib/api/workspace-helpers"; +import { + getUnsupportedLocalStorageMessage, + usesLocalStorage, +} from "@/lib/self-host-config"; import { isAllowedAssetUrl } from "@/lib/tasks/validate-asset-url"; export const dynamic = "force-dynamic"; @@ -18,6 +22,15 @@ export const dynamic = "force-dynamic"; async function handlePOST(req: NextRequest) { const userId = await requireAuth(); + if (usesLocalStorage()) { + return NextResponse.json( + { + error: getUnsupportedLocalStorageMessage("Audio transcription"), + }, + { status: 400 }, + ); + } + if (!process.env.ASSEMBLYAI_API_KEY) { return NextResponse.json( { error: "ASSEMBLYAI_API_KEY is not set" }, @@ -51,7 +64,10 @@ async function handlePOST(req: NextRequest) { if (!isAllowedAssetUrl(fileUrl)) { return NextResponse.json( - { error: "fileUrl origin is not allowed" }, + { + error: + "Audio transcription only accepts provider-reachable storage URLs configured for this deployment.", + }, { status: 400 }, ); } diff --git a/src/app/api/delete-file/route.ts b/src/app/api/delete-file/route.ts index e6eb4175..285e362a 100644 --- a/src/app/api/delete-file/route.ts +++ b/src/app/api/delete-file/route.ts @@ -1,14 +1,24 @@ import { headers } from "next/headers"; import { auth } from "@/lib/auth"; import { createClient } from "@supabase/supabase-js"; +import { existsSync } from "node:fs"; +import { unlink } from "node:fs/promises"; import { NextRequest, NextResponse } from "next/server"; +import { join } from "node:path"; +import { getStorageMode } from "@/lib/self-host-config"; -// Extract object path from a Supabase public file URL +// Extract object path from a supported public file URL function extractFilename(url: string): string | null { const supabaseMatch = url.match(/\/file-upload\/(.+)$/); if (supabaseMatch) { return supabaseMatch[1]; } + + const localMatch = url.match(/\/api\/files\/(.+)$/); + if (localMatch) { + return localMatch[1]; + } + return null; } @@ -47,6 +57,29 @@ export async function DELETE(request: NextRequest) { return NextResponse.json({ error: "Invalid filename" }, { status: 400 }); } + if (getStorageMode() === "local") { + if (filename.includes("/")) { + return NextResponse.json({ error: "Invalid filename" }, { status: 400 }); + } + + const uploadsDir = process.env.UPLOADS_DIR || join(process.cwd(), "uploads"); + const filePath = join(uploadsDir, filename); + + if (!existsSync(filePath)) { + return NextResponse.json({ + success: true, + message: "File not found (may have been deleted already)", + }); + } + + await unlink(filePath); + + return NextResponse.json({ + success: true, + message: "File deleted successfully", + }); + } + const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL; const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY; diff --git a/src/app/api/ocr/start/route.ts b/src/app/api/ocr/start/route.ts index 547f0dcf..06a31d7b 100644 --- a/src/app/api/ocr/start/route.ts +++ b/src/app/api/ocr/start/route.ts @@ -6,6 +6,10 @@ import { verifyWorkspaceAccess } from "@/lib/api/workspace-helpers"; import { withServerObservability } from "@/lib/with-server-observability"; import { filterOcrCandidates } from "@/lib/ocr/dispatch"; import { isAllowedOcrFileUrl } from "@/lib/ocr/url-validation"; +import { + getUnsupportedLocalStorageMessage, + usesLocalStorage, +} from "@/lib/self-host-config"; import { ocrDispatchWorkflow } from "@/workflows/ocr-dispatch"; import type { OcrCandidate } from "@/lib/ocr/types"; @@ -40,12 +44,22 @@ export const POST = withServerObservability(async function POST(req: NextRequest ); } + if (usesLocalStorage()) { + return NextResponse.json( + { error: getUnsupportedLocalStorageMessage("OCR") }, + { status: 400 }, + ); + } + const invalidCandidate = candidates.find( (candidate) => !isAllowedOcrFileUrl(candidate.fileUrl) ); if (invalidCandidate) { return NextResponse.json( - { error: "One or more OCR candidate URLs are not allowed" }, + { + error: + "OCR only accepts provider-reachable storage URLs configured for this deployment.", + }, { status: 400 } ); } diff --git a/src/app/api/office-conversion/convert-to-pdf/route.ts b/src/app/api/office-conversion/convert-to-pdf/route.ts index 298a71b8..a6a7332d 100644 --- a/src/app/api/office-conversion/convert-to-pdf/route.ts +++ b/src/app/api/office-conversion/convert-to-pdf/route.ts @@ -5,6 +5,10 @@ import { isValidDocumentConversionRequest, requestDocumentPdfConversion, } from "@/lib/uploads/document-conversion"; +import { + getUnsupportedLocalStorageMessage, + usesLocalStorage, +} from "@/lib/self-host-config"; /** * Proxies document-to-PDF conversion to the FastAPI backend. @@ -23,6 +27,13 @@ export async function POST(request: NextRequest) { const body = await request.json(); const { file_path, file_url } = body as { file_path?: string; file_url?: string }; + if (usesLocalStorage()) { + return NextResponse.json( + { error: getUnsupportedLocalStorageMessage("Document conversion") }, + { status: 400 }, + ); + } + if (!file_path || typeof file_path !== "string") { return NextResponse.json( { error: "file_path is required" }, @@ -40,7 +51,10 @@ export async function POST(request: NextRequest) { // SSRF: only our public file URLs (same origin or Supabase bucket) with a strict path shape if (!isValidDocumentConversionRequest(file_path, file_url)) { return NextResponse.json( - { error: "Invalid conversion source" }, + { + error: + "Document conversion only accepts provider-reachable storage URLs configured for this deployment.", + }, { status: 400 } ); } diff --git a/src/app/api/pdf/ocr/route.ts b/src/app/api/pdf/ocr/route.ts index 2ed9e17d..6f499ef2 100644 --- a/src/app/api/pdf/ocr/route.ts +++ b/src/app/api/pdf/ocr/route.ts @@ -5,6 +5,10 @@ import { ocrPdfFromUrl } from "@/lib/pdf/mistral-ocr"; import { logger } from "@/lib/utils/logger"; import { withServerObservability } from "@/lib/with-server-observability"; import { isAllowedOcrFileUrl } from "@/lib/ocr/url-validation"; +import { + getUnsupportedLocalStorageMessage, + usesLocalStorage, +} from "@/lib/self-host-config"; export const dynamic = "force-dynamic"; @@ -32,10 +36,23 @@ export const POST = withServerObservability(async function POST(req: NextRequest ); } + if (usesLocalStorage()) { + return NextResponse.json( + { error: getUnsupportedLocalStorageMessage("OCR") }, + { status: 400 }, + ); + } + const t0 = Date.now(); if (!isAllowedOcrFileUrl(fileUrl)) { - return NextResponse.json({ error: "Invalid fileUrl" }, { status: 400 }); + return NextResponse.json( + { + error: + "OCR only accepts provider-reachable storage URLs configured for this deployment.", + }, + { status: 400 }, + ); } let pathname: string; diff --git a/src/app/api/upload-file/route.ts b/src/app/api/upload-file/route.ts index f0ab9c08..e88a1ec9 100644 --- a/src/app/api/upload-file/route.ts +++ b/src/app/api/upload-file/route.ts @@ -1,16 +1,33 @@ import { headers } from "next/headers"; import { auth } from "@/lib/auth"; import { createClient } from "@supabase/supabase-js"; +import { existsSync } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; import { NextRequest, NextResponse } from "next/server"; +import { join } from "node:path"; import { getPreferredUploadContentType, - isOfficeDocument, getOfficeDocumentConvertUrl, } from "@/lib/uploads/office-document-validation"; +import { getStorageMode } from "@/lib/self-host-config"; import { withServerObservability } from "@/lib/with-server-observability"; export const maxDuration = 30; +async function saveFileLocally(file: File, filename: string): Promise { + const uploadsDir = process.env.UPLOADS_DIR || join(process.cwd(), "uploads"); + + if (!existsSync(uploadsDir)) { + await mkdir(uploadsDir, { recursive: true }); + } + + const bytes = await file.arrayBuffer(); + await writeFile(join(uploadsDir, filename), Buffer.from(bytes)); + + const appUrl = process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"; + return `${appUrl}/api/files/${filename}`; +} + export const POST = withServerObservability(async function POST(request: NextRequest) { try { // Get authenticated user from Better Auth @@ -25,8 +42,6 @@ export const POST = withServerObservability(async function POST(request: NextReq ); } - const userId = session.user.id; - // Get file from form data const formData = await request.formData(); const file = formData.get('file') as File; @@ -97,10 +112,21 @@ export const POST = withServerObservability(async function POST(request: NextReq const originalName = file.name; // Sanitize filename: remove spaces and special chars, keep only alphanumeric, dots, hyphens, underscores const sanitizedName = originalName.replace(/[^a-zA-Z0-9._-]/g, "_"); - const filename = isOfficeUpload + const storageMode = getStorageMode(); + const filename = storageMode === "supabase" && isOfficeUpload ? `uploads/${timestamp}-${random}-${sanitizedName}` : `${timestamp}-${random}-${sanitizedName}`; + if (storageMode === "local") { + const publicUrl = await saveFileLocally(file, filename); + + return NextResponse.json({ + success: true, + url: publicUrl, + filename, + }); + } + const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL; const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY; diff --git a/src/app/api/upload-url/route.ts b/src/app/api/upload-url/route.ts index c07d4ac8..8a6f9bb9 100644 --- a/src/app/api/upload-url/route.ts +++ b/src/app/api/upload-url/route.ts @@ -3,16 +3,14 @@ import { auth } from "@/lib/auth"; import { createClient } from "@supabase/supabase-js"; import { NextRequest, NextResponse } from 'next/server'; import { getOfficeDocumentConvertUrlFromMeta } from "@/lib/uploads/office-document-validation"; +import { getStorageMode } from "@/lib/self-host-config"; import { withServerObservability } from "@/lib/with-server-observability"; export const maxDuration = 10; /** - * Generates a signed upload URL for direct client-to-Supabase uploads. - * This avoids the Vercel 4.5MB serverless function body size limit. - * - * The client sends only the filename/metadata (tiny JSON payload), - * receives a signed URL, then uploads the file directly to Supabase. + * Generates a signed upload URL for direct storage uploads. + * In local mode the client falls back to /api/upload-file. */ async function handlePOST(request: NextRequest) { try { @@ -40,6 +38,14 @@ async function handlePOST(request: NextRequest) { const convertUrl = getOfficeDocumentConvertUrlFromMeta(filename, contentType); const isOfficeUpload = convertUrl !== null; + const storageMode = getStorageMode(); + + if (storageMode === "local") { + return NextResponse.json({ + mode: "local", + uploadUrl: "/api/upload-file", + }); + } // Supabase storage: generate a signed upload URL const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL; diff --git a/src/components/chat/ChatProvider.tsx b/src/components/chat/ChatProvider.tsx index 6a455fee..31dffa60 100644 --- a/src/components/chat/ChatProvider.tsx +++ b/src/components/chat/ChatProvider.tsx @@ -140,9 +140,9 @@ function describeChatError(error: Error): { (combined.includes("not valid") || combined.includes("invalid"))) ) { return { - title: "API key not valid", + title: "AI backend configuration error", description: - "Please check your GOOGLE_GENERATIVE_AI_API_KEY in your environment variables.", + "Please check the AI provider or gateway environment variables configured for your self-hosted deployment.", }; } return { diff --git a/src/hooks/workspace/use-workspace-presence.ts b/src/hooks/workspace/use-workspace-presence.ts index 3818ed88..9d751cf8 100644 --- a/src/hooks/workspace/use-workspace-presence.ts +++ b/src/hooks/workspace/use-workspace-presence.ts @@ -7,6 +7,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { RealtimeChannel } from "@supabase/supabase-js"; import { getSupabaseClient } from "@/lib/supabase-client"; +import { isSupabaseClientConfigured } from "@/lib/self-host-config"; export interface CollaboratorPresence { userId: string; @@ -49,6 +50,11 @@ export function useWorkspacePresence( const cleanup = useCallback(() => { if (channelRef.current) { + if (!isSupabaseClientConfigured()) { + channelRef.current = null; + return; + } + const supabase = getSupabaseClient(); supabase.removeChannel(channelRef.current); channelRef.current = null; @@ -61,6 +67,12 @@ export function useWorkspacePresence( return; } + if (!isSupabaseClientConfigured()) { + cleanup(); + setCollaborators([]); + return; + } + const supabase = getSupabaseClient(); const channel = supabase.channel(`workspace:${workspaceId}:presence`, { config: { diff --git a/src/lib/self-host-config.ts b/src/lib/self-host-config.ts new file mode 100644 index 00000000..536b9999 --- /dev/null +++ b/src/lib/self-host-config.ts @@ -0,0 +1,42 @@ +export type StorageMode = "local" | "supabase"; + +const ZERO_REQUIRED_MESSAGE = + "ThinkEx requires Zero for workspace sync in self-hosted development. Set NEXT_PUBLIC_ZERO_SERVER and start the Zero cache server with `pnpm dev`."; + +export function getStorageMode(): StorageMode { + return process.env.STORAGE_TYPE === "supabase" ? "supabase" : "local"; +} + +export function usesLocalStorage(): boolean { + return getStorageMode() === "local"; +} + +export function usesProviderReachableStorage(): boolean { + return getStorageMode() === "supabase"; +} + +export function isSupabaseClientConfigured(): boolean { + return Boolean( + process.env.NEXT_PUBLIC_SUPABASE_URL?.trim() && + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY?.trim(), + ); +} + +export function isSupabaseServerConfigured(): boolean { + return Boolean( + process.env.NEXT_PUBLIC_SUPABASE_URL?.trim() && + process.env.SUPABASE_SERVICE_ROLE_KEY?.trim(), + ); +} + +export function getUnsupportedLocalStorageMessage(feature: string): string { + return `${feature} is unavailable in core self-host mode with local file storage. Configure provider-reachable object storage before using ${feature.toLowerCase()}.`; +} + +export function getZeroConfigError(): string | null { + if (!process.env.NEXT_PUBLIC_ZERO_SERVER?.trim()) { + return ZERO_REQUIRED_MESSAGE; + } + + return null; +} diff --git a/src/lib/supabase-client.ts b/src/lib/supabase-client.ts index b2bbc2c0..206bd556 100644 --- a/src/lib/supabase-client.ts +++ b/src/lib/supabase-client.ts @@ -4,6 +4,7 @@ */ import { createClient, SupabaseClient } from '@supabase/supabase-js'; +import { isSupabaseClientConfigured } from "@/lib/self-host-config"; let supabaseClient: SupabaseClient | null = null; @@ -16,15 +17,15 @@ export function getSupabaseClient(): SupabaseClient { return supabaseClient; } - const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL; - const supabaseAnonKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY; - - if (!supabaseUrl || !supabaseAnonKey) { + if (!isSupabaseClientConfigured()) { throw new Error( 'Missing Supabase environment variables: NEXT_PUBLIC_SUPABASE_URL and NEXT_PUBLIC_SUPABASE_ANON_KEY are required' ); } + const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL!; + const supabaseAnonKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY!; + supabaseClient = createClient(supabaseUrl, supabaseAnonKey, { realtime: { params: { @@ -41,6 +42,10 @@ export function getSupabaseClient(): SupabaseClient { * Should be called after user logs in or token refreshes */ export async function setRealtimeAuth(accessToken: string): Promise { + if (!isSupabaseClientConfigured()) { + return; + } + const client = getSupabaseClient(); await client.realtime.setAuth(accessToken); } diff --git a/src/lib/uploads/client-upload.ts b/src/lib/uploads/client-upload.ts index 6eb7a223..6ae801a2 100644 --- a/src/lib/uploads/client-upload.ts +++ b/src/lib/uploads/client-upload.ts @@ -1,12 +1,6 @@ /** - * Client-side file upload utility that uploads directly to Supabase storage, - * bypassing the Vercel 4.5MB serverless function body size limit. - * - * Flow: - * 1. HEIC/HEIF images are converted to JPEG for browser compatibility - * 2. Client requests a signed upload URL from /api/upload-url (tiny JSON payload) - * 3. Client uploads the file directly to Supabase using the signed URL - * 4. Returns the public URL of the original uploaded file + * Client-side file upload utility that uploads directly to storage when + * available and falls back to the API route in local-file mode. */ import { convertHeicToJpegIfNeeded } from "./convert-heic"; @@ -74,8 +68,9 @@ async function convertOfficeUpload( } /** - * Upload a file directly to Supabase storage, bypassing the serverless function body limit. - * Small-file retry uses /api/upload-file (Supabase) when the signed PUT fails. + * Upload a file directly to storage when the current backend supports signed + * URLs. In local-file mode the API responds with `mode=local` and the client + * falls back to /api/upload-file. */ export async function uploadFileDirect( file: File, @@ -125,6 +120,20 @@ export async function uploadFileDirect( ); } + if (urlData.mode === "local") { + const result = await uploadViaApiRoute(file); + if (log) { + const total = performance.now() - t0; + console.info(`[PDF_UPLOAD] Local upload fallback: ${total.toFixed(0)}ms`); + } + + if (isOfficeUpload) { + return convertOfficeUpload(result.filename, result.url, file.name); + } + + return result; + } + // Step 2: Upload file directly to Supabase using the signed URL const { signedUrl, publicUrl, path } = urlData; const tPut = log ? performance.now() : 0; diff --git a/src/lib/zero/client.ts b/src/lib/zero/client.ts index c7cfec69..468a4727 100644 --- a/src/lib/zero/client.ts +++ b/src/lib/zero/client.ts @@ -1,4 +1,5 @@ import { Zero } from "@rocicorp/zero"; +import { getZeroConfigError } from "@/lib/self-host-config"; import { mutators } from "./mutators"; import { schema } from "./zero-schema.gen"; @@ -10,6 +11,11 @@ const appURL = process.env.NEXT_PUBLIC_APP_URL ?? "http://localhost:3000"; function createZeroInstance(params: { userId: string }) { + const configError = getZeroConfigError(); + if (configError) { + throw new Error(configError); + } + return new Zero({ schema, cacheURL: process.env.NEXT_PUBLIC_ZERO_SERVER!, diff --git a/src/lib/zero/provider.tsx b/src/lib/zero/provider.tsx index 299759f0..b61388ee 100644 --- a/src/lib/zero/provider.tsx +++ b/src/lib/zero/provider.tsx @@ -4,11 +4,13 @@ import { ZeroProvider as BaseZeroProvider } from "@rocicorp/zero/react"; import type { ReactNode } from "react"; import { useEffect, useMemo } from "react"; import { useSession } from "@/lib/auth-client"; +import { getZeroConfigError } from "@/lib/self-host-config"; import { destroyZero, getZero } from "./client"; export function ZeroProvider({ children }: { children: ReactNode }) { const { data: session, isPending } = useSession(); const userId = session?.user?.id ?? null; + const zeroConfigError = getZeroConfigError(); useEffect(() => { if (!userId) { @@ -16,6 +18,19 @@ export function ZeroProvider({ children }: { children: ReactNode }) { } }, [userId]); + if (zeroConfigError) { + return ( +
+
+

Zero is required for local development.

+

+ {zeroConfigError} +

+
+
+ ); + } + const zero = useMemo(() => { if (!userId) { return null; From a5a940553130eb21b9521e6c231a16b82628e2fd Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Sun, 26 Apr 2026 02:48:50 -0400 Subject: [PATCH 2/4] Fix storage mode fallback and local file serving --- src/app/api/files/[...path]/route.ts | 82 ++++++++++++++++++++++++++++ src/lib/self-host-config.ts | 21 ++++++- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 src/app/api/files/[...path]/route.ts diff --git a/src/app/api/files/[...path]/route.ts b/src/app/api/files/[...path]/route.ts new file mode 100644 index 00000000..8d44bbd3 --- /dev/null +++ b/src/app/api/files/[...path]/route.ts @@ -0,0 +1,82 @@ +import { headers } from "next/headers"; +import { auth } from "@/lib/auth"; +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { extname, join, resolve, sep } from "node:path"; +import { NextRequest, NextResponse } from "next/server"; +import { withServerObservability } from "@/lib/with-server-observability"; + +export const dynamic = "force-dynamic"; + +const CONTENT_TYPES: Record = { + ".aac": "audio/aac", + ".avif": "image/avif", + ".gif": "image/gif", + ".heic": "image/heic", + ".heif": "image/heif", + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".json": "application/json", + ".md": "text/markdown; charset=utf-8", + ".mp4": "video/mp4", + ".pdf": "application/pdf", + ".png": "image/png", + ".svg": "image/svg+xml", + ".txt": "text/plain; charset=utf-8", + ".wav": "audio/wav", + ".webm": "video/webm", + ".webp": "image/webp", + ".zip": "application/zip", +}; + +function getContentType(filename: string): string { + return CONTENT_TYPES[extname(filename).toLowerCase()] ?? "application/octet-stream"; +} + +function isPathInsideDirectory(filePath: string, directory: string): boolean { + return filePath === directory || filePath.startsWith(`${directory}${sep}`); +} + +async function handleGET( + _request: NextRequest, + { params }: { params: Promise<{ path: string[] }> }, +) { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { path } = await params; + const relativePath = path.join("/"); + if ( + !relativePath || + path.some((segment) => !segment || segment === "." || segment === "..") + ) { + return NextResponse.json({ error: "Invalid file path" }, { status: 400 }); + } + + const uploadsDir = resolve(process.env.UPLOADS_DIR || join(process.cwd(), "uploads")); + const filePath = resolve(uploadsDir, relativePath); + if (!isPathInsideDirectory(filePath, uploadsDir)) { + return NextResponse.json({ error: "Invalid file path" }, { status: 400 }); + } + + if (!existsSync(filePath)) { + return NextResponse.json({ error: "File not found" }, { status: 404 }); + } + + const file = await readFile(filePath); + return new NextResponse(file, { + headers: { + "Content-Type": getContentType(relativePath), + "Cache-Control": "private, max-age=0, must-revalidate", + }, + }); +} + +export const GET = withServerObservability(handleGET, { + routeName: "GET /api/files/[...path]", +}); diff --git a/src/lib/self-host-config.ts b/src/lib/self-host-config.ts index 536b9999..554911c9 100644 --- a/src/lib/self-host-config.ts +++ b/src/lib/self-host-config.ts @@ -4,7 +4,26 @@ const ZERO_REQUIRED_MESSAGE = "ThinkEx requires Zero for workspace sync in self-hosted development. Set NEXT_PUBLIC_ZERO_SERVER and start the Zero cache server with `pnpm dev`."; export function getStorageMode(): StorageMode { - return process.env.STORAGE_TYPE === "supabase" ? "supabase" : "local"; + const configuredMode = process.env.STORAGE_TYPE?.trim().toLowerCase(); + if (configuredMode === "supabase") { + return "supabase"; + } + + if (configuredMode === "local") { + return "local"; + } + + // Preserve the pre-existing production behavior when STORAGE_TYPE has not + // been added yet but Supabase credentials are already configured. + if ( + process.env.NEXT_PUBLIC_SUPABASE_URL?.trim() && + (process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY?.trim() || + process.env.SUPABASE_SERVICE_ROLE_KEY?.trim()) + ) { + return "supabase"; + } + + return "local"; } export function usesLocalStorage(): boolean { From f71b91d904179cade358e5d46c6b7e7f355270c4 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Sun, 26 Apr 2026 03:40:10 -0400 Subject: [PATCH 3/4] Harden local file serving and upload guards --- scripts/dev-zero.mjs | 3 ++- setup.sh | 10 ++++++++-- src/app/api/files/[...path]/route.ts | 21 +++++++++++++++++---- src/app/api/upload-file/route.ts | 12 +++++++++++- src/app/api/upload-url/route.ts | 12 +++++++++++- src/lib/uploads/client-upload.ts | 9 +++++---- 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/scripts/dev-zero.mjs b/scripts/dev-zero.mjs index 532657ab..015287ca 100644 --- a/scripts/dev-zero.mjs +++ b/scripts/dev-zero.mjs @@ -20,6 +20,7 @@ const REQUIRED_ENV_KEYS = [ "ZERO_APP_PUBLICATIONS", "ZERO_ADMIN_PASSWORD", "ZERO_APP_ID", + "ZERO_COOKIE_DOMAIN", "NEXT_PUBLIC_ZERO_SERVER", ]; @@ -74,7 +75,7 @@ function failMissingEnv(keys) { function spawnZero() { const pnpmCmd = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; - const forwardedArgs = [...process.argv.slice(2)]; + const forwardedArgs = process.argv.slice(2); if (forwardedArgs[0] === "--") { forwardedArgs.shift(); } diff --git a/setup.sh b/setup.sh index 8bd2b1b0..1decbf55 100755 --- a/setup.sh +++ b/setup.sh @@ -38,15 +38,21 @@ get_env_value() { grep -E "^${key}=" .env 2>/dev/null | head -n1 | cut -d'=' -f2- } +escape_sed_replacement() { + printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g' +} + set_env_value() { local key=$1 local value=$2 + local escaped_value + escaped_value=$(escape_sed_replacement "$value") if grep -q "^${key}=" .env 2>/dev/null; then if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "s|^${key}=.*|${key}=${value}|" .env + sed -i '' "s|^${key}=.*|${key}=${escaped_value}|" .env else - sed -i "s|^${key}=.*|${key}=${value}|" .env + sed -i "s|^${key}=.*|${key}=${escaped_value}|" .env fi else echo "${key}=${value}" >> .env diff --git a/src/app/api/files/[...path]/route.ts b/src/app/api/files/[...path]/route.ts index 8d44bbd3..2fcee9aa 100644 --- a/src/app/api/files/[...path]/route.ts +++ b/src/app/api/files/[...path]/route.ts @@ -37,6 +37,22 @@ function isPathInsideDirectory(filePath: string, directory: string): boolean { return filePath === directory || filePath.startsWith(`${directory}${sep}`); } +function getDownloadHeaders(relativePath: string): Headers { + const contentType = getContentType(relativePath); + const headers = new Headers({ + "Cache-Control": "private, max-age=0, must-revalidate", + "Content-Type": contentType, + "X-Content-Type-Options": "nosniff", + }); + + if (contentType === "image/svg+xml") { + const filename = relativePath.split("/").at(-1) ?? "download.svg"; + headers.set("Content-Disposition", `attachment; filename="${filename}"`); + } + + return headers; +} + async function handleGET( _request: NextRequest, { params }: { params: Promise<{ path: string[] }> }, @@ -70,10 +86,7 @@ async function handleGET( const file = await readFile(filePath); return new NextResponse(file, { - headers: { - "Content-Type": getContentType(relativePath), - "Cache-Control": "private, max-age=0, must-revalidate", - }, + headers: getDownloadHeaders(relativePath), }); } diff --git a/src/app/api/upload-file/route.ts b/src/app/api/upload-file/route.ts index e88a1ec9..0b446bb9 100644 --- a/src/app/api/upload-file/route.ts +++ b/src/app/api/upload-file/route.ts @@ -9,7 +9,10 @@ import { getPreferredUploadContentType, getOfficeDocumentConvertUrl, } from "@/lib/uploads/office-document-validation"; -import { getStorageMode } from "@/lib/self-host-config"; +import { + getStorageMode, + getUnsupportedLocalStorageMessage, +} from "@/lib/self-host-config"; import { withServerObservability } from "@/lib/with-server-observability"; export const maxDuration = 30; @@ -118,6 +121,13 @@ export const POST = withServerObservability(async function POST(request: NextReq : `${timestamp}-${random}-${sanitizedName}`; if (storageMode === "local") { + if (isOfficeUpload) { + return NextResponse.json( + { error: getUnsupportedLocalStorageMessage("Document conversion") }, + { status: 400 }, + ); + } + const publicUrl = await saveFileLocally(file, filename); return NextResponse.json({ diff --git a/src/app/api/upload-url/route.ts b/src/app/api/upload-url/route.ts index 8a6f9bb9..2ab66221 100644 --- a/src/app/api/upload-url/route.ts +++ b/src/app/api/upload-url/route.ts @@ -3,7 +3,10 @@ import { auth } from "@/lib/auth"; import { createClient } from "@supabase/supabase-js"; import { NextRequest, NextResponse } from 'next/server'; import { getOfficeDocumentConvertUrlFromMeta } from "@/lib/uploads/office-document-validation"; -import { getStorageMode } from "@/lib/self-host-config"; +import { + getStorageMode, + getUnsupportedLocalStorageMessage, +} from "@/lib/self-host-config"; import { withServerObservability } from "@/lib/with-server-observability"; export const maxDuration = 10; @@ -41,6 +44,13 @@ async function handlePOST(request: NextRequest) { const storageMode = getStorageMode(); if (storageMode === "local") { + if (isOfficeUpload) { + return NextResponse.json( + { error: getUnsupportedLocalStorageMessage("Document conversion") }, + { status: 400 }, + ); + } + return NextResponse.json({ mode: "local", uploadUrl: "/api/upload-file", diff --git a/src/lib/uploads/client-upload.ts b/src/lib/uploads/client-upload.ts index 6ae801a2..75609be8 100644 --- a/src/lib/uploads/client-upload.ts +++ b/src/lib/uploads/client-upload.ts @@ -8,6 +8,7 @@ import { getPreferredUploadContentType, isOfficeDocument, } from "./office-document-validation"; +import { getUnsupportedLocalStorageMessage } from "@/lib/self-host-config"; const MAX_FILE_SIZE_BYTES = 200 * 1024 * 1024; // 200MB @@ -121,16 +122,16 @@ export async function uploadFileDirect( } if (urlData.mode === "local") { + if (isOfficeUpload) { + throw new Error(getUnsupportedLocalStorageMessage("Document conversion")); + } + const result = await uploadViaApiRoute(file); if (log) { const total = performance.now() - t0; console.info(`[PDF_UPLOAD] Local upload fallback: ${total.toFixed(0)}ms`); } - if (isOfficeUpload) { - return convertOfficeUpload(result.filename, result.url, file.name); - } - return result; } From c43bd22b25246e61ab416f6a8a049d5063d6e4cf Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Sun, 26 Apr 2026 17:25:05 -0400 Subject: [PATCH 4/4] Scope local files to the uploading user --- src/app/api/delete-file/route.ts | 33 +++++++++++++++- src/app/api/files/[...path]/route.ts | 9 +++++ src/app/api/upload-file/route.ts | 58 +++++++++++++++++++++++----- src/lib/zero/provider.tsx | 16 ++++---- 4 files changed, 97 insertions(+), 19 deletions(-) diff --git a/src/app/api/delete-file/route.ts b/src/app/api/delete-file/route.ts index 285e362a..237993a2 100644 --- a/src/app/api/delete-file/route.ts +++ b/src/app/api/delete-file/route.ts @@ -22,6 +22,31 @@ function extractFilename(url: string): string | null { return null; } +function parseOwnedLocalPath(filename: string): { ownerId: string; fileName: string } | null { + if (filename.includes("..") || filename.includes("\\")) { + return null; + } + + const segments = filename.split("/"); + if (segments.length !== 2) { + return null; + } + + const [ownerId, fileName] = segments; + if ( + !ownerId || + ownerId === "." || + ownerId === ".." || + !fileName || + fileName === "." || + fileName === ".." + ) { + return null; + } + + return { ownerId, fileName }; +} + export async function DELETE(request: NextRequest) { try { // Get authenticated user from Better Auth @@ -58,12 +83,16 @@ export async function DELETE(request: NextRequest) { } if (getStorageMode() === "local") { - if (filename.includes("/")) { + const parsedPath = parseOwnedLocalPath(filename); + if (!parsedPath) { return NextResponse.json({ error: "Invalid filename" }, { status: 400 }); } + if (parsedPath.ownerId !== session.user.id) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } const uploadsDir = process.env.UPLOADS_DIR || join(process.cwd(), "uploads"); - const filePath = join(uploadsDir, filename); + const filePath = join(uploadsDir, parsedPath.ownerId, parsedPath.fileName); if (!existsSync(filePath)) { return NextResponse.json({ diff --git a/src/app/api/files/[...path]/route.ts b/src/app/api/files/[...path]/route.ts index 2fcee9aa..a2a3d1f7 100644 --- a/src/app/api/files/[...path]/route.ts +++ b/src/app/api/files/[...path]/route.ts @@ -66,6 +66,11 @@ async function handleGET( } const { path } = await params; + if (path.length !== 2) { + return NextResponse.json({ error: "Invalid file path" }, { status: 400 }); + } + + const [ownerId] = path; const relativePath = path.join("/"); if ( !relativePath || @@ -74,6 +79,10 @@ async function handleGET( return NextResponse.json({ error: "Invalid file path" }, { status: 400 }); } + if (ownerId !== session.user.id) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + const uploadsDir = resolve(process.env.UPLOADS_DIR || join(process.cwd(), "uploads")); const filePath = resolve(uploadsDir, relativePath); if (!isPathInsideDirectory(filePath, uploadsDir)) { diff --git a/src/app/api/upload-file/route.ts b/src/app/api/upload-file/route.ts index 0b446bb9..a0ba5bfa 100644 --- a/src/app/api/upload-file/route.ts +++ b/src/app/api/upload-file/route.ts @@ -4,7 +4,7 @@ import { createClient } from "@supabase/supabase-js"; import { existsSync } from "node:fs"; import { mkdir, writeFile } from "node:fs/promises"; import { NextRequest, NextResponse } from "next/server"; -import { join } from "node:path"; +import { join, resolve, sep } from "node:path"; import { getPreferredUploadContentType, getOfficeDocumentConvertUrl, @@ -17,18 +17,57 @@ import { withServerObservability } from "@/lib/with-server-observability"; export const maxDuration = 30; -async function saveFileLocally(file: File, filename: string): Promise { - const uploadsDir = process.env.UPLOADS_DIR || join(process.cwd(), "uploads"); +function getUploadsDir(): string { + return resolve(process.env.UPLOADS_DIR || join(process.cwd(), "uploads")); +} + +function validateStorageSegment(value: string, label: string): string { + if ( + !value || + value === "." || + value === ".." || + value.includes("/") || + value.includes("\\") + ) { + throw new Error(`Invalid ${label}`); + } + + return value; +} + +function isPathInsideDirectory(filePath: string, directory: string): boolean { + return filePath === directory || filePath.startsWith(`${directory}${sep}`); +} + +async function saveFileLocally( + file: File, + userId: string, + filename: string, +): Promise<{ storagePath: string; publicUrl: string }> { + const uploadsDir = getUploadsDir(); + const ownerId = validateStorageSegment(userId, "user id"); + const safeFilename = validateStorageSegment(filename, "filename"); + const targetDir = resolve(uploadsDir, ownerId); + const filePath = resolve(targetDir, safeFilename); + + if (!isPathInsideDirectory(targetDir, uploadsDir)) { + throw new Error("Invalid user storage path"); + } + if (!isPathInsideDirectory(filePath, uploadsDir)) { + throw new Error("Invalid file storage path"); + } - if (!existsSync(uploadsDir)) { - await mkdir(uploadsDir, { recursive: true }); + if (!existsSync(targetDir)) { + await mkdir(targetDir, { recursive: true }); } const bytes = await file.arrayBuffer(); - await writeFile(join(uploadsDir, filename), Buffer.from(bytes)); + await writeFile(filePath, Buffer.from(bytes)); const appUrl = process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"; - return `${appUrl}/api/files/${filename}`; + const storagePath = `${ownerId}/${safeFilename}`; + const publicUrl = `${appUrl}/api/files/${encodeURIComponent(ownerId)}/${encodeURIComponent(safeFilename)}`; + return { storagePath, publicUrl }; } export const POST = withServerObservability(async function POST(request: NextRequest) { @@ -44,6 +83,7 @@ export const POST = withServerObservability(async function POST(request: NextReq { status: 401 } ); } + const userId = session.user.id; // Get file from form data const formData = await request.formData(); @@ -128,12 +168,12 @@ export const POST = withServerObservability(async function POST(request: NextReq ); } - const publicUrl = await saveFileLocally(file, filename); + const { publicUrl, storagePath } = await saveFileLocally(file, userId, filename); return NextResponse.json({ success: true, url: publicUrl, - filename, + filename: storagePath, }); } diff --git a/src/lib/zero/provider.tsx b/src/lib/zero/provider.tsx index b61388ee..b171796c 100644 --- a/src/lib/zero/provider.tsx +++ b/src/lib/zero/provider.tsx @@ -18,6 +18,14 @@ export function ZeroProvider({ children }: { children: ReactNode }) { } }, [userId]); + const zero = useMemo(() => { + if (zeroConfigError || !userId) { + return null; + } + + return getZero({ userId }); + }, [userId, zeroConfigError]); + if (zeroConfigError) { return (
@@ -31,14 +39,6 @@ export function ZeroProvider({ children }: { children: ReactNode }) { ); } - const zero = useMemo(() => { - if (!userId) { - return null; - } - - return getZero({ userId }); - }, [userId]); - if (isPending || !zero) { // Show a minimal loading state instead of null to avoid // blank screen flash during session check / anonymous session creation.