diff --git a/README.md b/README.md index 518ed882..537988c0 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ rli blueprint from-dockerfile # Create a blueprint from a Dockerfile rli object list # List objects rli object get # Get object details rli object download # Download object to local file -rli object upload # Upload a file as an object +rli object upload # Upload file(s) or directory as an obj... rli object delete # Delete an object (irreversible) ``` diff --git a/package.json b/package.json index 78d0cc83..5a6a01a6 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "ink-link": "5.0.0", "ink-spinner": "5.0.0", "ink-text-input": "6.0.0", + "nanotar": "^0.3.0", "react": "19.2.0", "yaml": "2.8.3", "zustand": "5.0.10" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ada46644..465c79fa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: ink-text-input: specifier: 6.0.0 version: 6.0.0(ink@6.6.0(@types/react@19.2.10)(react@19.2.0))(react@19.2.0) + nanotar: + specifier: ^0.3.0 + version: 0.3.0 react: specifier: 19.2.0 version: 19.2.0 @@ -2358,6 +2361,9 @@ packages: resolution: {integrity: sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + nanotar@0.3.0: + resolution: {integrity: sha512-Kv2JYYiCzt16Kt5QwAc9BFG89xfPNBx+oQL4GQXD9nLqPkZBiNaqaCWtwnbk/q7UVsTYevvM1b0UF8zmEI4pCg==} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -5914,6 +5920,8 @@ snapshots: mute-stream@1.0.0: {} + nanotar@0.3.0: {} + natural-compare@1.4.0: {} negotiator@1.0.0: {} diff --git a/src/commands/object/upload.ts b/src/commands/object/upload.ts index 4ce9dbcb..b5f4fde1 100644 --- a/src/commands/object/upload.ts +++ b/src/commands/object/upload.ts @@ -2,13 +2,15 @@ * Upload object command */ -import { readFile, stat } from "fs/promises"; -import { extname } from "path"; +import { lstat, readFile, readdir } from "fs/promises"; +import { dirname, extname, relative, resolve, sep } from "path"; +import { createTar, createTarGzip } from "nanotar"; +import type { TarFileInput } from "nanotar"; import { getClient } from "../../utils/client.js"; import { output, outputError } from "../../utils/output.js"; interface UploadObjectOptions { - path: string; + paths: string[]; name: string; contentType?: string; public?: boolean; @@ -32,24 +34,212 @@ const CONTENT_TYPE_MAP: Record = { ".tar.gz": "tgz", }; +/** + * Recursively collect all files and directories under the given paths into + * nanotar entries. Normalizes permissions: uid/gid 1000, directories and + * executable files get mode 755, everything else gets 644. Preserves mtime. + * + * Entry names are always relative to `archiveRoot` and never contain leading + * `../` segments, preventing path traversal in the generated archive. + */ +async function collectEntries( + paths: string[], + archiveRoot: string, + precomputedStats?: Map>>, +): Promise { + const entries: TarFileInput[] = []; + + for (const p of paths) { + const absPath = resolve(p); + let relPath = relative(archiveRoot, absPath); + + // Guard against path traversal: entry names must not escape the archive root + if (relPath.startsWith("..")) { + throw new Error( + `Path "${absPath}" is outside the archive root "${archiveRoot}". All paths must share a common ancestor directory.`, + ); + } + + let stats = precomputedStats?.get(absPath); + if (!stats) { + try { + stats = await lstat(absPath); + } catch (err) { + throw new Error(`Cannot read path: ${relPath}`, { cause: err }); + } + } + + if (stats.isSymbolicLink()) { + throw new Error( + `Path is a symlink: ${relPath}. Resolve the symlink or pass the target path directly.`, + ); + } + + if (stats.isDirectory()) { + entries.push({ + name: relPath.endsWith("/") ? relPath : relPath + "/", + attrs: { + mode: "755", + uid: 1000, + gid: 1000, + // nanotar expects mtime in milliseconds and converts to seconds internally + mtime: Number(stats.mtimeMs), + }, + }); + const children = (await readdir(absPath)).sort(); + const childPaths = children.map((c) => resolve(absPath, c)); + if (childPaths.length > 0) { + entries.push(...(await collectEntries(childPaths, archiveRoot))); + } + } else { + const isExecutable = (Number(stats.mode) & 0o111) !== 0; + let data; + try { + data = await readFile(absPath); + } catch (err) { + throw new Error(`Cannot read file: ${relPath}`, { cause: err }); + } + entries.push({ + name: relPath, + data, + attrs: { + mode: isExecutable ? "755" : "644", + uid: 1000, + gid: 1000, + // nanotar expects mtime in milliseconds and converts to seconds internally + mtime: Number(stats.mtimeMs), + }, + }); + } + } + + return entries; +} + +/** + * Compute the deepest common directory for a list of absolute paths. + * Used as the archive root so entry names are always relative and safe. + */ +function commonAncestor(absPaths: string[]): string { + if (absPaths.length === 0) return process.cwd(); + if (absPaths.length === 1) return dirname(absPaths[0]); + const parts = absPaths.map((p) => p.split(sep)); + const common: string[] = []; + for (let i = 0; i < parts[0].length; i++) { + const segment = parts[0][i]; + if (parts.every((p) => p[i] === segment)) { + common.push(segment); + } else { + break; + } + } + return common.join(sep) || sep; +} + +/** + * Create a tar (or tgz) archive as a Buffer from the given filesystem paths. + * + * Walks directories recursively and normalizes all entries to uid/gid 1000, + * mode 644 (non-executable files) or 755 (executable files and directories). + * Entry names are relative to the common ancestor of the provided paths. + */ +export async function createTarBuffer( + paths: string[], + gzip: boolean, + precomputedStats?: Map>>, +): Promise { + const absPaths = paths.map((p) => resolve(p)); + const archiveRoot = commonAncestor(absPaths); + const entries = await collectEntries(paths, archiveRoot, precomputedStats); + + if (gzip) { + const data = await createTarGzip(entries); + return Buffer.from(data); + } + + const data = createTar(entries); + return Buffer.from(data); +} + export async function uploadObject(options: UploadObjectOptions) { try { const client = getClient(); + const { paths, name, contentType, output: outputFormat } = options; + + if (paths.length === 0) { + outputError("At least one path is required"); + return; + } - // Check if file exists and get stats - const stats = await stat(options.path); - const fileBuffer = await readFile(options.path); + // Validate all paths exist (use lstat to match collectEntries and detect symlinks) + // Key by resolved absolute path so collectEntries can reuse stats + const statsMap = new Map>>(); + for (const p of paths) { + try { + const s = await lstat(p); + if (s.isSymbolicLink()) { + outputError( + `Path is a symlink: ${p}. Resolve the symlink or pass the target path directly.`, + ); + return; + } + statsMap.set(resolve(p), s); + } catch { + outputError(`Path does not exist: ${p}`); + return; + } + } + + const isTarType = contentType === "tar" || contentType === "tgz"; + const isSinglePath = paths.length === 1; + const firstStats = isSinglePath + ? statsMap.get(resolve(paths[0]))! + : undefined; + const singleIsDir = isSinglePath && firstStats!.isDirectory(); + + // Multi-path requires tar/tgz content type + if (paths.length > 1 && !isTarType) { + outputError( + "Multiple paths require --content-type tar or --content-type tgz", + ); + return; + } + + // Directory without tar/tgz type + if (singleIsDir && !isTarType) { + outputError( + "Cannot upload a directory directly. Use --content-type tar or --content-type tgz to create an archive.", + ); + return; + } + + let fileBuffer: Buffer; + let detectedContentType: ContentType; + let fileSize: number; + + const shouldCreateArchive = isTarType && (paths.length > 1 || singleIsDir); + + if (shouldCreateArchive) { + const gzip = contentType === "tgz"; + fileBuffer = await createTarBuffer(paths, gzip, statsMap); + detectedContentType = contentType as ContentType; + fileSize = fileBuffer.length; + } else { + // Single file upload (existing behavior) + const filePath = paths[0]; + fileBuffer = await readFile(filePath); + fileSize = fileBuffer.length; - // Auto-detect content type if not provided - let detectedContentType: ContentType = options.contentType as ContentType; - if (!detectedContentType) { - const ext = extname(options.path).toLowerCase(); - detectedContentType = CONTENT_TYPE_MAP[ext] || "unspecified"; + detectedContentType = contentType as ContentType; + if (!detectedContentType) { + const ext = extname(filePath).toLowerCase(); + detectedContentType = CONTENT_TYPE_MAP[ext] || "unspecified"; + } } // Step 1: Create the object const createResponse = await client.objects.create({ - name: options.name, + name, content_type: detectedContentType, }); @@ -71,16 +261,16 @@ export async function uploadObject(options: UploadObjectOptions) { const result = { id: createResponse.id, - name: options.name, + name, contentType: detectedContentType, - size: stats.size, + size: fileSize, }; // Default: just output the ID for easy scripting - if (!options.output || options.output === "text") { + if (!outputFormat || outputFormat === "text") { console.log(result.id); } else { - output(result, { format: options.output, defaultFormat: "json" }); + output(result, { format: outputFormat, defaultFormat: "json" }); } } catch (error) { outputError("Failed to upload object", error); diff --git a/src/utils/commands.ts b/src/utils/commands.ts index 70f920d8..ff9bd0e6 100644 --- a/src/utils/commands.ts +++ b/src/utils/commands.ts @@ -648,8 +648,10 @@ export function createProgram(): Command { }); object - .command("upload ") - .description("Upload a file as an object") + .command("upload ") + .description( + "Upload file(s) or directory as an object. Multiple paths with --content-type tar|tgz creates an archive.", + ) .option("--name ", "Object name (required)") .option( "--content-type ", @@ -660,14 +662,14 @@ export function createProgram(): Command { "-o, --output [format]", "Output format: text|json|yaml (default: text)", ) - .action(async (path, options) => { + .action(async (paths, options) => { const { uploadObject } = await import("../commands/object/upload.js"); if (!options.output) { const { runInteractiveCommand } = await import("../utils/interactiveCommand.js"); - await runInteractiveCommand(() => uploadObject({ path, ...options })); + await runInteractiveCommand(() => uploadObject({ paths, ...options })); } else { - await uploadObject({ path, ...options }); + await uploadObject({ paths, ...options }); } }); diff --git a/tests/__tests__/commands/object/upload.test.ts b/tests/__tests__/commands/object/upload.test.ts new file mode 100644 index 00000000..0e00c9e9 --- /dev/null +++ b/tests/__tests__/commands/object/upload.test.ts @@ -0,0 +1,330 @@ +/** + * Tests for object upload command - tar/tgz archive creation and validation + */ + +import { jest, describe, it, expect, beforeEach, afterEach } from "@jest/globals"; +import { mkdtemp, writeFile, mkdir, rm, chmod, utimes, symlink } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import { parseTar } from "nanotar"; + +// Mock client and output +const mockCreate = jest.fn(); +const mockComplete = jest.fn(); +jest.unstable_mockModule("@/utils/client.js", () => ({ + getClient: () => ({ + objects: { + create: mockCreate, + complete: mockComplete, + }, + }), +})); + +const mockOutput = jest.fn(); +const mockOutputError = jest.fn(); +jest.unstable_mockModule("@/utils/output.js", () => ({ + output: mockOutput, + outputError: mockOutputError, +})); + +// Mock fetch for upload +const mockFetch = jest.fn(); +globalThis.fetch = mockFetch; + +describe("createTarBuffer", () => { + let testDir: string; + + beforeEach(async () => { + jest.clearAllMocks(); + testDir = await mkdtemp(join(tmpdir(), "rl-upload-test-")); + }); + + afterEach(async () => { + await rm(testDir, { recursive: true, force: true }); + }); + + it("creates a valid tar buffer from multiple files", async () => { + await writeFile(join(testDir, "a.txt"), "hello"); + await writeFile(join(testDir, "b.txt"), "world"); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer( + [join(testDir, "a.txt"), join(testDir, "b.txt")], + false, + ); + + expect(buffer).toBeInstanceOf(Buffer); + expect(buffer.length).toBeGreaterThan(0); + + const entries = parseTar(buffer); + const names = entries.map((e) => e.name); + expect(names).toHaveLength(2); + expect(names.some((n) => n.endsWith("a.txt"))).toBe(true); + expect(names.some((n) => n.endsWith("b.txt"))).toBe(true); + }); + + it("creates a valid tgz buffer (gzipped)", async () => { + await writeFile(join(testDir, "file.txt"), "compressed content"); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer( + [join(testDir, "file.txt")], + true, + ); + + expect(buffer).toBeInstanceOf(Buffer); + // Gzip magic bytes: 0x1f 0x8b + expect(buffer[0]).toBe(0x1f); + expect(buffer[1]).toBe(0x8b); + }); + + it("creates a tar from a directory", async () => { + const subDir = join(testDir, "mydir"); + await mkdir(subDir); + await writeFile(join(subDir, "nested.txt"), "nested content"); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer([subDir], false); + + expect(buffer).toBeInstanceOf(Buffer); + expect(buffer.length).toBeGreaterThan(0); + + const entries = parseTar(buffer); + const dirEntry = entries.find((e) => e.name.endsWith("mydir/")); + const fileEntry = entries.find((e) => e.name.endsWith("nested.txt")); + expect(dirEntry).toBeDefined(); + expect(fileEntry).toBeDefined(); + }); + + it("normalizes uid/gid to 1000 for all entries", async () => { + await writeFile(join(testDir, "file.txt"), "content"); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer([join(testDir, "file.txt")], false); + + // Verify uid/gid by reading the raw tar header bytes directly. + // nanotar's parseTar has an octal parsing quirk, so read the raw field. + const uid = buffer.toString("ascii", 108, 115).replace(/\0/g, "").trim(); + const gid = buffer.toString("ascii", 116, 123).replace(/\0/g, "").trim(); + expect(parseInt(uid, 8)).toBe(1000); + expect(parseInt(gid, 8)).toBe(1000); + }); + + it("sets mode 644 for non-executable files and 755 for executable files", async () => { + const normalFile = join(testDir, "normal.txt"); + const execFile = join(testDir, "script.sh"); + await writeFile(normalFile, "data"); + await writeFile(execFile, "#!/bin/sh\necho hi"); + await chmod(execFile, 0o755); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer([normalFile, execFile], false); + + const entries = parseTar(buffer); + const normal = entries.find((e) => e.name.endsWith("normal.txt")); + const exec = entries.find((e) => e.name.endsWith("script.sh")); + expect(normal?.attrs?.mode).toContain("644"); + expect(exec?.attrs?.mode).toContain("755"); + }); + + it("sets mode 755 for directories", async () => { + const subDir = join(testDir, "subdir"); + await mkdir(subDir); + await writeFile(join(subDir, "file.txt"), "content"); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer([subDir], false); + + const entries = parseTar(buffer); + const dir = entries.find((e) => e.name.endsWith("subdir/")); + expect(dir?.attrs?.mode).toContain("755"); + }); + + it("errors on symlinks inside a directory tree", async () => { + const subDir = join(testDir, "with-symlink"); + await mkdir(subDir); + await writeFile(join(subDir, "real.txt"), "real content"); + await symlink(join(subDir, "real.txt"), join(subDir, "link.txt")); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + await expect(createTarBuffer([subDir], false)).rejects.toThrow( + /symlink/i, + ); + }); + + it("preserves mtime from the filesystem", async () => { + const filePath = join(testDir, "dated.txt"); + await writeFile(filePath, "content"); + // Set a known mtime: 2024-01-15T00:00:00Z + const knownTime = new Date("2024-01-15T00:00:00Z"); + await utimes(filePath, knownTime, knownTime); + + const { createTarBuffer } = await import("@/commands/object/upload.js"); + const buffer = await createTarBuffer([filePath], false); + + const entries = parseTar(buffer); + const entry = entries.find((e) => e.name.endsWith("dated.txt")); + expect(entry?.attrs?.mtime).toBeDefined(); + // parseTar returns mtime in seconds (raw tar format) + const expectedSec = Math.floor(knownTime.getTime() / 1000); + expect(entry?.attrs?.mtime).toBe(expectedSec); + }); +}); + +describe("uploadObject", () => { + let testDir: string; + + beforeEach(async () => { + jest.clearAllMocks(); + testDir = await mkdtemp(join(tmpdir(), "rl-upload-test-")); + mockCreate.mockResolvedValue({ + id: "obj_test123", + upload_url: "https://example.com/upload", + }); + mockFetch.mockResolvedValue({ ok: true } as Response); + mockComplete.mockResolvedValue({}); + }); + + afterEach(async () => { + await rm(testDir, { recursive: true, force: true }); + }); + + it("uploads a single file with existing behavior", async () => { + const filePath = join(testDir, "test.txt"); + await writeFile(filePath, "test content"); + const logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [filePath], + name: "test-object", + }); + + logSpy.mockRestore(); + + expect(mockCreate).toHaveBeenCalledWith({ + name: "test-object", + content_type: "text", + }); + expect(mockFetch).toHaveBeenCalled(); + expect(mockComplete).toHaveBeenCalledWith("obj_test123"); + }); + + it("errors when multiple paths given without tar/tgz content type", async () => { + const file1 = join(testDir, "a.txt"); + const file2 = join(testDir, "b.txt"); + await writeFile(file1, "a"); + await writeFile(file2, "b"); + + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [file1, file2], + name: "test-object", + contentType: "text", + }); + + expect(mockOutputError).toHaveBeenCalledWith( + "Multiple paths require --content-type tar or --content-type tgz", + ); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("errors when directory given without tar/tgz content type", async () => { + const dir = join(testDir, "mydir"); + await mkdir(dir); + + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [dir], + name: "test-object", + }); + + expect(mockOutputError).toHaveBeenCalledWith( + "Cannot upload a directory directly. Use --content-type tar or --content-type tgz to create an archive.", + ); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("errors when path does not exist", async () => { + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [join(testDir, "nonexistent.txt")], + name: "test-object", + }); + + expect(mockOutputError).toHaveBeenCalledWith( + expect.stringContaining("Path does not exist"), + ); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("creates tar archive from multiple files when content type is tar", async () => { + const file1 = join(testDir, "a.txt"); + const file2 = join(testDir, "b.txt"); + await writeFile(file1, "file a"); + await writeFile(file2, "file b"); + const logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [file1, file2], + name: "multi-file-archive", + contentType: "tar", + }); + + logSpy.mockRestore(); + + expect(mockCreate).toHaveBeenCalledWith({ + name: "multi-file-archive", + content_type: "tar", + }); + expect(mockFetch).toHaveBeenCalled(); + expect(mockComplete).toHaveBeenCalledWith("obj_test123"); + }); + + it("creates tgz archive from a directory when content type is tgz", async () => { + const dir = join(testDir, "mydir"); + await mkdir(dir); + await writeFile(join(dir, "file.txt"), "content"); + const logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [dir], + name: "dir-archive", + contentType: "tgz", + }); + + logSpy.mockRestore(); + + expect(mockCreate).toHaveBeenCalledWith({ + name: "dir-archive", + content_type: "tgz", + }); + expect(mockFetch).toHaveBeenCalled(); + }); + + it("uploads single tar file as-is without creating archive", async () => { + const filePath = join(testDir, "existing.tar"); + await writeFile(filePath, "fake tar content"); + const logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + + const { uploadObject } = await import("@/commands/object/upload.js"); + await uploadObject({ + paths: [filePath], + name: "existing-archive", + contentType: "tar", + }); + + logSpy.mockRestore(); + + expect(mockCreate).toHaveBeenCalledWith({ + name: "existing-archive", + content_type: "tar", + }); + // Should upload the raw file content, not create a tar of the tar + const fetchCall = mockFetch.mock.calls[0]; + const body = fetchCall[1]?.body as Buffer; + expect(body.toString()).toBe("fake tar content"); + }); +});