diff --git a/.changeset/add-doc-fetch-command.md b/.changeset/add-doc-fetch-command.md new file mode 100644 index 00000000000..875f321450d --- /dev/null +++ b/.changeset/add-doc-fetch-command.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli': minor +--- + +Add `shopify doc fetch` to download Markdown documents from shopify.dev. diff --git a/docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts b/docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts new file mode 100644 index 00000000000..5137929f611 --- /dev/null +++ b/docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts @@ -0,0 +1,30 @@ +// This is an autogenerated file. Don't edit this file manually. +/** + * The following flags are available for the `doc fetch` command: + * @publicDocs + */ +export interface docfetch { + /** + * Disable color output. + * @environment SHOPIFY_FLAG_NO_COLOR + */ + '--no-color'?: '' + + /** + * Write the document to this file path instead of printing it to stdout. + * @environment SHOPIFY_FLAG_OUTPUT + */ + '--output '?: string + + /** + * The shopify.dev URL to fetch. + * @environment SHOPIFY_FLAG_URL + */ + '--url ': string + + /** + * Increase the verbosity of the output. + * @environment SHOPIFY_FLAG_VERBOSE + */ + '--verbose'?: '' +} diff --git a/docs-shopify.dev/commands/interfaces/search.interface.ts b/docs-shopify.dev/commands/interfaces/search.interface.ts index 041775f0570..618c86f1f7f 100644 --- a/docs-shopify.dev/commands/interfaces/search.interface.ts +++ b/docs-shopify.dev/commands/interfaces/search.interface.ts @@ -4,5 +4,15 @@ * @publicDocs */ export interface search { + /** + * Disable color output. + * @environment SHOPIFY_FLAG_NO_COLOR + */ + '--no-color'?: '' + /** + * Increase the verbosity of the output. + * @environment SHOPIFY_FLAG_VERBOSE + */ + '--verbose'?: '' } diff --git a/docs-shopify.dev/generated/generated_docs_data_v2.json b/docs-shopify.dev/generated/generated_docs_data_v2.json index e1c8ee7cc63..9924ab4f4a9 100644 --- a/docs-shopify.dev/generated/generated_docs_data_v2.json +++ b/docs-shopify.dev/generated/generated_docs_data_v2.json @@ -2742,6 +2742,52 @@ "value": "export interface configautoupgradestatus {\n\n}" } }, + "docfetch": { + "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts": { + "filePath": "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts", + "name": "docfetch", + "description": "The following flags are available for the `doc fetch` command:", + "isPublicDocs": true, + "members": [ + { + "filePath": "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts", + "syntaxKind": "PropertySignature", + "name": "--no-color", + "value": "''", + "description": "Disable color output.", + "isOptional": true, + "environmentValue": "SHOPIFY_FLAG_NO_COLOR" + }, + { + "filePath": "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts", + "syntaxKind": "PropertySignature", + "name": "--output ", + "value": "string", + "description": "Write the document to this file path instead of printing it to stdout.", + "isOptional": true, + "environmentValue": "SHOPIFY_FLAG_OUTPUT" + }, + { + "filePath": "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts", + "syntaxKind": "PropertySignature", + "name": "--url ", + "value": "string", + "description": "The shopify.dev URL to fetch.", + "environmentValue": "SHOPIFY_FLAG_URL" + }, + { + "filePath": "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts", + "syntaxKind": "PropertySignature", + "name": "--verbose", + "value": "''", + "description": "Increase the verbosity of the output.", + "isOptional": true, + "environmentValue": "SHOPIFY_FLAG_VERBOSE" + } + ], + "value": "export interface docfetch {\n /**\n * Disable color output.\n * @environment SHOPIFY_FLAG_NO_COLOR\n */\n '--no-color'?: ''\n\n /**\n * Write the document to this file path instead of printing it to stdout.\n * @environment SHOPIFY_FLAG_OUTPUT\n */\n '--output '?: string\n\n /**\n * The shopify.dev URL to fetch.\n * @environment SHOPIFY_FLAG_URL\n */\n '--url ': string\n\n /**\n * Increase the verbosity of the output.\n * @environment SHOPIFY_FLAG_VERBOSE\n */\n '--verbose'?: ''\n}" + } + }, "docsearch": { "docs-shopify.dev/commands/interfaces/doc-search.interface.ts": { "filePath": "docs-shopify.dev/commands/interfaces/doc-search.interface.ts", @@ -4181,8 +4227,27 @@ "name": "search", "description": "The following flags are available for the `search` command:", "isPublicDocs": true, - "members": [], - "value": "export interface search {\n\n}" + "members": [ + { + "filePath": "docs-shopify.dev/commands/interfaces/search.interface.ts", + "syntaxKind": "PropertySignature", + "name": "--no-color", + "value": "''", + "description": "Disable color output.", + "isOptional": true, + "environmentValue": "SHOPIFY_FLAG_NO_COLOR" + }, + { + "filePath": "docs-shopify.dev/commands/interfaces/search.interface.ts", + "syntaxKind": "PropertySignature", + "name": "--verbose", + "value": "''", + "description": "Increase the verbosity of the output.", + "isOptional": true, + "environmentValue": "SHOPIFY_FLAG_VERBOSE" + } + ], + "value": "export interface search {\n /**\n * Disable color output.\n * @environment SHOPIFY_FLAG_NO_COLOR\n */\n '--no-color'?: ''\n\n /**\n * Increase the verbosity of the output.\n * @environment SHOPIFY_FLAG_VERBOSE\n */\n '--verbose'?: ''\n}" } }, "storeauth": { diff --git a/packages/cli/README.md b/packages/cli/README.md index 7acacf69324..f679fd449dd 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -39,6 +39,7 @@ * [`shopify config autoupgrade off`](#shopify-config-autoupgrade-off) * [`shopify config autoupgrade on`](#shopify-config-autoupgrade-on) * [`shopify config autoupgrade status`](#shopify-config-autoupgrade-status) +* [`shopify doc fetch`](#shopify-doc-fetch) * [`shopify doc search`](#shopify-doc-search) * [`shopify help [command] [flags]`](#shopify-help-command-flags) * [`shopify hydrogen build`](#shopify-hydrogen-build) @@ -1214,6 +1215,35 @@ DESCRIPTION Run `shopify config autoupgrade on` or `shopify config autoupgrade off` to configure it. ``` +## `shopify doc fetch` + +Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like a centrally-served skill. For finding the relevant pieces of content across shopify.dev instead, use `doc search`. + +``` +USAGE + $ shopify doc fetch --url [--no-color] [--output ] [--verbose] + +FLAGS + --no-color [env: SHOPIFY_FLAG_NO_COLOR] Disable color output. + --output= [env: SHOPIFY_FLAG_OUTPUT] Write the document to this file path instead of printing it to stdout. + --url= (required) [env: SHOPIFY_FLAG_URL] The shopify.dev URL to fetch. + --verbose [env: SHOPIFY_FLAG_VERBOSE] Increase the verbosity of the output. + +DESCRIPTION + Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this + tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like + a centrally-served skill. For finding the relevant pieces of content across shopify.dev instead, use `doc search`. + +EXAMPLES + # fetch the Markdown version of a Shopify.dev page + + $ shopify doc fetch --url https://shopify.dev/docs/api/shopify-cli + + # save the document to a file instead of printing it + + $ shopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --output docs/shopify-cli.md +``` + ## `shopify doc search` Query the shopify.dev vector store and print the most relevant documentation chunks as JSON. Best for programmatic discovery — surfacing the relevant pieces of documentation for a topic, rather than retrieving a whole document. To download a full document verbatim, use `doc fetch`. @@ -2114,14 +2144,20 @@ DESCRIPTION ## `shopify search [query]` -Starts a search on shopify.dev. +Search shopify.dev for the most relevant content matching a query. Best for discovery — surfacing the relevant pieces of documentation for a topic, rather than retrieving a whole document. To download a full document verbatim, use `doc fetch`. ``` USAGE $ shopify search [query] +FLAGS + --no-color [env: SHOPIFY_FLAG_NO_COLOR] Disable color output. + --verbose [env: SHOPIFY_FLAG_VERBOSE] Increase the verbosity of the output. + DESCRIPTION - Starts a search on shopify.dev. + Search shopify.dev for the most relevant content matching a query. Best for discovery — surfacing the relevant pieces + of documentation for a topic, rather than retrieving a whole document. To download a full document verbatim, use `doc + fetch`. EXAMPLES # open the search modal on Shopify.dev diff --git a/packages/cli/oclif.manifest.json b/packages/cli/oclif.manifest.json index 66642ffcfde..b7f25b77665 100644 --- a/packages/cli/oclif.manifest.json +++ b/packages/cli/oclif.manifest.json @@ -3435,6 +3435,61 @@ "strict": true, "summary": "Watch and prints out changes to an app." }, + "doc:fetch": { + "aliases": [ + ], + "args": { + }, + "description": "Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like a centrally-served skill. For finding the relevant pieces of content across shopify.dev instead, use `doc search`.", + "enableJsonFlag": false, + "examples": [ + "# fetch the Markdown version of a Shopify.dev page\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli", + "# save the document to a file instead of printing it\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --output docs/shopify-cli.md" + ], + "flags": { + "no-color": { + "allowNo": false, + "description": "Disable color output.", + "env": "SHOPIFY_FLAG_NO_COLOR", + "hidden": false, + "name": "no-color", + "type": "boolean" + }, + "output": { + "description": "Write the document to this file path instead of printing it to stdout.", + "env": "SHOPIFY_FLAG_OUTPUT", + "hasDynamicHelp": false, + "multiple": false, + "name": "output", + "type": "option" + }, + "url": { + "description": "The shopify.dev URL to fetch.", + "env": "SHOPIFY_FLAG_URL", + "hasDynamicHelp": false, + "multiple": false, + "name": "url", + "required": true, + "type": "option" + }, + "verbose": { + "allowNo": false, + "description": "Increase the verbosity of the output.", + "env": "SHOPIFY_FLAG_VERBOSE", + "hidden": false, + "name": "verbose", + "type": "boolean" + } + }, + "hasDynamicHelp": false, + "hiddenAliases": [ + ], + "id": "doc:fetch", + "pluginAlias": "@shopify/cli", + "pluginName": "@shopify/cli", + "pluginType": "core", + "strict": true + }, "doc:search": { "aliases": [ ], @@ -5725,12 +5780,28 @@ "name": "query" } }, - "description": "Starts a search on shopify.dev.", + "description": "Search shopify.dev for the most relevant content matching a query. Best for discovery — surfacing the relevant pieces of documentation for a topic, rather than retrieving a whole document. To download a full document verbatim, use `doc fetch`.", "enableJsonFlag": false, "examples": [ "# open the search modal on Shopify.dev\n shopify search\n\n # search for a term on Shopify.dev\n shopify search \n\n # search for a phrase on Shopify.dev\n shopify search \"\"\n " ], "flags": { + "no-color": { + "allowNo": false, + "description": "Disable color output.", + "env": "SHOPIFY_FLAG_NO_COLOR", + "hidden": false, + "name": "no-color", + "type": "boolean" + }, + "verbose": { + "allowNo": false, + "description": "Increase the verbosity of the output.", + "env": "SHOPIFY_FLAG_VERBOSE", + "hidden": false, + "name": "verbose", + "type": "boolean" + } }, "hasDynamicHelp": false, "hiddenAliases": [ diff --git a/packages/cli/src/cli/commands/doc/fetch.ts b/packages/cli/src/cli/commands/doc/fetch.ts new file mode 100644 index 00000000000..089915af01b --- /dev/null +++ b/packages/cli/src/cli/commands/doc/fetch.ts @@ -0,0 +1,34 @@ +import {docFetchService} from '../../services/commands/doc/fetch.js' +import Command from '@shopify/cli-kit/node/base-command' +import {globalFlags} from '@shopify/cli-kit/node/cli' +import {Flags} from '@oclif/core' + +export default class DocFetch extends Command { + static description = + 'Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like a centrally-served skill. For finding the relevant pieces of content across shopify.dev instead, use `doc search`.' + + static examples = [ + `# fetch the Markdown version of a Shopify.dev page +shopify doc fetch --url https://shopify.dev/docs/api/shopify-cli`, + `# save the document to a file instead of printing it +shopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --output docs/shopify-cli.md`, + ] + + static flags = { + ...globalFlags, + url: Flags.string({ + description: 'The shopify.dev URL to fetch.', + env: 'SHOPIFY_FLAG_URL', + required: true, + }), + output: Flags.string({ + description: 'Write the document to this file path instead of printing it to stdout.', + env: 'SHOPIFY_FLAG_OUTPUT', + }), + } + + async run(): Promise { + const {flags} = await this.parse(DocFetch) + await docFetchService(flags.url, flags.output) + } +} diff --git a/packages/cli/src/cli/commands/search.ts b/packages/cli/src/cli/commands/search.ts index bb1346b8341..07d96c3d75c 100644 --- a/packages/cli/src/cli/commands/search.ts +++ b/packages/cli/src/cli/commands/search.ts @@ -1,9 +1,11 @@ import {searchService} from '../services/commands/search.js' import Command from '@shopify/cli-kit/node/base-command' +import {globalFlags} from '@shopify/cli-kit/node/cli' import {Args} from '@oclif/core' export default class Search extends Command { - static description = 'Starts a search on shopify.dev.' + static description = + 'Search shopify.dev for the most relevant content matching a query. Best for discovery — surfacing the relevant pieces of documentation for a topic, rather than retrieving a whole document. To download a full document verbatim, use `doc fetch`.' static usage = `search [query]` @@ -23,6 +25,10 @@ export default class Search extends Command { query: Args.string(), } + static flags = { + ...globalFlags, + } + async run(): Promise { const {args} = await this.parse(Search) await searchService(args.query) diff --git a/packages/cli/src/cli/services/commands/doc/fetch.test.ts b/packages/cli/src/cli/services/commands/doc/fetch.test.ts new file mode 100644 index 00000000000..8406cbe9ba9 --- /dev/null +++ b/packages/cli/src/cli/services/commands/doc/fetch.test.ts @@ -0,0 +1,61 @@ +import {docFetchService} from './fetch.js' +import {describe, expect, test, vi, beforeEach} from 'vitest' +import {fetch} from '@shopify/cli-kit/node/http' +import {outputResult} from '@shopify/cli-kit/node/output' +import {AbortError} from '@shopify/cli-kit/node/error' +import {mkdir, writeFile} from '@shopify/cli-kit/node/fs' +import {dirname, resolvePath} from '@shopify/cli-kit/node/path' + +vi.mock('@shopify/cli-kit/node/http') +vi.mock('@shopify/cli-kit/node/output') +vi.mock('@shopify/cli-kit/node/fs') + +const okResponse = (body: string) => + ({ok: true, status: 200, statusText: 'OK', text: () => Promise.resolve(body)}) as any + +beforeEach(() => { + vi.mocked(fetch).mockResolvedValue(okResponse('# Doc')) +}) + +describe('docFetchService', () => { + test('requests Markdown and prints the body to stdout', async () => { + await docFetchService('https://shopify.dev/docs/api/shopify-cli') + + expect(fetch).toHaveBeenCalledWith('https://shopify.dev/docs/api/shopify-cli', { + headers: {Accept: 'text/markdown', 'X-Shopify-Surface': 'cli'}, + }) + expect(outputResult).toHaveBeenCalledWith('# Doc') + }) + + test('accepts shopify.dev subdomains', async () => { + await docFetchService('https://www.shopify.dev/docs') + + expect(fetch).toHaveBeenCalledOnce() + }) + + test('rejects URLs from disallowed hosts without fetching', async () => { + await expect(docFetchService('https://example.com/docs')).rejects.toThrowError(AbortError) + expect(fetch).not.toHaveBeenCalled() + }) + + test('rejects malformed URLs without fetching', async () => { + await expect(docFetchService('not a url')).rejects.toThrowError(AbortError) + expect(fetch).not.toHaveBeenCalled() + }) + + test('writes the document to the output path instead of stdout', async () => { + await docFetchService('https://shopify.dev/docs/api/shopify-cli', 'docs/shopify-cli.md') + + const expectedPath = resolvePath('docs/shopify-cli.md') + expect(mkdir).toHaveBeenCalledWith(dirname(expectedPath)) + expect(writeFile).toHaveBeenCalledWith(expectedPath, '# Doc') + expect(outputResult).not.toHaveBeenCalled() + }) + + test('throws when the response is not ok', async () => { + vi.mocked(fetch).mockResolvedValue({ok: false, status: 404, statusText: 'Not Found'} as any) + + await expect(docFetchService('https://shopify.dev/missing')).rejects.toThrowError(AbortError) + expect(outputResult).not.toHaveBeenCalled() + }) +}) diff --git a/packages/cli/src/cli/services/commands/doc/fetch.ts b/packages/cli/src/cli/services/commands/doc/fetch.ts new file mode 100644 index 00000000000..1575ccea31f --- /dev/null +++ b/packages/cli/src/cli/services/commands/doc/fetch.ts @@ -0,0 +1,53 @@ +import {fetch} from '@shopify/cli-kit/node/http' +import {outputInfo, outputResult} from '@shopify/cli-kit/node/output' +import {AbortError} from '@shopify/cli-kit/node/error' +import {mkdir, writeFile} from '@shopify/cli-kit/node/fs' +import {dirname, resolvePath} from '@shopify/cli-kit/node/path' + +// Every page on shopify.dev has a Markdown representation, which is the clean, +// parseable content agents want — so we always request it. +const MARKDOWN_CONTENT_TYPE = 'text/markdown' + +// Identifies the CLI as the calling surface to shopify.dev, so traffic +// originating from the CLI can be attributed as such. +const SURFACE_HEADER = 'X-Shopify-Surface' +const SURFACE = 'cli' + +// Hosts whose documents are allowed to be fetched. A URL matches when its +// hostname is one of these or a subdomain of one of these. +const ALLOWED_HOSTS = ['shopify.dev'] + +export async function docFetchService(url: string, outputPath?: string) { + let parsedURL: URL + try { + parsedURL = new URL(url) + } catch { + throw new AbortError(`Invalid URL: ${url}`) + } + + const {hostname} = parsedURL + const isAllowed = ALLOWED_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`)) + if (!isAllowed) { + throw new AbortError(`Only documents from the following hosts can be fetched: ${ALLOWED_HOSTS.join(', ')}.`) + } + + const response = await fetch(url, {headers: {Accept: MARKDOWN_CONTENT_TYPE, [SURFACE_HEADER]: SURFACE}}) + + if (!response.ok) { + throw new AbortError(`Failed to fetch ${url}: ${response.status} ${response.statusText}`) + } + + const body = await response.text() + + // When an output path is provided, write the document to disk (creating any + // missing parent directories) instead of printing it to stdout. + if (outputPath) { + const absolutePath = resolvePath(outputPath) + await mkdir(dirname(absolutePath)) + await writeFile(absolutePath, body) + outputInfo(`Saved ${url} to ${absolutePath}`) + return + } + + outputResult(body) +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 51e203e1b3d..b5269642172 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -10,6 +10,7 @@ import KitchenSinkStatic from './cli/commands/kitchen-sink/static.js' import KitchenSink from './cli/commands/kitchen-sink/index.js' import Doctor from './cli/commands/doctor-release/doctor-release.js' import DoctorTheme from './cli/commands/doctor-release/theme/index.js' +import DocFetch from './cli/commands/doc/fetch.js' import DocSearch from './cli/commands/doc/search.js' import DocsGenerate from './cli/commands/docs/generate.js' import HelpCommand from './cli/commands/help.js' @@ -158,6 +159,7 @@ export const COMMANDS: any = { 'kitchen-sink:static': KitchenSinkStatic, 'doctor-release': Doctor, 'doctor-release:theme': DoctorTheme, + 'doc:fetch': DocFetch, 'doc:search': DocSearch, 'docs:generate': DocsGenerate, 'notifications:list': List, diff --git a/packages/e2e/data/snapshots/commands.txt b/packages/e2e/data/snapshots/commands.txt index aea57d18e66..1b19a19bf9f 100644 --- a/packages/e2e/data/snapshots/commands.txt +++ b/packages/e2e/data/snapshots/commands.txt @@ -50,6 +50,7 @@ │ ├─ on │ └─ status ├─ doc +│ ├─ fetch │ └─ search ├─ help ├─ hydrogen