Merge pull request #23691 from storybookjs/indexer-api

Indexing: Introduce new experimental `indexer` API - NESTED PR!

Author: JReinhold

code/addons/docs/src/preset.ts

@@ -3,13 +3,7 @@ import remarkSlug from 'remark-slug';
 import remarkExternalLinks from 'remark-external-links';
 import { dedent } from 'ts-dedent';
 
-import type {
-  IndexerOptions,
-  StoryIndexer,
-  DocsOptions,
-  Options,
-  StorybookConfig,
-} from '@storybook/types';
+import type { DocsOptions, Indexer, Options, StorybookConfig } from '@storybook/types';
 import type { CsfPluginOptions } from '@storybook/csf-plugin';
 import type { JSXOptions, CompileOptions } from '@storybook/mdx2-csf';
 import { global } from '@storybook/global';
@@ -135,23 +129,35 @@ async function webpack(
   return result;
 }
 
-const storyIndexers = (indexers: StoryIndexer[] | null) => {
-  const mdxIndexer = async (fileName: string, opts: IndexerOptions) => {
+export const createStoriesMdxIndexer = (legacyMdx1?: boolean): Indexer => ({
+  test: /(stories|story)\.mdx$/,
+  index: async (fileName, opts) => {
     let code = (await fs.readFile(fileName, 'utf-8')).toString();
-    const { compile } = global.FEATURES?.legacyMdx1
+    const { compile } = legacyMdx1
       ? await import('@storybook/mdx1-csf')
       : await import('@storybook/mdx2-csf');
     code = await compile(code, {});
-    return loadCsf(code, { ...opts, fileName }).parse();
-  };
-  return [
-    {
-      test: /(stories|story)\.mdx$/,
-      indexer: mdxIndexer,
-    },
-    ...(indexers || []),
-  ];
-};
+    const csf = loadCsf(code, { ...opts, fileName }).parse();
+
+    const { indexInputs, stories } = csf;
+
+    return indexInputs.map((input, index) => {
+      const docsOnly = stories[index].parameters?.docsOnly;
+      const tags = input.tags ? input.tags : [];
+      if (docsOnly) {
+        tags.push('stories-mdx-docsOnly');
+      }
+      // the mdx-csf compiler automatically adds the 'stories-mdx' tag to meta, here' we're just making sure it is always there
+      if (!tags.includes('stories-mdx')) {
+        tags.push('stories-mdx');
+      }
+      return { ...input, tags };
+    });
+  },
+});
+
+const indexers: StorybookConfig['experimental_indexers'] = (existingIndexers) =>
+  [createStoriesMdxIndexer(global.FEATURES?.legacyMdx1)].concat(existingIndexers || []);
 
 const docs = (docsOptions: DocsOptions) => {
   return {
@@ -170,9 +176,9 @@ export const addons: StorybookConfig['addons'] = [
  * something down the dependency chain is using typescript namespaces, which are not supported by rollup-plugin-dts
  */
 const webpackX = webpack as any;
-const storyIndexersX = storyIndexers as any;
+const indexersX = indexers as any;
 const docsX = docs as any;
 
 ensureReactPeerDeps();
 
-export { webpackX as webpack, storyIndexersX as storyIndexers, docsX as docs };
+export { webpackX as webpack, indexersX as experimental_indexers, docsX as docs };

code/lib/cli/src/generators/SERVER/index.ts

@@ -3,7 +3,7 @@ import type { Generator } from '../types';
 
 const generator: Generator = async (packageManager, npmOptions, options) => {
   await baseGenerator(packageManager, npmOptions, options, 'server', {
-    extensions: ['json'],
+    extensions: ['json', 'yaml', 'yml'],
   });
 };
 

code/lib/core-common/src/utils/validate-config.ts

@@ -6,7 +6,9 @@ const renderers = ['html', 'preact', 'react', 'server', 'svelte', 'vue', 'vue3',
 
 const rendererNames = [...renderers, ...renderers.map((renderer) => `@storybook/${renderer}`)];
 
-export function validateFrameworkName(frameworkName: string | undefined) {
+export function validateFrameworkName(
+  frameworkName: string | undefined
+): asserts frameworkName is string {
   const automigrateMessage = `Please run 'npx storybook@next automigrate' to automatically fix your config.
 
   See the migration guide for more information: 

code/lib/core-server/package.json

@@ -98,6 +98,7 @@
     "ws": "^8.2.3"
   },
   "devDependencies": {
+    "@storybook/addon-docs": "workspace:*",
     "@types/compression": "^1.7.0",
     "@types/ip": "^1.1.0",
     "@types/node-fetch": "^2.5.7",

code/lib/core-server/src/build-dev.ts

@@ -31,13 +31,15 @@ export async function buildDevStandalone(
   options: CLIOptions & LoadOptions & BuilderOptions
 ): Promise<{ port: number; address: string; networkAddress: string }> {
   const { packageJson, versionUpdates } = options;
-  const { version } = packageJson;
-  invariant(version !== undefined, 'Expected package.json version to be defined.');
+  invariant(
+    packageJson.version !== undefined,
+    `Expected package.json#version to be defined in the "${packageJson.name}" package}`
+  );
   // updateInfo are cached, so this is typically pretty fast
   const [port, versionCheck] = await Promise.all([
     getServerPort(options.port),
     versionUpdates
-      ? updateCheck(version)
+      ? updateCheck(packageJson.version)
       : Promise.resolve({ success: false, cached: false, data: {}, time: Date.now() }),
   ]);
 
@@ -64,10 +66,9 @@ export async function buildDevStandalone(
 
   const config = await loadMainConfig(options);
   const { framework } = config;
-  invariant(framework, 'framework is required in Storybook v7');
   const corePresets = [];
 
-  const frameworkName = typeof framework === 'string' ? framework : framework.name;
+  const frameworkName = typeof framework === 'string' ? framework : framework?.name;
   validateFrameworkName(frameworkName);
 
   corePresets.push(join(frameworkName, 'preset'));
@@ -84,7 +85,8 @@ export async function buildDevStandalone(
   });
 
   const { renderer, builder, disableTelemetry } = await presets.apply<CoreConfig>('core', {});
-  invariant(builder, 'no builder configured!');
+
+  invariant(builder, 'No builder configured in core.builder');
 
   if (!options.disableTelemetry && !disableTelemetry) {
     if (versionCheck.success && !versionCheck.cached) {
@@ -98,9 +100,8 @@ export async function buildDevStandalone(
     getManagerBuilder(),
   ]);
 
-  const resolvedRenderer = renderer
-    ? resolveAddonName(options.configDir, renderer, options)
-    : undefined;
+  const resolvedRenderer = renderer && resolveAddonName(options.configDir, renderer, options);
+
   // Load second pass: all presets are applied in order
   presets = await loadAllPresets({
     corePresets: [
@@ -128,10 +129,10 @@ export async function buildDevStandalone(
     fullOptions
   );
 
-  const previewTotalTime = previewResult && previewResult.totalTime;
-  const managerTotalTime = managerResult ? managerResult.totalTime : undefined;
-  const previewStats = previewResult && previewResult.stats;
-  const managerStats = managerResult && managerResult.stats;
+  const previewTotalTime = previewResult?.totalTime;
+  const managerTotalTime = managerResult?.totalTime;
+  const previewStats = previewResult?.stats;
+  const managerStats = managerResult?.stats;
 
   if (options.webpackStatsJson) {
     const target = options.webpackStatsJson === true ? options.outputDir : options.webpackStatsJson;
@@ -162,7 +163,7 @@ export async function buildDevStandalone(
     if (!options.quiet) {
       outputStartupInformation({
         updateInfo: versionCheck,
-        version,
+        version: packageJson.version,
         name,
         address,
         networkAddress,

code/lib/core-server/src/build-static.ts

@@ -107,14 +107,16 @@ export async function buildStaticStandalone(options: BuildStaticStandaloneOption
     ...options,
   });
 
-  const [features, core, staticDirs, storyIndexers, stories, docsOptions] = await Promise.all([
-    presets.apply<StorybookConfig['features']>('features'),
-    presets.apply<CoreConfig>('core'),
-    presets.apply<StorybookConfig['staticDirs']>('staticDirs'),
-    presets.apply('storyIndexers', []),
-    presets.apply('stories'),
-    presets.apply<DocsOptions>('docs', {}),
-  ]);
+  const [features, core, staticDirs, indexers, deprecatedStoryIndexers, stories, docsOptions] =
+    await Promise.all([
+      presets.apply<StorybookConfig['features']>('features'),
+      presets.apply<CoreConfig>('core'),
+      presets.apply<StorybookConfig['staticDirs']>('staticDirs'),
+      presets.apply('experimental_indexers', []),
+      presets.apply('storyIndexers', []),
+      presets.apply('stories'),
+      presets.apply<DocsOptions>('docs', {}),
+    ]);
 
   const fullOptions: Options = {
     ...options,
@@ -164,27 +166,27 @@ export async function buildStaticStandalone(options: BuildStaticStandaloneOption
     const normalizedStories = normalizeStories(stories, directories);
     const generator = new StoryIndexGenerator(normalizedStories, {
       ...directories,
-      storyIndexers,
+      storyIndexers: deprecatedStoryIndexers,
+      indexers,
       docs: docsOptions,
       storiesV2Compatibility: !features?.storyStoreV7,
       storyStoreV7: !!features?.storyStoreV7,
     });
 
-    const initializedStoryIndexGeneratorPromise = generator.initialize().then(() => generator);
+    initializedStoryIndexGenerator = generator.initialize().then(() => generator);
     effects.push(
       extractStoriesJson(
         join(options.outputDir, 'stories.json'),
-        initializedStoryIndexGeneratorPromise,
+        initializedStoryIndexGenerator as Promise<StoryIndexGenerator>,
         convertToIndexV3
       )
     );
     effects.push(
       extractStoriesJson(
         join(options.outputDir, 'index.json'),
-        initializedStoryIndexGeneratorPromise
+        initializedStoryIndexGenerator as Promise<StoryIndexGenerator>
       )
     );
-    initializedStoryIndexGenerator = initializedStoryIndexGeneratorPromise;
   }
 
   if (!core?.disableProjectJson) {

code/lib/core-server/src/presets/common-preset.ts

@@ -11,13 +11,12 @@ import {
 import type {
   CLIOptions,
   CoreConfig,
-  IndexerOptions,
+  Indexer,
   Options,
   PresetPropertyFn,
   StorybookConfig,
-  StoryIndexer,
 } from '@storybook/types';
-import { loadCsf, printConfig, readConfig } from '@storybook/csf-tools';
+import { printConfig, readConfig, readCsf } from '@storybook/csf-tools';
 import { join } from 'path';
 import { dedent } from 'ts-dedent';
 import fetch from 'node-fetch';
@@ -195,20 +194,15 @@ export const features = async (
   legacyDecoratorFileOrder: false,
 });
 
-export const storyIndexers = async (indexers?: StoryIndexer[]) => {
-  const csfIndexer = async (fileName: string, opts: IndexerOptions) => {
-    const code = (await readFile(fileName, 'utf-8')).toString();
-    return loadCsf(code, { ...opts, fileName }).parse();
-  };
-  return [
-    {
-      test: /(stories|story)\.(m?js|ts)x?$/,
-      indexer: csfIndexer,
-    },
-    ...(indexers || []),
-  ];
+export const csfIndexer: Indexer = {
+  test: /\.stories\.(m?js|ts)x?$/,
+  index: async (fileName, options) => (await readCsf(fileName, options)).parse().indexInputs,
 };
 
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export const experimental_indexers: StorybookConfig['experimental_indexers'] = (existingIndexers) =>
+  [csfIndexer].concat(existingIndexers || []);
+
 export const frameworkOptions = async (
   _: never,
   options: Options