Fix include rules not working for subdirectory paths

file-manifest/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Unreleased
+ - Added glob pattern support for all rules (include, exclude, ignore, ignore-destination) using picomatch
+ - Fixed include rules not working for subdirectory paths (e.g. `include frontend/out`)
+ - Changed exclude rules to take priority over include rules
+
+# 0.1.0
+ - Initial public release.

file-manifest/README.md

@@ -39,6 +39,51 @@ to specify paths that shouldn't be deleted.
     ignore-destination dest
 ```
 
+## Glob patterns
+
+All rules support glob patterns (powered by [picomatch](https://github.com/micromatch/picomatch)):
+
+| Pattern | Description | Example |
+|---------|-------------|---------|
+| `*` | Match any characters except `/` | `include *.json` matches top-level JSON files |
+| `**` | Match any number of directories | `include src/**/*.ts` matches all `.ts` files under `src/` |
+| `?` | Match a single character | `include file-?` matches `file-1`, `file-2`, etc. |
+| `{a,b}` | Match any of the comma-separated patterns | `include *.{js,ts}` matches JS and TS files |
+
+### Glob examples
+
+```
+    include src/**/*.ts
+    exclude **/*.test.ts
+    include dist/*.js
+    include config-*
+```
+
+**Note:** `*` only matches within a single path segment. Use `**` to match across directories:
+- `include *.js` matches `index.js` but not `src/index.js`
+- `include **/*.js` matches both `index.js` and `src/index.js`
+
+## Subdirectory includes
+
+You can include a specific subdirectory without including its parent's other contents:
+
+```
+    include frontend/out
+```
+
+This will traverse into `frontend/` but only include the `out/` subdirectory and its contents. Other files and directories inside `frontend/` are not included.
+
+## Rule priority
+
+Exclude rules take priority over include rules. If a path matches both an include and exclude pattern, the exclude wins:
+
+```
+    include src/**/*.ts
+    exclude **/*.test.ts
+```
+
+This includes all TypeScript files under `src/` except test files.
+
 
 # API
 
@@ -62,9 +107,9 @@ Takes a source directory and rules configuration, returns a table of files that
 **Example:**
 ```typescript
 const files = await resolveFileList('/path/to/source', `
-    include src
+    include src/**/*.ts
     include docs
-    exclude src/build
+    exclude **/*.test.ts
     exclude .git
 `);
 

file-manifest/package.json

@@ -11,10 +11,12 @@
   "dependencies": {
     "@facetlayer/concurrency-limit": "^0.1.0",
     "@facetlayer/qc": "^0.1.0",
-    "@facetlayer/streams": "^1.0.0"
+    "@facetlayer/streams": "^1.0.0",
+    "picomatch": "^4.0.4"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",
+    "@types/picomatch": "^4.0.3",
     "typescript": "^5.2.0",
     "vitest": "^3.0.6"
   }

file-manifest/src/__tests__/resolveFileList.test.ts

@@ -82,6 +82,114 @@ it("exclude a nested file", async () => {
     `);
 });
 
+it("include a subdirectory", async () => {
+    const files = await resolveFileList(sampleDir, `
+        include dir-2/subdir-1
+    `);
+    expect(normalizePathsForSnapshot(files.listAll())).toMatchInlineSnapshot(`
+      [
+        {
+          "id": 1,
+          "relPath": "dir-2/subdir-1/file-6",
+          "sourcePath": "dir-2/subdir-1/file-6",
+        },
+      ]
+    `);
+});
+
+it("include a file inside a subdirectory", async () => {
+    const files = await resolveFileList(sampleDir, `
+        include dir-2/file-4
+    `);
+    expect(normalizePathsForSnapshot(files.listAll())).toMatchInlineSnapshot(`
+      [
+        {
+          "id": 1,
+          "relPath": "dir-2/file-4",
+          "sourcePath": "dir-2/file-4",
+        },
+      ]
+    `);
+});
+
+// Glob pattern tests
+
+const globSampleDir = Path.resolve(__dirname, 'samplefiles-glob');
+
+function normalizeGlobPaths(files: FileEntry[]): string[] {
+    return files.map(f => Path.relative(globSampleDir, f.sourcePath)).sort();
+}
+
+it("glob: include with wildcard matching directories", async () => {
+    const files = await resolveFileList(sampleDir, `
+        include dir-*
+    `);
+    expect(normalizePathsForSnapshot(files.listAll()).map(f => f.relPath)).toEqual([
+        "dir-1/file-3",
+        "dir-2/file-4",
+        "dir-2/file-5",
+        "dir-2/subdir-1/file-6",
+    ]);
+});
+
+it("glob: include with wildcard matching files", async () => {
+    const files = await resolveFileList(sampleDir, `
+        include file-*
+    `);
+    expect(normalizePathsForSnapshot(files.listAll()).map(f => f.relPath)).toEqual([
+        "file-1",
+        "file-2",
+    ]);
+});
+
+it("glob: include with ** to match deep paths", async () => {
+    const files = await resolveFileList(globSampleDir, `
+        include src/**/*.ts
+    `);
+    expect(normalizeGlobPaths(files.listAll())).toEqual([
+        "src/index.test.ts",
+        "src/index.ts",
+        "src/lib/helper.test.ts",
+        "src/lib/helper.ts",
+        "src/utils.ts",
+    ]);
+});
+
+it("glob: exclude with ** pattern", async () => {
+    const files = await resolveFileList(globSampleDir, `
+        include src/**/*.ts
+        exclude **/*.test.ts
+    `);
+    expect(normalizeGlobPaths(files.listAll())).toEqual([
+        "src/index.ts",
+        "src/lib/helper.ts",
+        "src/utils.ts",
+    ]);
+});
+
+it("glob: include with extension wildcard at top level", async () => {
+    const files = await resolveFileList(globSampleDir, `
+        include *.json
+    `);
+    expect(normalizeGlobPaths(files.listAll())).toEqual([
+        "package.json",
+    ]);
+});
+
+it("glob: mix glob and exact patterns", async () => {
+    const files = await resolveFileList(globSampleDir, `
+        include src
+        include *.json
+        exclude src/lib
+    `);
+    expect(normalizeGlobPaths(files.listAll())).toEqual([
+        "package.json",
+        "src/index.test.ts",
+        "src/index.ts",
+        "src/utils.ts",
+    ]);
+});
+
 it("exclude a nested directory", async () => {
     const files = await resolveFileList(sampleDir, `
         include dir-2

file-manifest/src/findLeftoverFiles.ts

@@ -2,6 +2,7 @@
 import { ParsedRules } from './resolveFileList';
 import { RuleType } from './FileMatchRule';
 import { FileList } from './FileList';
+import picomatch from 'picomatch';
 import Path from 'path';
 import Fs from 'fs/promises';
 
@@ -31,7 +32,7 @@ export async function findLeftoverFiles(targetDir: string, incomingFiles: FileLi
             // Check if any rule tells us to ignore this destination file
             let shouldIgnore = false;
             for (const rule of ruleConfig) {
-                if ((rule.type === RuleType.IgnoreDestination || rule.type === RuleType.Ignore) && rule.pattern === relPath) {
+                if ((rule.type === RuleType.IgnoreDestination || rule.type === RuleType.Ignore) && picomatch.isMatch(relPath, rule.pattern)) {
                     shouldIgnore = true;
                     break;
                 }

file-manifest/src/parseRulesFile.ts

@@ -7,7 +7,7 @@ export function parseRulesFile(ruleConfig: string): FileMatchRule[] {
 
     for (const query of queries) {
         const command = query.command;
-        const pattern = query.tags?.[0]?.attr;
+        const pattern = query.tags?.map(t => t.attr).join('');
 
         if (!pattern) {
             throw new Error(`Missing pattern for ${command} rule`);

file-manifest/src/resolveFileList.ts

@@ -2,6 +2,7 @@
 
 import Path from 'path'
 import Fs from 'fs/promises'
+import picomatch from 'picomatch';
 import { FileMatchRule, RuleType } from './FileMatchRule';
 import { parseRulesFile } from './parseRulesFile';
 import { FileList } from './FileList';
@@ -50,22 +51,63 @@ export async function resolveFileList(sourceDir: string, ruleConfig: string | Pa
     }
 
     function shouldInclude(localPath: string, defaultValue: boolean) {
-        // Returns whether the file should be included (based on the config rules)
+        // Returns whether the file should be included (based on the config rules).
+        // Exclude rules take priority over include rules.
         const relPath = getRelPath(localPath);
 
         for (const rule of rules) {
-            if (rule.type === RuleType.Include && rule.pattern === relPath)
-                return true;
+            if ((rule.type === RuleType.Exclude || rule.type === RuleType.Ignore) && picomatch.isMatch(relPath, rule.pattern))
+                return false;
         }
 
         for (const rule of rules) {
-            if ((rule.type === RuleType.Exclude || rule.type === RuleType.Ignore) && rule.pattern === relPath)
-                return false;
+            if (rule.type === RuleType.Include && picomatch.isMatch(relPath, rule.pattern))
+                return true;
         }
 
         return defaultValue;
     }
 
+    function couldContainMatch(dirRelPath: string, pattern: string): boolean {
+        // Check if a directory could contain files that match the given pattern
+        // by walking through path segments and pattern segments together.
+        const dirParts = dirRelPath.split('/');
+        const patParts = pattern.split('/');
+
+        let pi = 0;
+        let di = 0;
+
+        while (di < dirParts.length && pi < patParts.length) {
+            if (patParts[pi] === '**') {
+                // ** can match any number of segments - directory could contain matches
+                return true;
+            }
+            if (picomatch.isMatch(dirParts[di], patParts[pi])) {
+                di++;
+                pi++;
+            } else {
+                return false;
+            }
+        }
+
+        // If we consumed all dir segments and still have pattern segments left,
+        // then this directory is an ancestor of potential matches.
+        return di === dirParts.length && pi < patParts.length;
+    }
+
+    function isAncestorOfInclude(localPath: string) {
+        // Check if this directory is an ancestor of any include pattern, so we
+        // know to traverse into it even when it's not directly included.
+        const relPath = getRelPath(localPath);
+
+        for (const rule of rules) {
+            if (rule.type === RuleType.Include && couldContainMatch(relPath, rule.pattern))
+                return true;
+        }
+
+        return false;
+    }
+
     async function recursiveIncludeSubDirectory(localDir: string, assumeIncludeContents: boolean) {
         // Include the contents of this directory.
         //
@@ -80,12 +122,16 @@ export async function resolveFileList(sourceDir: string, ruleConfig: string | Pa
         for (const dirRelFile of dirContents) {
             const localSubFile = Path.join(localDir, dirRelFile);
 
-            if (!shouldInclude(localSubFile, assumeIncludeContents)) {
+            if (await isDirectory(localSubFile)) {
+                if (shouldInclude(localSubFile, assumeIncludeContents)) {
+                    await recursiveIncludeSubDirectory(localSubFile, true);
+                } else if (isAncestorOfInclude(localSubFile)) {
+                    await recursiveIncludeSubDirectory(localSubFile, false);
+                }
                 continue;
             }
 
-            if (await isDirectory(localSubFile)) {
-                await recursiveIncludeSubDirectory(localSubFile, true);
+            if (!shouldInclude(localSubFile, assumeIncludeContents)) {
                 continue;
             }
 

goobernetes/CHANGELOG.md

@@ -1,3 +1,7 @@
+# Unreleased
+ - Added glob pattern support for include/exclude/ignore rules (e.g. `include src/**/*.ts`, `exclude **/*.test.js`)
+ - Fixed include rules not working for subdirectory paths (e.g. `include frontend/out`)
+
 # 0.3.3
  - Update sqlite3 version
 

goobernetes/README.md

@@ -134,6 +134,12 @@ ignore web/.next
 - `exclude <path>`: Exclude files or directories from the deployment.
 - `ignore <path>`: Ignore a path or directory on the source side or receiving side.
 
+All file rules support glob patterns:
+- `include src/**/*.ts` — include all TypeScript files under `src/`
+- `exclude **/*.test.js` — exclude test files at any depth
+- `include config-*` — include files/directories matching a wildcard
+- `include frontend/out` — include only a specific subdirectory
+
 ### Example Configuration
 
 ```