FlatFilers · amarbakir · Oct 30, 2024
diff --git a/.changeset/slow-bottles-roll.md b/.changeset/slow-bottles-roll.md
@@ -0,0 +1,6 @@
+---
+'@flatfile/plugin-json-extractor': minor
+'@flatfile/plugin-json-multisheet-extractor': patch
+---
+
+Initialize the JSON multi-sheet extractor plugin
diff --git a/plugins/json-extractor/package.json b/plugins/json-extractor/package.json
@@ -61,7 +61,7 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@flatfile/util-extractor": "^2.2.0"
+    "@flatfile/util-extractor": "^2.3.0"
   },
   "devDependencies": {
     "@flatfile/bundler-config-tsup": "^0.1.0"

diff --git a/plugins/json-extractor/src/parser.ts b/plugins/json-extractor/src/parser.ts
@@ -1,4 +1,4 @@
-import { WorkbookCapture } from '@flatfile/util-extractor'
+import { WorkbookCapture, parseSheet } from '@flatfile/util-extractor'
 
 export function parseBuffer(buffer: Buffer): WorkbookCapture {
   try {
@@ -13,52 +13,27 @@ export function parseBuffer(buffer: Buffer): WorkbookCapture {
     const parsedData = JSON.parse(fileContents) || []
     const results = Array.isArray(parsedData) ? parsedData : [parsedData]
 
-    // Ensure all items are objects
-    const filteredResults = results.filter(
-      (item) => typeof item === 'object' && item !== null
-    )
+    var sheetCapture = parseSheet(results);
 
-    if (filteredResults.length === 0) {
+    if (isEmpty(sheetCapture)) {
       return {} as WorkbookCapture
     }
 
-    // Custom flatten function
-    const flattenObject = (obj: any, parent: string = '', res: any = {}) => {
-      for (let key in obj) {
-        const propName = parent ? parent + '.' + key : key
-        if (typeof obj[key] === 'object') {
-          flattenObject(obj[key], propName, res)
-        } else {
-          res[propName] = obj[key]
-        }
-      }
-      return res
-    }
-
-    // Flatten the first item to determine headers
-    const firstItem = flattenObject(filteredResults[0])
-    const headers = Object.keys(firstItem).filter((header) => header !== '')
-
-    // Flatten and filter all rows
-    const filteredData = filteredResults.map((row) => {
-      const flattedRow = flattenObject(row)
-      return headers.reduce((filteredRow, header) => {
-        const cell = flattedRow[header]
-        filteredRow[header] = {
-          value: Array.isArray(cell) ? JSON.stringify(cell) : cell,
-        }
-        return filteredRow
-      }, {})
-    })
-
     return {
-      [sheetName]: {
-        headers,
-        data: filteredData,
-      },
+      [sheetName]: sheetCapture,
     } as WorkbookCapture
   } catch (error) {
     console.error('An error occurred:', error)
     throw error
   }
 }
+
+function isEmpty(obj) {
+  for (const prop in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, prop)) {
+      return false;
+    }
+  }
+
+  return true;
+}
diff --git a/plugins/json-multisheet-extractor/CHANGELOG.md b/plugins/json-multisheet-extractor/CHANGELOG.md
@@ -0,0 +1,11 @@
+# @flatfile/plugin-json-extractor
+
+## 0.1.0
+
+### Minor Changes
+
+- 65b526d: Initial release of the JSON multi-sheet extractor
+
+### Patch Changes
+
+- b557dde: add readme
diff --git a/plugins/json-multisheet-extractor/README.md b/plugins/json-multisheet-extractor/README.md
@@ -0,0 +1,78 @@
+<!-- START_INFOCARD -->
+
+The `@flatfile/json-multisheet-extractor` plugin parses a JSON file and extracts second-level nested objects into first level defined Sheets in Flatfile.
+
+**Event Type:** 
+`listener.on('file:created')`
+
+**Supported file types:** 
+`.json`
+
+<!-- END_INFOCARD -->
+
+> When embedding Flatfile, this plugin should be deployed in a server-side listener. [Learn more](/docs/orchestration/listeners#listener-types)
+
+
+
+## Parameters
+
+#### `options.chunkSize` - `default: "10_000"` - `number` - (optional)
+The `chunkSize` parameter allows you to specify the quantity of records to in each chunk.
+
+
+#### `options.parallel` - `default: "1"` - `number` - (optional)
+The `parallel` parameter allows you to specify the number of chunks to process in parallel.
+
+
+
+## API Calls
+
+- `api.files.download`
+- `api.files.get`
+- `api.files.update`
+- `api.jobs.ack`
+- `api.jobs.complete`
+- `api.jobs.create`
+- `api.jobs.fail`
+- `api.jobs.update`
+- `api.records.insert`
+- `api.workbooks.create`
+
+
+
+## Usage
+
+Listen for a JSON file to be uploaded to Flatfile. The platform will then extract the file automatically. Once complete, the file will be ready for import in the Files area.
+
+```bash Install
+npm i @flatfile/plugin-json-multisheet-extractor
+```
+
+```js import
+import { JSONMultiSheetExtractor } from "@flatfile/plugin-json-multisheet-extractor";
+```
+
+```js listener.js
+listener.use(JSONMultiSheetExtractor());
+```
+
+
+
+### Full Example
+
+In this example, the `JSONMultiSheetExtractor` is initialized, and then registered as middleware with the Flatfile listener. When a JSON file is uploaded, the plugin will extract the structured data and process it the extractor's parser.
+
+```javascript
+import { JSONMultiSheetExtractor } from "@flatfile/plugin-json-multisheet-extractor";
+
+// Initialize the JSON multi-sheet extractor
+const jsonMultiSheetExtractor = JSONMultiSheetExtractor();
+
+// Register the extractor as a middleware for the Flatfile listener
+listener.use(jsonMultiSheetExtractor);
+
+// When a JSON file is uploaded, the data will be extracted and processed using the extractor's parser.
+```
+
+See a working example in our [flatfile-docs-kitchen-sink](https://github.com/FlatFilers/flatfile-docs-kitchen-sink/blob/main/typescript/extractors/index.ts) Github repo.
+
diff --git a/plugins/json-multisheet-extractor/jest.config.cjs b/plugins/json-multisheet-extractor/jest.config.cjs
@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}
diff --git a/plugins/json-multisheet-extractor/package.json b/plugins/json-multisheet-extractor/package.json
@@ -0,0 +1,69 @@
+{
+  "name": "@flatfile/plugin-json-multisheet-extractor",
+  "version": "0.1.0",
+  "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/plugins/json-multisheet-extractor",
+  "description": "A plugin for parsing json files into multiple sheets in Flatfile.",
+  "registryMetadata": {
+    "category": "extractors"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">= 16"
+  },
+  "exports": {
+    ".": {
+      "node": {
+        "types": {
+          "import": "./dist/index.d.ts",
+          "require": "./dist/index.d.cts"
+        },
+        "import": "./dist/index.js",
+        "require": "./dist/index.cjs"
+      },
+      "browser": {
+        "types": {
+          "import": "./dist/index.d.ts",
+          "require": "./dist/index.d.cts"
+        },
+        "import": "./dist/index.browser.js",
+        "require": "./dist/index.browser.cjs"
+      },
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "source": "./src/index.ts",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "build:watch": "tsup --watch",
+    "build:prod": "NODE_ENV=production tsup",
+    "checks": "tsc --noEmit && attw --pack . && publint .",
+    "lint": "tsc --noEmit",
+    "test": "jest src/*.spec.ts --detectOpenHandles",
+    "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles",
+    "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles"
+  },
+  "keywords": [
+    "flatfile-plugins",
+    "category-extractors"
+  ],
+  "author": "Amaar Bakir",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "plugins/json-multisheet-extractor"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@flatfile/util-extractor": "^2.3.0"
+  },
+  "devDependencies": {
+    "@flatfile/bundler-config-tsup": "^0.1.0"
+  }
+}
diff --git a/plugins/json-multisheet-extractor/ref/test-basic.json b/plugins/json-multisheet-extractor/ref/test-basic.json
@@ -0,0 +1,122 @@
+{
+  "contacts": [
+    {
+      "First Name": "Tony",
+      "Last Name": "Lamb",
+      "Email": "me@opbaj.tp",
+      "Address": {
+        "Street": "123 Main Street",
+        "City": "Springfield",
+        "State": "ST",
+        "Zip": "12345",
+        "Coordinates": {
+          "Latitude": "40.7128° N",
+          "Longitude": "74.0060° W"
+        }
+      },
+      "Father": {
+        "First Name": "Father_First_1",
+        "Last Name": "Father_Last_1",
+        "Father": {
+          "First Name": "Father_First_2",
+          "Last Name": "Father_Last_2",
+          "Father": {
+            "First Name": "Father_First_3",
+            "Last Name": "Father_Last_3",
+            "Father": {
+              "First Name": "Father_First_4",
+              "Last Name": "Father_Last_4",
+              "Father": {
+                "First Name": "Father_First_5",
+                "Last Name": "Father_Last_5",
+                "Father": null
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "First Name": "Christian",
+      "Last Name": "Ramos",
+      "Email": "uw@ag.tg",
+      "Address": {
+        "Street": "456 Elm Street",
+        "City": "Greenville",
+        "State": "GT",
+        "Zip": "67890",
+        "Coordinates": {
+          "Latitude": "40.7128° N",
+          "Longitude": "74.0060° W"
+        }
+      },
+      "Father": {
+        "First Name": "Father_First_1",
+        "Last Name": "Father_Last_1",
+        "Father": {
+          "First Name": "Father_First_2",
+          "Last Name": "Father_Last_2",
+          "Father": {
+            "First Name": "Father_First_3",
+            "Last Name": "Father_Last_3",
+            "Father": {
+              "First Name": "Father_First_4",
+              "Last Name": "Father_Last_4",
+              "Father": {
+                "First Name": "Father_First_5",
+                "Last Name": "Father_Last_5",
+                "Father": null
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "First Name": "Frederick",
+      "Last Name": "Boyd",
+      "Email": "kempur@ascebec.gs",
+      "Address": {
+        "Street": "789 Oak Street",
+        "City": "Rivertown",
+        "State": "RT",
+        "Zip": "10112",
+        "Coordinates": {
+          "Latitude": "40.7128° N",
+          "Longitude": "74.0060° W"
+        }
+      },
+      "Father": {
+        "First Name": "Father_First_1",
+        "Last Name": "Father_Last_1",
+        "Father": {
+          "First Name": "Father_First_2",
+          "Last Name": "Father_Last_2",
+          "Father": {
+            "First Name": "Father_First_3",
+            "Last Name": "Father_Last_3",
+            "Father": {
+              "First Name": "Father_First_4",
+              "Last Name": "Father_Last_4",
+              "Father": {
+                "First Name": "Father_First_5",
+                "Last Name": "Father_Last_5",
+                "Father": null
+              }
+            }
+          }
+        }
+      }
+    }
+  ],
+  "orders": [
+    {
+      "ID": "1234",
+      "Amount": 5678
+    },
+    {
+      "ID": "9876",
+      "Amount": 5432
+    }
+  ]
+}