diff --git a/.changeset/slow-bottles-roll.md b/.changeset/slow-bottles-roll.md new file mode 100644 index 000000000..046fb04be --- /dev/null +++ b/.changeset/slow-bottles-roll.md @@ -0,0 +1,6 @@ +--- +'@flatfile/plugin-json-extractor': minor +'@flatfile/plugin-json-multisheet-extractor': patch +--- + +Initialize the JSON multi-sheet extractor plugin diff --git a/plugins/json-extractor/package.json b/plugins/json-extractor/package.json index b3eda79f9..58bdaebad 100644 --- a/plugins/json-extractor/package.json +++ b/plugins/json-extractor/package.json @@ -61,7 +61,7 @@ }, "license": "ISC", "dependencies": { - "@flatfile/util-extractor": "^2.2.0" + "@flatfile/util-extractor": "^2.3.0" }, "devDependencies": { "@flatfile/bundler-config-tsup": "^0.1.0" diff --git a/plugins/json-extractor/src/parser.ts b/plugins/json-extractor/src/parser.ts index c0f6c8264..7d241088b 100644 --- a/plugins/json-extractor/src/parser.ts +++ b/plugins/json-extractor/src/parser.ts @@ -1,4 +1,4 @@ -import { WorkbookCapture } from '@flatfile/util-extractor' +import { WorkbookCapture, parseSheet } from '@flatfile/util-extractor' export function parseBuffer(buffer: Buffer): WorkbookCapture { try { @@ -13,52 +13,27 @@ export function parseBuffer(buffer: Buffer): WorkbookCapture { const parsedData = JSON.parse(fileContents) || [] const results = Array.isArray(parsedData) ? parsedData : [parsedData] - // Ensure all items are objects - const filteredResults = results.filter( - (item) => typeof item === 'object' && item !== null - ) + var sheetCapture = parseSheet(results); - if (filteredResults.length === 0) { + if (isEmpty(sheetCapture)) { return {} as WorkbookCapture } - // Custom flatten function - const flattenObject = (obj: any, parent: string = '', res: any = {}) => { - for (let key in obj) { - const propName = parent ? parent + '.' + key : key - if (typeof obj[key] === 'object') { - flattenObject(obj[key], propName, res) - } else { - res[propName] = obj[key] - } - } - return res - } - - // Flatten the first item to determine headers - const firstItem = flattenObject(filteredResults[0]) - const headers = Object.keys(firstItem).filter((header) => header !== '') - - // Flatten and filter all rows - const filteredData = filteredResults.map((row) => { - const flattedRow = flattenObject(row) - return headers.reduce((filteredRow, header) => { - const cell = flattedRow[header] - filteredRow[header] = { - value: Array.isArray(cell) ? JSON.stringify(cell) : cell, - } - return filteredRow - }, {}) - }) - return { - [sheetName]: { - headers, - data: filteredData, - }, + [sheetName]: sheetCapture, } as WorkbookCapture } catch (error) { console.error('An error occurred:', error) throw error } } + +function isEmpty(obj) { + for (const prop in obj) { + if (Object.prototype.hasOwnProperty.call(obj, prop)) { + return false; + } + } + + return true; +} \ No newline at end of file diff --git a/plugins/json-multisheet-extractor/CHANGELOG.md b/plugins/json-multisheet-extractor/CHANGELOG.md new file mode 100644 index 000000000..e88bce63f --- /dev/null +++ b/plugins/json-multisheet-extractor/CHANGELOG.md @@ -0,0 +1,11 @@ +# @flatfile/plugin-json-extractor + +## 0.1.0 + +### Minor Changes + +- 65b526d: Initial release of the JSON multi-sheet extractor + +### Patch Changes + +- b557dde: add readme diff --git a/plugins/json-multisheet-extractor/README.md b/plugins/json-multisheet-extractor/README.md new file mode 100644 index 000000000..9afbb6867 --- /dev/null +++ b/plugins/json-multisheet-extractor/README.md @@ -0,0 +1,78 @@ + + +The `@flatfile/json-multisheet-extractor` plugin parses a JSON file and extracts second-level nested objects into first level defined Sheets in Flatfile. + +**Event Type:** +`listener.on('file:created')` + +**Supported file types:** +`.json` + + + +> When embedding Flatfile, this plugin should be deployed in a server-side listener. [Learn more](/docs/orchestration/listeners#listener-types) + + + +## Parameters + +#### `options.chunkSize` - `default: "10_000"` - `number` - (optional) +The `chunkSize` parameter allows you to specify the quantity of records to in each chunk. + + +#### `options.parallel` - `default: "1"` - `number` - (optional) +The `parallel` parameter allows you to specify the number of chunks to process in parallel. + + + +## API Calls + +- `api.files.download` +- `api.files.get` +- `api.files.update` +- `api.jobs.ack` +- `api.jobs.complete` +- `api.jobs.create` +- `api.jobs.fail` +- `api.jobs.update` +- `api.records.insert` +- `api.workbooks.create` + + + +## Usage + +Listen for a JSON file to be uploaded to Flatfile. The platform will then extract the file automatically. Once complete, the file will be ready for import in the Files area. + +```bash Install +npm i @flatfile/plugin-json-multisheet-extractor +``` + +```js import +import { JSONMultiSheetExtractor } from "@flatfile/plugin-json-multisheet-extractor"; +``` + +```js listener.js +listener.use(JSONMultiSheetExtractor()); +``` + + + +### Full Example + +In this example, the `JSONMultiSheetExtractor` is initialized, and then registered as middleware with the Flatfile listener. When a JSON file is uploaded, the plugin will extract the structured data and process it the extractor's parser. + +```javascript +import { JSONMultiSheetExtractor } from "@flatfile/plugin-json-multisheet-extractor"; + +// Initialize the JSON multi-sheet extractor +const jsonMultiSheetExtractor = JSONMultiSheetExtractor(); + +// Register the extractor as a middleware for the Flatfile listener +listener.use(jsonMultiSheetExtractor); + +// When a JSON file is uploaded, the data will be extracted and processed using the extractor's parser. +``` + +See a working example in our [flatfile-docs-kitchen-sink](https://github.com/FlatFilers/flatfile-docs-kitchen-sink/blob/main/typescript/extractors/index.ts) Github repo. + diff --git a/plugins/json-multisheet-extractor/jest.config.cjs b/plugins/json-multisheet-extractor/jest.config.cjs new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/plugins/json-multisheet-extractor/jest.config.cjs @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/plugins/json-multisheet-extractor/package.json b/plugins/json-multisheet-extractor/package.json new file mode 100644 index 000000000..7c03b7d0d --- /dev/null +++ b/plugins/json-multisheet-extractor/package.json @@ -0,0 +1,69 @@ +{ + "name": "@flatfile/plugin-json-multisheet-extractor", + "version": "0.1.0", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/plugins/json-multisheet-extractor", + "description": "A plugin for parsing json files into multiple sheets in Flatfile.", + "registryMetadata": { + "category": "extractors" + }, + "type": "module", + "engines": { + "node": ">= 16" + }, + "exports": { + ".": { + "node": { + "types": { + "import": "./dist/index.d.ts", + "require": "./dist/index.d.cts" + }, + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "browser": { + "types": { + "import": "./dist/index.d.ts", + "require": "./dist/index.d.cts" + }, + "import": "./dist/index.browser.js", + "require": "./dist/index.browser.cjs" + }, + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "tsup", + "build:watch": "tsup --watch", + "build:prod": "NODE_ENV=production tsup", + "checks": "tsc --noEmit && attw --pack . && publint .", + "lint": "tsc --noEmit", + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" + }, + "keywords": [ + "flatfile-plugins", + "category-extractors" + ], + "author": "Amaar Bakir", + "repository": { + "type": "git", + "url": "git+https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/json-multisheet-extractor" + }, + "license": "ISC", + "dependencies": { + "@flatfile/util-extractor": "^2.3.0" + }, + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.1.0" + } +} diff --git a/plugins/json-multisheet-extractor/ref/test-basic.json b/plugins/json-multisheet-extractor/ref/test-basic.json new file mode 100644 index 000000000..ac44af7e8 --- /dev/null +++ b/plugins/json-multisheet-extractor/ref/test-basic.json @@ -0,0 +1,122 @@ +{ + "contacts": [ + { + "First Name": "Tony", + "Last Name": "Lamb", + "Email": "me@opbaj.tp", + "Address": { + "Street": "123 Main Street", + "City": "Springfield", + "State": "ST", + "Zip": "12345", + "Coordinates": { + "Latitude": "40.7128° N", + "Longitude": "74.0060° W" + } + }, + "Father": { + "First Name": "Father_First_1", + "Last Name": "Father_Last_1", + "Father": { + "First Name": "Father_First_2", + "Last Name": "Father_Last_2", + "Father": { + "First Name": "Father_First_3", + "Last Name": "Father_Last_3", + "Father": { + "First Name": "Father_First_4", + "Last Name": "Father_Last_4", + "Father": { + "First Name": "Father_First_5", + "Last Name": "Father_Last_5", + "Father": null + } + } + } + } + } + }, + { + "First Name": "Christian", + "Last Name": "Ramos", + "Email": "uw@ag.tg", + "Address": { + "Street": "456 Elm Street", + "City": "Greenville", + "State": "GT", + "Zip": "67890", + "Coordinates": { + "Latitude": "40.7128° N", + "Longitude": "74.0060° W" + } + }, + "Father": { + "First Name": "Father_First_1", + "Last Name": "Father_Last_1", + "Father": { + "First Name": "Father_First_2", + "Last Name": "Father_Last_2", + "Father": { + "First Name": "Father_First_3", + "Last Name": "Father_Last_3", + "Father": { + "First Name": "Father_First_4", + "Last Name": "Father_Last_4", + "Father": { + "First Name": "Father_First_5", + "Last Name": "Father_Last_5", + "Father": null + } + } + } + } + } + }, + { + "First Name": "Frederick", + "Last Name": "Boyd", + "Email": "kempur@ascebec.gs", + "Address": { + "Street": "789 Oak Street", + "City": "Rivertown", + "State": "RT", + "Zip": "10112", + "Coordinates": { + "Latitude": "40.7128° N", + "Longitude": "74.0060° W" + } + }, + "Father": { + "First Name": "Father_First_1", + "Last Name": "Father_Last_1", + "Father": { + "First Name": "Father_First_2", + "Last Name": "Father_Last_2", + "Father": { + "First Name": "Father_First_3", + "Last Name": "Father_Last_3", + "Father": { + "First Name": "Father_First_4", + "Last Name": "Father_Last_4", + "Father": { + "First Name": "Father_First_5", + "Last Name": "Father_Last_5", + "Father": null + } + } + } + } + } + } + ], + "orders": [ + { + "ID": "1234", + "Amount": 5678 + }, + { + "ID": "9876", + "Amount": 5432 + } + ] +} \ No newline at end of file diff --git a/plugins/json-multisheet-extractor/src/index.ts b/plugins/json-multisheet-extractor/src/index.ts new file mode 100644 index 000000000..38399a1f6 --- /dev/null +++ b/plugins/json-multisheet-extractor/src/index.ts @@ -0,0 +1,12 @@ +import { Extractor } from '@flatfile/util-extractor' +import { parseBuffer } from './parser' + +export const JSONMultiSheetExtractor = (options?: { + chunkSize?: number + parallel?: number + debug?: boolean +}) => { + return Extractor('.json', 'json', parseBuffer, options) +} + +export const jsonParser = parseBuffer \ No newline at end of file diff --git a/plugins/json-multisheet-extractor/src/parser.spec.ts b/plugins/json-multisheet-extractor/src/parser.spec.ts new file mode 100644 index 000000000..1fb784aa7 --- /dev/null +++ b/plugins/json-multisheet-extractor/src/parser.spec.ts @@ -0,0 +1,170 @@ +import * as fs from 'fs' +import * as path from 'path' +import { parseBuffer } from './parser' + +describe('parser', function () { + const buffer: Buffer = fs.readFileSync( + path.join(__dirname, '../ref/test-basic.json') + ) + + test('JSON to WorkbookCapture', () => { + const capture = parseBuffer(buffer) + expect(capture).toEqual({ + contacts: { + headers: [ + 'First Name', + 'Last Name', + 'Email', + 'Address.Street', + 'Address.City', + 'Address.State', + 'Address.Zip', + 'Address.Coordinates.Latitude', + 'Address.Coordinates.Longitude', + 'Father.First Name', + 'Father.Last Name', + 'Father.Father.First Name', + 'Father.Father.Last Name', + 'Father.Father.Father.First Name', + 'Father.Father.Father.Last Name', + 'Father.Father.Father.Father.First Name', + 'Father.Father.Father.Father.Last Name', + 'Father.Father.Father.Father.Father.First Name', + 'Father.Father.Father.Father.Father.Last Name', + ], + data: [ + { + 'First Name': { value: 'Tony' }, + 'Last Name': { value: 'Lamb' }, + Email: { value: 'me@opbaj.tp' }, + 'Address.Street': { value: '123 Main Street' }, + 'Address.City': { value: 'Springfield' }, + 'Address.State': { value: 'ST' }, + 'Address.Zip': { value: '12345' }, + 'Address.Coordinates.Latitude': { value: '40.7128° N' }, + 'Address.Coordinates.Longitude': { value: '74.0060° W' }, + 'Father.First Name': { value: 'Father_First_1' }, + 'Father.Last Name': { value: 'Father_Last_1' }, + 'Father.Father.First Name': { value: 'Father_First_2' }, + 'Father.Father.Last Name': { value: 'Father_Last_2' }, + 'Father.Father.Father.First Name': { value: 'Father_First_3' }, + 'Father.Father.Father.Last Name': { value: 'Father_Last_3' }, + 'Father.Father.Father.Father.First Name': { + value: 'Father_First_4', + }, + 'Father.Father.Father.Father.Last Name': { value: 'Father_Last_4' }, + 'Father.Father.Father.Father.Father.First Name': { + value: 'Father_First_5', + }, + 'Father.Father.Father.Father.Father.Last Name': { + value: 'Father_Last_5', + }, + }, + { + 'First Name': { value: 'Christian' }, + 'Last Name': { value: 'Ramos' }, + Email: { value: 'uw@ag.tg' }, + 'Address.Street': { value: '456 Elm Street' }, + 'Address.City': { value: 'Greenville' }, + 'Address.State': { value: 'GT' }, + 'Address.Zip': { value: '67890' }, + 'Address.Coordinates.Latitude': { value: '40.7128° N' }, + 'Address.Coordinates.Longitude': { value: '74.0060° W' }, + 'Father.First Name': { value: 'Father_First_1' }, + 'Father.Last Name': { value: 'Father_Last_1' }, + 'Father.Father.First Name': { value: 'Father_First_2' }, + 'Father.Father.Last Name': { value: 'Father_Last_2' }, + 'Father.Father.Father.First Name': { value: 'Father_First_3' }, + 'Father.Father.Father.Last Name': { value: 'Father_Last_3' }, + 'Father.Father.Father.Father.First Name': { + value: 'Father_First_4', + }, + 'Father.Father.Father.Father.Last Name': { value: 'Father_Last_4' }, + 'Father.Father.Father.Father.Father.First Name': { + value: 'Father_First_5', + }, + 'Father.Father.Father.Father.Father.Last Name': { + value: 'Father_Last_5', + }, + }, + { + 'First Name': { value: 'Frederick' }, + 'Last Name': { value: 'Boyd' }, + Email: { value: 'kempur@ascebec.gs' }, + 'Address.Street': { value: '789 Oak Street' }, + 'Address.City': { value: 'Rivertown' }, + 'Address.State': { value: 'RT' }, + 'Address.Zip': { value: '10112' }, + 'Address.Coordinates.Latitude': { value: '40.7128° N' }, + 'Address.Coordinates.Longitude': { value: '74.0060° W' }, + 'Father.First Name': { value: 'Father_First_1' }, + 'Father.Last Name': { value: 'Father_Last_1' }, + 'Father.Father.First Name': { value: 'Father_First_2' }, + 'Father.Father.Last Name': { value: 'Father_Last_2' }, + 'Father.Father.Father.First Name': { value: 'Father_First_3' }, + 'Father.Father.Father.Last Name': { value: 'Father_Last_3' }, + 'Father.Father.Father.Father.First Name': { + value: 'Father_First_4', + }, + 'Father.Father.Father.Father.Last Name': { value: 'Father_Last_4' }, + 'Father.Father.Father.Father.Father.First Name': { + value: 'Father_First_5', + }, + 'Father.Father.Father.Father.Father.Last Name': { + value: 'Father_Last_5', + }, + }, + ], + metadata: undefined, + }, + orders: { + headers: [ + 'ID', + 'Amount', + ], + data: [ + { + ID: { value: "1234" }, + Amount: { value: 5678 } + }, + { + ID: { value: "9876" }, + Amount: { value: 5432 } + } + ], + metadata: undefined, + }, + }) + }) + it('has contact headers', () => { + const headers = parseBuffer(buffer).contacts.headers + expect(headers).toEqual([ + 'First Name', + 'Last Name', + 'Email', + 'Address.Street', + 'Address.City', + 'Address.State', + 'Address.Zip', + 'Address.Coordinates.Latitude', + 'Address.Coordinates.Longitude', + 'Father.First Name', + 'Father.Last Name', + 'Father.Father.First Name', + 'Father.Father.Last Name', + 'Father.Father.Father.First Name', + 'Father.Father.Father.Last Name', + 'Father.Father.Father.Father.First Name', + 'Father.Father.Father.Father.Last Name', + 'Father.Father.Father.Father.Father.First Name', + 'Father.Father.Father.Father.Father.Last Name', + ]) + }) + it('has order headers', () => { + const headers = parseBuffer(buffer).orders.headers + expect(headers).toEqual([ + 'ID', + 'Amount', + ]) + }) +}) diff --git a/plugins/json-multisheet-extractor/src/parser.ts b/plugins/json-multisheet-extractor/src/parser.ts new file mode 100644 index 000000000..2e95941f5 --- /dev/null +++ b/plugins/json-multisheet-extractor/src/parser.ts @@ -0,0 +1,42 @@ +import { WorkbookCapture, parseSheet } from '@flatfile/util-extractor' + + +export function parseBuffer(buffer: Buffer): WorkbookCapture { + try { + const fileContents = buffer.toString('utf8') + + if (!fileContents) { + console.log('Invalid file contents') + return {} as WorkbookCapture + } + + const parsedData = JSON.parse(fileContents) + if (typeof parsedData !== 'object' || parsedData === null) { + console.error("Invalid input: data must be an object."); + return {}; + } + + const results = {}; + + Object.keys(parsedData).forEach(sheet => { + const value = parsedData[sheet]; + + // Check if key is a string and value is an array + if (typeof sheet === 'string' && Array.isArray(value)) { + try { + const result = parseSheet(value); + results[sheet] = result; // Store result in the results object + } catch (error) { + console.error(`Error processing sheet "${sheet}":`, error); + } + } else { + console.warn(`Skipping key "${sheet}" - not a string/array pair.`); + } + }); + + return results as WorkbookCapture; + } catch (error) { + console.error('An error occurred:', error) + throw error + } +} diff --git a/plugins/json-multisheet-extractor/tsup.config.mjs b/plugins/json-multisheet-extractor/tsup.config.mjs new file mode 100644 index 000000000..bd6420831 --- /dev/null +++ b/plugins/json-multisheet-extractor/tsup.config.mjs @@ -0,0 +1,3 @@ +import { defineConfig } from '@flatfile/bundler-config-tsup' + +export default defineConfig({}) diff --git a/utils/extractor/package.json b/utils/extractor/package.json index 648864a4b..2a3f60291 100644 --- a/utils/extractor/package.json +++ b/utils/extractor/package.json @@ -1,6 +1,6 @@ { "name": "@flatfile/util-extractor", - "version": "2.2.1", + "version": "2.3.0", "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/utils/extractor", "description": "A library containing common utilities and helpers for extractors.", "keywords": [ diff --git a/utils/extractor/src/index.ts b/utils/extractor/src/index.ts index 02296aed0..4e88dfaf6 100644 --- a/utils/extractor/src/index.ts +++ b/utils/extractor/src/index.ts @@ -293,3 +293,53 @@ export type SheetCapture = { data: Flatfile.RecordData[] metadata?: { rowHeaders: number[] } } + +export function parseSheet(jsonArray: any[]): SheetCapture { + try { + // Ensure all items are objects + const filteredResults = jsonArray.filter( + (item) => typeof item === 'object' && item !== null + ) + + if (filteredResults.length === 0) { + return {} as SheetCapture + } + + // Custom flatten function + const flattenObject = (obj: any, parent: string = '', res: any = {}) => { + for (let key in obj) { + const propName = parent ? parent + '.' + key : key + if (typeof obj[key] === 'object') { + flattenObject(obj[key], propName, res) + } else { + res[propName] = obj[key] + } + } + return res + } + + // Flatten the first item to determine headers + const firstItem = flattenObject(filteredResults[0]) + const headers = Object.keys(firstItem).filter((header) => header !== '') + + // Flatten and filter all rows + const filteredData = filteredResults.map((row) => { + const flattedRow = flattenObject(row) + return headers.reduce((filteredRow, header) => { + const cell = flattedRow[header] + filteredRow[header] = { + value: Array.isArray(cell) ? JSON.stringify(cell) : cell, + } + return filteredRow + }, {}) + }) + + return { + headers, + data: filteredData, + } as SheetCapture; + } catch (error) { + console.error('An error occurred:', error) + throw error + } +} \ No newline at end of file