diff --git a/.changeset/flat-gifts-change.md b/.changeset/flat-gifts-change.md new file mode 100644 index 000000000..858a26df7 --- /dev/null +++ b/.changeset/flat-gifts-change.md @@ -0,0 +1,6 @@ +--- +'@flatfile/plugin-delimiter-extractor': minor +'@flatfile/plugin-xlsx-extractor': minor +--- + +This release provides a new option `skipEmptyLines` to skip empty lines. diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index 5e9c414b2..c49e38049 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -1,25 +1,30 @@ -import type { FlatfileListener, FlatfileEvent } from '@flatfile/listener' -import { ExcelExtractor } from '@flatfile/plugin-xlsx-extractor' +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { automap } from '@flatfile/plugin-automap' import { DelimiterExtractor } from '@flatfile/plugin-delimiter-extractor' -import { automap } from "@flatfile/plugin-automap"; - +import { ExcelExtractor } from '@flatfile/plugin-xlsx-extractor' export default async function (listener: FlatfileListener) { - listener.use(ExcelExtractor()) - listener.use(DelimiterExtractor('txt', { delimiter: ',' })) + listener.use( + ExcelExtractor({ + skipEmptyLines: true, + }) + ) + listener.use( + DelimiterExtractor('txt', { delimiter: ',', skipEmptyLines: true }) + ) listener.use( automap({ - accuracy: "confident", - defaultTargetSheet: "contacts", + accuracy: 'confident', + defaultTargetSheet: 'contacts', matchFilename: /test/, debug: true, onFailure: (event: FlatfileEvent) => { // send an SMS, an email, post to an endpoint, etc. console.error( `Please visit https://spaces.flatfile.com/space/${event.context.spaceId}/files?mode=import to manually import file.` - ); + ) }, }) - ); + ) } diff --git a/package-lock.json b/package-lock.json index fe6b131b5..faf3a962f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "flatfilers/*", "plugins/*", "support/*", - "utils/*" + "utils/*", + "validators/*" ], "devDependencies": { "@changesets/cli": "^2.26.1", @@ -3649,10 +3650,6 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" }, - "node_modules/@flatfile/plugin-ai-extractor": { - "resolved": "plugins/ai-extractor", - "link": true - }, "node_modules/@flatfile/plugin-autocast": { "resolved": "plugins/autocast", "link": true @@ -3709,10 +3706,6 @@ "resolved": "plugins/graphql-schema", "link": true }, - "node_modules/@flatfile/plugin-isbn-validator": { - "resolved": "plugins/isbn-validator", - "link": true - }, "node_modules/@flatfile/plugin-job-handler": { "resolved": "plugins/job-handler", "link": true @@ -9472,6 +9465,7 @@ "version": "2.6.11", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dev": true, "dependencies": { "@types/node": "*", "form-data": "^4.0.0" @@ -13025,23 +13019,6 @@ "node": ">= 6" } }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -17976,24 +17953,6 @@ "node": ">= 0.10.5" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -18260,50 +18219,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/openai": { - "version": "4.63.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.63.0.tgz", - "integrity": "sha512-Y9V4KODbmrOpqiOmCDVnPfMxMqKLOx8Hwcdn/r8mePq4yv7FSXGnxCs8/jZKO7zCB/IVPWihpJXwJNAIOEiZ2g==", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.23.8" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.50", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.50.tgz", - "integrity": "sha512-xonK+NRrMBRtkL1hVCc3G+uXtjh1Al4opBLjqVmipe5ZAaBYWW6cNAiBVZ1BvmkBhep698rP3UM3aRAdSALuhg==", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/openai/node_modules/agentkeepalive": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", - "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/openapi-typescript-codegen": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/openapi-typescript-codegen/-/openapi-typescript-codegen-0.23.0.tgz", @@ -22497,14 +22412,6 @@ "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", "dev": true }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "engines": { - "node": ">= 14" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -22902,7 +22809,7 @@ "version": "3.23.8", "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", - "devOptional": true, + "dev": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -22910,6 +22817,7 @@ "plugins/ai-extractor": { "name": "@flatfile/plugin-ai-extractor", "version": "0.1.0", + "extraneous": true, "license": "ISC", "dependencies": { "@flatfile/util-extractor": "^2.1.4", @@ -22924,14 +22832,14 @@ }, "plugins/autocast": { "name": "@flatfile/plugin-autocast", - "version": "1.0.1", + "version": "2.0.0", "license": "ISC", "dependencies": { "@flatfile/hooks": "^1.4.1", - "@flatfile/util-common": "^1.3.8" + "@flatfile/util-common": "^1.4.0" }, "devDependencies": { - "@flatfile/plugin-record-hook": "^1.6.1", + "@flatfile/plugin-record-hook": "^1.7.0", "@flatfile/rollup-config": "0.1.1" }, "engines": { @@ -22940,19 +22848,19 @@ "peerDependencies": { "@flatfile/api": "^1.9.13", "@flatfile/listener": "^1.0.5", - "@flatfile/plugin-record-hook": "^1.6.1" + "@flatfile/plugin-record-hook": "^1.7.0" } }, "plugins/automap": { "name": "@flatfile/plugin-automap", - "version": "0.4.1", + "version": "0.5.0", "license": "ISC", "dependencies": { "@flatfile/common-plugin-utils": "^1.0.2", "remeda": "^1.23.0" }, "devDependencies": { - "@flatfile/utils-testing": "^0.2.1" + "@flatfile/utils-testing": "^0.3.0" }, "engines": { "node": ">= 16" @@ -22964,10 +22872,10 @@ }, "plugins/constraints": { "name": "@flatfile/plugin-constraints", - "version": "2.0.1", + "version": "3.0.0", "license": "ISC", "devDependencies": { - "@flatfile/plugin-record-hook": "^1.6.1", + "@flatfile/plugin-record-hook": "^1.7.0", "@flatfile/rollup-config": "0.1.1" }, "engines": { @@ -22976,16 +22884,16 @@ "peerDependencies": { "@flatfile/api": "^1.9.13", "@flatfile/listener": "^1.0.5", - "@flatfile/plugin-record-hook": "^1.6.1" + "@flatfile/plugin-record-hook": "^1.7.0" } }, "plugins/dedupe": { "name": "@flatfile/plugin-dedupe", - "version": "1.1.5", + "version": "1.2.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.5", - "@flatfile/util-common": "^1.3.8" + "@flatfile/plugin-job-handler": "^0.6.0", + "@flatfile/util-common": "^1.4.0" }, "devDependencies": { "@faker-js/faker": "^7.6.0", @@ -23021,7 +22929,7 @@ }, "plugins/dxp-configure": { "name": "@flatfile/plugin-dxp-configure", - "version": "1.1.3", + "version": "1.2.0", "license": "ISC", "devDependencies": { "@flatfile/configure": "^1.0.1", @@ -23037,11 +22945,11 @@ }, "plugins/export-workbook": { "name": "@flatfile/plugin-export-workbook", - "version": "0.3.1", + "version": "1.0.0", "license": "ISC", "devDependencies": { - "@flatfile/plugin-job-handler": "^0.5.5", - "@flatfile/util-common": "^1.3.8" + "@flatfile/plugin-job-handler": "^0.6.0", + "@flatfile/util-common": "^1.4.0" }, "engines": { "node": ">= 16" @@ -23049,15 +22957,15 @@ "peerDependencies": { "@flatfile/api": "^1.9.13", "@flatfile/listener": "^1.0.5", - "@flatfile/plugin-job-handler": "^0.5.5", - "@flatfile/util-common": "^1.3.8", + "@flatfile/plugin-job-handler": "^0.6.0", + "@flatfile/util-common": "^1.4.0", "remeda": "^1.14.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" } }, "plugins/foreign-db-extractor": { "name": "@flatfile/plugin-foreign-db-extractor", - "version": "0.1.3", + "version": "0.2.0", "license": "ISC", "dependencies": { "cross-fetch": "^4.0.0", @@ -23076,10 +22984,10 @@ }, "plugins/graphql-schema": { "name": "@flatfile/plugin-graphql-schema", - "version": "1.2.2", + "version": "1.3.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-space-configure": "^0.5.1", + "@flatfile/plugin-space-configure": "^0.6.0", "change-case": "^5.4.3", "cross-fetch": "^4.0.0", "graphql": "^16.8.1" @@ -23094,6 +23002,7 @@ "plugins/isbn-validator": { "name": "@flatfile/plugin-isbn-validator", "version": "1.0.0", + "extraneous": true, "license": "MIT", "dependencies": { "@flatfile/plugin-record-hook": "^1.6.1" @@ -23107,29 +23016,16 @@ "@flatfile/listener": "^1.0.5" } }, - "plugins/isbn-validator/node_modules/typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", - "dev": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, "plugins/job-handler": { "name": "@flatfile/plugin-job-handler", - "version": "0.5.5", + "version": "0.6.0", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.8" + "@flatfile/util-common": "^1.4.0" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.2.1" + "@flatfile/utils-testing": "^0.3.0" }, "engines": { "node": ">= 16" @@ -23155,10 +23051,10 @@ }, "plugins/json-schema": { "name": "@flatfile/plugin-convert-json-schema", - "version": "0.3.4", + "version": "0.4.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-space-configure": "^0.5.3", + "@flatfile/plugin-space-configure": "^0.6.0", "cross-fetch": "^4.0.0" }, "engines": { @@ -23171,11 +23067,11 @@ }, "plugins/merge-connection": { "name": "@flatfile/plugin-connect-via-merge", - "version": "0.3.3", + "version": "0.4.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-convert-openapi-schema": "^0.2.4", - "@flatfile/plugin-job-handler": "^0.5.5", + "@flatfile/plugin-convert-openapi-schema": "^0.3.0", + "@flatfile/plugin-job-handler": "^0.6.0", "@mergeapi/merge-node-client": "^1.0.4" }, "engines": { @@ -23188,10 +23084,10 @@ }, "plugins/openapi-schema": { "name": "@flatfile/plugin-convert-openapi-schema", - "version": "0.2.4", + "version": "0.3.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-space-configure": "^0.5.3", + "@flatfile/plugin-space-configure": "^0.6.0", "cross-fetch": "^4.0.0" }, "devDependencies": { @@ -23207,11 +23103,11 @@ }, "plugins/pdf-extractor": { "name": "@flatfile/plugin-pdf-extractor", - "version": "0.2.3", + "version": "0.3.0", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.8", - "@flatfile/util-file-buffer": "^0.3.3", + "@flatfile/util-common": "^1.4.0", + "@flatfile/util-file-buffer": "^0.4.0", "cross-fetch": "^4.0.0", "form-data": "^4.0.0", "fs-extra": "^11.1.1", @@ -23262,7 +23158,7 @@ }, "plugins/psv-extractor": { "name": "@flatfile/plugin-psv-extractor", - "version": "1.8.3", + "version": "1.9.0", "license": "ISC", "engines": { "node": ">= 16" @@ -23274,10 +23170,10 @@ }, "plugins/record-hook": { "name": "@flatfile/plugin-record-hook", - "version": "1.6.1", + "version": "1.7.0", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.8" + "@flatfile/util-common": "^1.4.0" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1" @@ -23293,11 +23189,11 @@ }, "plugins/rollout": { "name": "@flatfile/plugin-rollout", - "version": "1.0.2", + "version": "1.1.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.5", - "@flatfile/util-common": "^1.3.8", + "@flatfile/plugin-job-handler": "^0.6.0", + "@flatfile/util-common": "^1.4.0", "async": "^3.2.5", "modern-async": "^2.0.0" }, @@ -23314,15 +23210,15 @@ }, "plugins/space-configure": { "name": "@flatfile/plugin-space-configure", - "version": "0.5.3", + "version": "0.6.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.5" + "@flatfile/plugin-job-handler": "^0.6.0" }, "devDependencies": { "@flatfile/api": "^1.9.13", "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.2.1" + "@flatfile/utils-testing": "^0.3.0" }, "engines": { "node": ">= 16" @@ -23334,11 +23230,11 @@ }, "plugins/sql-ddl-converter": { "name": "@flatfile/plugin-convert-sql-ddl", - "version": "0.1.4", + "version": "0.2.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-convert-json-schema": "^0.3.4", - "@flatfile/plugin-space-configure": "^0.5.3", + "@flatfile/plugin-convert-json-schema": "^0.4.0", + "@flatfile/plugin-space-configure": "^0.6.0", "sql-ddl-to-json-schema": "^4.1.0" }, "engines": { @@ -23351,7 +23247,7 @@ }, "plugins/tsv-extractor": { "name": "@flatfile/plugin-tsv-extractor", - "version": "1.7.3", + "version": "1.8.0", "license": "ISC", "engines": { "node": ">= 16" @@ -23371,7 +23267,7 @@ }, "devDependencies": { "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.2.0" + "@flatfile/utils-testing": "^0.3.0" }, "engines": { "node": ">= 16" @@ -23379,17 +23275,17 @@ }, "plugins/webhook-egress": { "name": "@flatfile/plugin-webhook-egress", - "version": "1.3.3", + "version": "1.4.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.5", - "@flatfile/util-common": "^1.3.8", - "@flatfile/util-response-rejection": "^1.3.6", + "@flatfile/plugin-job-handler": "^0.6.0", + "@flatfile/util-common": "^1.4.0", + "@flatfile/util-response-rejection": "^1.4.0", "cross-fetch": "^4.0.0" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.2.1", + "@flatfile/utils-testing": "^0.3.0", "jest-fetch-mock": "^3.0.3" }, "engines": { @@ -23402,7 +23298,7 @@ }, "plugins/webhook-event-forwarder": { "name": "@flatfile/plugin-webhook-event-forwarder", - "version": "0.3.3", + "version": "0.4.0", "license": "ISC", "dependencies": { "cross-fetch": "^4.0.0" @@ -23421,10 +23317,10 @@ }, "plugins/xlsx-extractor": { "name": "@flatfile/plugin-xlsx-extractor", - "version": "3.1.4", + "version": "3.1.5", "license": "ISC", "dependencies": { - "@flatfile/util-extractor": "^2.1.3", + "@flatfile/util-extractor": "^2.1.5", "remeda": "^1.14.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" }, @@ -23437,10 +23333,10 @@ }, "plugins/xml-extractor": { "name": "@flatfile/plugin-xml-extractor", - "version": "0.6.2", + "version": "0.7.0", "license": "ISC", "dependencies": { - "@flatfile/util-extractor": "^2.1.2", + "@flatfile/util-extractor": "^2.1.6", "remeda": "^1.24.0", "xml-json-format": "^1.0.8" }, @@ -23453,16 +23349,16 @@ }, "plugins/yaml-schema": { "name": "@flatfile/plugin-convert-yaml-schema", - "version": "0.2.4", + "version": "0.3.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-convert-json-schema": "^0.3.4", - "@flatfile/plugin-space-configure": "^0.5.3", + "@flatfile/plugin-convert-json-schema": "^0.4.0", + "@flatfile/plugin-space-configure": "^0.6.0", "cross-fetch": "^4.0.0", "js-yaml": "^4.1.0" }, "devDependencies": { - "@flatfile/utils-testing": "^0.2.1", + "@flatfile/utils-testing": "^0.3.0", "express": "^4.18.2", "jest-fetch-mock": "^3.0.3" }, @@ -23492,17 +23388,17 @@ }, "plugins/zip-extractor": { "name": "@flatfile/plugin-zip-extractor", - "version": "0.5.4", + "version": "0.6.0", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.5", - "@flatfile/util-common": "^1.3.8", - "@flatfile/util-file-buffer": "^0.3.3", + "@flatfile/plugin-job-handler": "^0.6.0", + "@flatfile/util-common": "^1.4.0", + "@flatfile/util-file-buffer": "^0.4.0", "adm-zip": "^0.5.10", "modern-async": "^2.0.0" }, "devDependencies": { - "@flatfile/utils-testing": "^0.2.1", + "@flatfile/utils-testing": "^0.3.0", "@types/adm-zip": "^0.4.3" }, "engines": { @@ -23523,7 +23419,7 @@ }, "utils/common": { "name": "@flatfile/util-common", - "version": "1.3.8", + "version": "1.4.0", "license": "ISC", "dependencies": { "@flatfile/cross-env-config": "^0.0.6", @@ -23547,11 +23443,11 @@ }, "utils/extractor": { "name": "@flatfile/util-extractor", - "version": "2.1.4", + "version": "2.1.6", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.7", - "@flatfile/util-file-buffer": "^0.3.2" + "@flatfile/util-common": "^1.4.0", + "@flatfile/util-file-buffer": "^0.4.0" }, "engines": { "node": ">= 16" @@ -23577,7 +23473,7 @@ }, "utils/file-buffer": { "name": "@flatfile/util-file-buffer", - "version": "0.3.3", + "version": "0.4.0", "license": "ISC", "engines": { "node": ">= 16" @@ -23589,10 +23485,10 @@ }, "utils/response-rejection": { "name": "@flatfile/util-response-rejection", - "version": "1.3.6", + "version": "1.4.0", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.8" + "@flatfile/util-common": "^1.4.0" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1" @@ -23606,7 +23502,7 @@ }, "utils/testing": { "name": "@flatfile/utils-testing", - "version": "0.2.1", + "version": "0.3.0", "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.13", diff --git a/plugins/delimiter-extractor/ref/csv.txt b/plugins/delimiter-extractor/ref/csv.txt new file mode 100644 index 000000000..871f047ec --- /dev/null +++ b/plugins/delimiter-extractor/ref/csv.txt @@ -0,0 +1,9 @@ +header1,header2,header3 +column1,column2,column3 +,, +,, +column4,column5,column6 +column7,column8,column9 +,, +,, +,, diff --git a/plugins/delimiter-extractor/src/parser.spec.ts b/plugins/delimiter-extractor/src/parser.spec.ts index 442f6e2c4..b09a50190 100644 --- a/plugins/delimiter-extractor/src/parser.spec.ts +++ b/plugins/delimiter-extractor/src/parser.spec.ts @@ -11,6 +11,10 @@ describe('parser', () => { path.join(__dirname, '../ref/test-complex.pound') ) + const csvBuffer: Buffer = fs.readFileSync( + path.join(__dirname, '../ref/csv.txt') + ) + test('colon to WorkbookCapture', async () => { expect(await parseBuffer(colonBasicBuffer, { delimiter: '#' })).toEqual({ Sheet1: { @@ -135,4 +139,62 @@ describe('parser', () => { parseBuffer(colonBasicBuffer, { delimiter: '#' }) ) }) + test('skip empty lines', async () => { + const parsedBuffer = await parseBuffer(csvBuffer, { + delimiter: ',', + skipEmptyLines: true, + }) + const data = parsedBuffer.Sheet1.data + expect(data).toEqual([ + { + header1: { value: 'column1' }, + header2: { value: 'column2' }, + header3: { value: 'column3' }, + }, + { + header1: { value: 'column4' }, + header2: { value: 'column5' }, + header3: { value: 'column6' }, + }, + { + header1: { value: 'column7' }, + header2: { value: 'column8' }, + header3: { value: 'column9' }, + }, + ]) + }) + test("don'tskip empty lines", async () => { + const parsedBuffer = await parseBuffer(csvBuffer, { + delimiter: ',', + skipEmptyLines: false, + }) + const data = parsedBuffer.Sheet1.data + expect(data).toEqual([ + { + header1: { value: 'column1' }, + header2: { value: 'column2' }, + header3: { value: 'column3' }, + }, + { + header1: { value: '' }, + header2: { value: '' }, + header3: { value: '' }, + }, + { + header1: { value: '' }, + header2: { value: '' }, + header3: { value: '' }, + }, + { + header1: { value: 'column4' }, + header2: { value: 'column5' }, + header3: { value: 'column6' }, + }, + { + header1: { value: 'column7' }, + header2: { value: 'column8' }, + header3: { value: 'column9' }, + }, + ]) + }) }) diff --git a/plugins/delimiter-extractor/src/parser.ts b/plugins/delimiter-extractor/src/parser.ts index 9fca375b3..206f00017 100644 --- a/plugins/delimiter-extractor/src/parser.ts +++ b/plugins/delimiter-extractor/src/parser.ts @@ -15,6 +15,11 @@ export async function parseBuffer( options: ParseBufferOptions ): Promise { try { + const skipEmptyLines = options?.headerSelectionEnabled + ? false + : options?.skipEmptyLines === false + ? false + : 'greedy' const fileContents = buffer.toString('utf8') const results: ParseResult> = Papa.parse( fileContents, @@ -32,13 +37,11 @@ export async function parseBuffer( ], dynamicTyping: options?.dynamicTyping || false, header: false, - skipEmptyLines: options?.headerSelectionEnabled - ? false - : options?.skipEmptyLines || 'greedy', + skipEmptyLines, } ) - const rows = results.data + let rows = results.data if (!rows || !rows.length) { console.log('No data found in the file') return {} as WorkbookCapture @@ -62,18 +65,23 @@ export async function parseBuffer( return } + while ( + rows.length > 0 && + Object.values(rows[rows.length - 1]).every(isNullOrWhitespace) + ) { + rows.pop() + } + const columnHeaders = options?.headerSelectionEnabled ? letters : header const headers = prependNonUniqueHeaderColumns(columnHeaders) - const data: Flatfile.RecordData[] = rows - .filter((row) => !Object.values(row).every(isNullOrWhitespace)) - .map((row) => { - const mappedRow = mapKeys(row, (key) => headers[key]) - return mapValues(mappedRow, (value) => ({ - value: transform(value), - })) as Flatfile.RecordData - }) + const data: Flatfile.RecordData[] = rows.map((row) => { + const mappedRow = mapKeys(row, (key) => headers[key]) + return mapValues(mappedRow, (value) => ({ + value: transform(value), + })) as Flatfile.RecordData + }) let metadata: { rowHeaders: number[] } | null diff --git a/plugins/xlsx-extractor/ref/test-empty-rows.xlsx b/plugins/xlsx-extractor/ref/test-empty-rows.xlsx new file mode 100644 index 000000000..7513199d4 Binary files /dev/null and b/plugins/xlsx-extractor/ref/test-empty-rows.xlsx differ diff --git a/plugins/xlsx-extractor/src/index.ts b/plugins/xlsx-extractor/src/index.ts index c0f667097..5746cfd07 100644 --- a/plugins/xlsx-extractor/src/index.ts +++ b/plugins/xlsx-extractor/src/index.ts @@ -18,6 +18,7 @@ export interface ExcelExtractorOptions { readonly rawNumbers?: boolean readonly dateNF?: string readonly headerDetectionOptions?: GetHeadersOptions + readonly skipEmptyLines?: boolean readonly chunkSize?: number readonly parallel?: number readonly debug?: boolean diff --git a/plugins/xlsx-extractor/src/parser.spec.ts b/plugins/xlsx-extractor/src/parser.spec.ts index 5b691a382..772eba76a 100644 --- a/plugins/xlsx-extractor/src/parser.spec.ts +++ b/plugins/xlsx-extractor/src/parser.spec.ts @@ -4,90 +4,145 @@ import * as path from 'path' import { parseBuffer } from './parser' describe('parser', () => { - const buffer: Buffer = fs.readFileSync( - path.join(__dirname, '../ref/test-basic.xlsx') - ) - let capture: WorkbookCapture - beforeAll(async () => { - capture = await parseBuffer(buffer) + describe('test-basic.xlsx', () => { + const buffer: Buffer = fs.readFileSync( + path.join(__dirname, '../ref/test-basic.xlsx') + ) + let capture: WorkbookCapture + beforeAll(async () => { + capture = await parseBuffer(buffer) + }) + test('Excel to WorkbookCapture', async () => { + expect(capture.Departments).toEqual({ + headers: ['Code', 'Details', 'BranchName', 'Tenant'], + data: [ + { + Code: { value: 'Personal Care' }, + Details: { value: 'Personal Care Department' }, + BranchName: { value: null }, + Tenant: { value: 'notdata' }, + }, + { + Code: { value: ' ' }, + Details: { value: null }, + BranchName: { value: null }, + Tenant: { value: null }, + }, + { + Code: { value: 'Home Nursing' }, + Details: { value: 'Home Nursing Department' }, + BranchName: { value: null }, + Tenant: { value: 'notdata' }, + }, + ], + metadata: undefined, + }) + }) + + describe('test-basic.xlsx', () => { + test('finds all the sheet names', async () => { + expect(Object.keys(capture)).toEqual([ + 'Departments', + 'Clients', + 'Rebates-Purchases', + ]) + }) + + test('finds the header names', () => { + expect(capture['Departments'].headers).toEqual([ + 'Code', + 'Details', + 'BranchName', + 'Tenant', + ]) + }) + + test('finds values', () => { + expect(capture['Departments'].data.length).toEqual(3) + }) + + test('non-unique header values are prepended', () => { + expect(capture['Rebates-Purchases'].headers).toEqual([ + 'Name', + 'Group', + 'Rebates', + 'Purchases', + 'Rebates_1', + 'Purchases_1', + 'Rebates_2', + 'Purchases_2', + 'Rebates_3', + 'Purchases_3', + 'Rebates_4', + 'Purchases_4', + 'Rebates_5', + 'Purchases_5', + 'Rebates_6', + 'Purchases_6', + 'Rebates_7', + 'Purchases_7', + 'Rebates_8', + 'Purchases_8', + 'Rebates_9', + 'Purchases_9', + 'Rebates_10', + 'Purchases_10', + 'Rebates_11', + 'Purchases_11', + ]) + }) + }) }) - test('Excel to WorkbookCapture', async () => { - expect(capture.Departments).toEqual({ - headers: ['Code', 'Details', 'BranchName', 'Tenant'], - data: [ + describe('skipEmptyLines', () => { + const buffer: Buffer = fs.readFileSync( + path.join(__dirname, '../ref/test-empty-rows.xlsx') + ) + let capture: WorkbookCapture + test('skips empty lines', async () => { + const capture = await parseBuffer(buffer, { skipEmptyLines: true }) + const data = capture['Sheet1'].data + expect(data.length).toEqual(3) + expect(data).toEqual([ { - Code: { value: 'Personal Care' }, - Details: { value: 'Personal Care Department' }, - BranchName: { value: null }, - Tenant: { value: 'notdata' }, + header1: { value: 'column1' }, + header2: { value: 'column2' }, + header3: { value: 'column3' }, }, { - Code: { value: ' ' }, - Details: { value: null }, - BranchName: { value: null }, - Tenant: { value: null }, + header1: { value: 'column4' }, + header2: { value: 'column5' }, + header3: { value: 'column6' }, }, { - Code: { value: 'Home Nursing' }, - Details: { value: 'Home Nursing Department' }, - BranchName: { value: null }, - Tenant: { value: 'notdata' }, + header1: { value: 'column7' }, + header2: { value: 'column8' }, + header3: { value: 'column9' }, }, - ], - metadata: undefined, - }) - }) - - describe('test-basic.xlsx', () => { - test('finds all the sheet names', async () => { - expect(Object.keys(capture)).toEqual([ - 'Departments', - 'Clients', - 'Rebates-Purchases', ]) }) - - test('finds the header names', () => { - expect(capture['Departments'].headers).toEqual([ - 'Code', - 'Details', - 'BranchName', - 'Tenant', - ]) - }) - - test('finds values', () => { - expect(capture['Departments'].data.length).toEqual(3) - }) - - test('non-unique header values are prepended', () => { - expect(capture['Rebates-Purchases'].headers).toEqual([ - 'Name', - 'Group', - 'Rebates', - 'Purchases', - 'Rebates_1', - 'Purchases_1', - 'Rebates_2', - 'Purchases_2', - 'Rebates_3', - 'Purchases_3', - 'Rebates_4', - 'Purchases_4', - 'Rebates_5', - 'Purchases_5', - 'Rebates_6', - 'Purchases_6', - 'Rebates_7', - 'Purchases_7', - 'Rebates_8', - 'Purchases_8', - 'Rebates_9', - 'Purchases_9', - 'Rebates_10', - 'Purchases_10', - 'Rebates_11', - 'Purchases_11', + test("doesn't skip empty lines", async () => { + const capture = await parseBuffer(buffer, { skipEmptyLines: false }) + const data = capture['Sheet1'].data + expect(data.length).toEqual(6) + expect(data).toEqual([ + { + header1: { value: 'column1' }, + header2: { value: 'column2' }, + header3: { value: 'column3' }, + }, + {}, + { + header1: { value: 'column4' }, + header2: { value: 'column5' }, + header3: { value: 'column6' }, + }, + {}, + {}, + { + header1: { value: 'column7' }, + header2: { value: 'column8' }, + header3: { value: 'column9' }, + }, ]) }) }) diff --git a/plugins/xlsx-extractor/src/parser.ts b/plugins/xlsx-extractor/src/parser.ts index 2cfe7fe37..810b89664 100644 --- a/plugins/xlsx-extractor/src/parser.ts +++ b/plugins/xlsx-extractor/src/parser.ts @@ -66,6 +66,7 @@ export async function parseBuffer( algorithm: 'default', }, headerSelectionEnabled: options?.headerSelectionEnabled ?? false, + skipEmptyLines: options?.skipEmptyLines ?? false, debug: options?.debug, }) if (!processedSheet) { @@ -89,6 +90,7 @@ type ConvertSheetArgs = { raw: boolean headerDetectionOptions: GetHeadersOptions headerSelectionEnabled: boolean + skipEmptyLines: boolean debug?: boolean } @@ -104,6 +106,7 @@ async function convertSheet({ raw, headerDetectionOptions, headerSelectionEnabled, + skipEmptyLines, debug, }: ConvertSheetArgs): Promise { let rows = XLSX.utils.sheet_to_json>(sheet, { @@ -111,7 +114,7 @@ async function convertSheet({ defval: null, rawNumbers, raw, - blankrows: true, + blankrows: headerSelectionEnabled || !skipEmptyLines, }) // return if there are no rows @@ -154,13 +157,20 @@ async function convertSheet({ const excelHeader = toExcelHeader(columnHeaders, columnKeys) const headers = prependNonUniqueHeaderColumns(excelHeader) + while ( + rows.length > 0 && + Object.values(rows[rows.length - 1]).every(isNullOrWhitespace) + ) { + rows.pop() + } + const data = rows.map((row) => mapValues( - mapKeys(row, (key) => headers[key]), + mapKeys(row, (key) => headers[key as keyof typeof headers]), (value) => ({ value }) ) ) - let metadata: { rowHeaders: number[] } | null + let metadata: { rowHeaders: number[] } | undefined = undefined if (headerSelectionEnabled) { metadata = { rowHeaders: [skip], @@ -173,3 +183,6 @@ async function convertSheet({ metadata, } } + +const isNullOrWhitespace = (value: any) => + value === null || (typeof value === 'string' && value.trim() === '')