From 57f446a9fe1e6b6e8e6628a2710bfe1e3e51e821 Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Wed, 20 Dec 2023 13:49:28 -0500 Subject: [PATCH 1/4] update @pinecone-database/pinecone version to support global control plane --- .env.example | 1 - README.md | 3 +- package-lock.json | 140 +++++++++++++++++++++++++++++-------- package.json | 2 +- src/embeddings.ts | 55 ++++++++------- src/index.ts | 56 ++++++++++----- src/recommend.ts | 57 ++++++++------- src/types.ts | 17 +++-- src/utils/chunkedUpsert.ts | 57 +++++++-------- src/utils/csvLoader.ts | 8 +-- src/utils/dataLoader.ts | 10 ++- src/utils/fileSplitter.ts | 14 ++-- src/utils/util.ts | 10 +-- 13 files changed, 274 insertions(+), 156 deletions(-) diff --git a/.env.example b/.env.example index 4cbd9e5..aefa63d 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,2 @@ PINECONE_API_KEY= -PINECONE_ENVIRONMENT= PINECONE_INDEX= \ No newline at end of file diff --git a/README.md b/README.md index 23617cd..48bc206 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,10 @@ Copy the template file: cp .env.example .env ``` -And fill in your API key and environment details: +And fill in your API key and index name: ```sh PINECONE_API_KEY= -PINECONE_ENVIRONMENT= PINECONE_INDEX=article-recommendations ``` diff --git a/package-lock.json b/package-lock.json index 8053c59..8feef59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^1.0.0", + "@pinecone-database/pinecone": "^1.1.2-spruceDev.20231219000826", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", @@ -63,6 +63,25 @@ "node": ">=6.9.0" } }, + "node_modules/@edge-runtime/primitives": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@edge-runtime/primitives/-/primitives-4.0.5.tgz", + "integrity": "sha512-t7QiN5d/KpXgCvIfSt6Nm9Hj3WVdNgc5CpOD73jasY+9EvTI7Ngdj5cXvjcHrPcmYWJZMySPgeEeoL/1N/Llag==", + "engines": { + "node": ">=16" + } + }, + "node_modules/@edge-runtime/types": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@edge-runtime/types/-/types-2.2.7.tgz", + "integrity": "sha512-9MTwGooICP7+ZsX9BTy6YCRzOr4tP6RFRymsc8CaKORfvuAHgLZUQaLwILfQ94tddufVXcBwq637VfEd3ZXbWA==", + "dependencies": { + "@edge-runtime/primitives": "4.0.5" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@esbuild-kit/cjs-loader": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@esbuild-kit/cjs-loader/-/cjs-loader-2.4.2.tgz", @@ -731,14 +750,17 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.0.0.tgz", - "integrity": "sha512-CtsfbK4qTDjnS56FVH64FEWNVnhwOyheBlLe3e9T6o9Gaxc00f/079JWUUiZ1lrkc3K/YkmlYYOXbdGyKP2z3A==", + "version": "1.1.2-spruceDev.20231219000826", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.2-spruceDev.20231219000826.tgz", + "integrity": "sha512-W7S3J20TyN6eABWn6/bmBggMsvImNvCEfJh885Qu7XSGtzirpPKDLyiqm9fqqBpM2O20SI6ngjfI0sYURR72HQ==", "dependencies": { - "@sinclair/typebox": "^0.28.15", - "@types/web": "^0.0.99", + "@edge-runtime/types": "^2.2.3", + "@sinclair/typebox": "^0.29.0", + "@types/node": "^18.11.17", "ajv": "^8.12.0", - "cross-fetch": "^3.1.5" + "cross-fetch": "^3.1.5", + "encoding": "^0.1.13", + "typescript": "^4.9.4" }, "engines": { "node": ">=14.0.0" @@ -764,6 +786,18 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, + "node_modules/@pinecone-database/pinecone/node_modules/typescript": { + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -819,9 +853,9 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "node_modules/@sinclair/typebox": { - "version": "0.28.20", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.28.20.tgz", - "integrity": "sha512-QCF3BGfacwD+3CKhGsMeixnwOmX4AWgm61nKkNdRStyLVu0mpVFYlDSY8gVBOOED1oSwzbJauIWl/+REj8K5+w==" + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==" }, "node_modules/@streamparser/json": { "version": "0.0.6", @@ -1122,11 +1156,6 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, - "node_modules/@types/web": { - "version": "0.0.99", - "resolved": "https://registry.npmjs.org/@types/web/-/web-0.0.99.tgz", - "integrity": "sha512-xMz3tOvtkZzc7RpQrDNiLe5sfMmP+fz8bOxHIZ/U8qXyvzDX4L4Ss1HCjor/O9DSelba+1iXK1VM7lruX28hiQ==" - }, "node_modules/@types/webgl-ext": { "version": "0.0.30", "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", @@ -2332,6 +2361,25 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -7002,6 +7050,19 @@ "regenerator-runtime": "^0.13.11" } }, + "@edge-runtime/primitives": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@edge-runtime/primitives/-/primitives-4.0.5.tgz", + "integrity": "sha512-t7QiN5d/KpXgCvIfSt6Nm9Hj3WVdNgc5CpOD73jasY+9EvTI7Ngdj5cXvjcHrPcmYWJZMySPgeEeoL/1N/Llag==" + }, + "@edge-runtime/types": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/@edge-runtime/types/-/types-2.2.7.tgz", + "integrity": "sha512-9MTwGooICP7+ZsX9BTy6YCRzOr4tP6RFRymsc8CaKORfvuAHgLZUQaLwILfQ94tddufVXcBwq637VfEd3ZXbWA==", + "requires": { + "@edge-runtime/primitives": "4.0.5" + } + }, "@esbuild-kit/cjs-loader": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@esbuild-kit/cjs-loader/-/cjs-loader-2.4.2.tgz", @@ -7395,14 +7456,17 @@ } }, "@pinecone-database/pinecone": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.0.0.tgz", - "integrity": "sha512-CtsfbK4qTDjnS56FVH64FEWNVnhwOyheBlLe3e9T6o9Gaxc00f/079JWUUiZ1lrkc3K/YkmlYYOXbdGyKP2z3A==", + "version": "1.1.2-spruceDev.20231219000826", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.2-spruceDev.20231219000826.tgz", + "integrity": "sha512-W7S3J20TyN6eABWn6/bmBggMsvImNvCEfJh885Qu7XSGtzirpPKDLyiqm9fqqBpM2O20SI6ngjfI0sYURR72HQ==", "requires": { - "@sinclair/typebox": "^0.28.15", - "@types/web": "^0.0.99", + "@edge-runtime/types": "^2.2.3", + "@sinclair/typebox": "^0.29.0", + "@types/node": "^18.11.17", "ajv": "^8.12.0", - "cross-fetch": "^3.1.5" + "cross-fetch": "^3.1.5", + "encoding": "^0.1.13", + "typescript": "^4.9.4" }, "dependencies": { "ajv": { @@ -7420,6 +7484,11 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "typescript": { + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==" } } }, @@ -7478,9 +7547,9 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@sinclair/typebox": { - "version": "0.28.20", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.28.20.tgz", - "integrity": "sha512-QCF3BGfacwD+3CKhGsMeixnwOmX4AWgm61nKkNdRStyLVu0mpVFYlDSY8gVBOOED1oSwzbJauIWl/+REj8K5+w==" + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==" }, "@streamparser/json": { "version": "0.0.6", @@ -7736,11 +7805,6 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, - "@types/web": { - "version": "0.0.99", - "resolved": "https://registry.npmjs.org/@types/web/-/web-0.0.99.tgz", - "integrity": "sha512-xMz3tOvtkZzc7RpQrDNiLe5sfMmP+fz8bOxHIZ/U8qXyvzDX4L4Ss1HCjor/O9DSelba+1iXK1VM7lruX28hiQ==" - }, "@types/webgl-ext": { "version": "0.0.30", "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", @@ -8592,6 +8656,24 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "requires": { + "iconv-lite": "^0.6.2" + }, + "dependencies": { + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", diff --git a/package.json b/package.json index bb71ac6..51a1506 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "author": "", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^1.0.0", + "@pinecone-database/pinecone": "^1.1.2-spruceDev.20231219000826", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", diff --git a/src/embeddings.ts b/src/embeddings.ts index 3f5fa39..05617fa 100644 --- a/src/embeddings.ts +++ b/src/embeddings.ts @@ -1,11 +1,13 @@ import { randomUUID } from "crypto"; import { Pipeline, pipeline, AutoConfig } from "@xenova/transformers"; -import type { PineconeRecord, RecordMetadata } from "@pinecone-database/pinecone"; -import type { Document } from 'langchain/document'; +import type { + PineconeRecord, + RecordMetadata, +} from "@pinecone-database/pinecone"; +import type { Document } from "langchain/document"; import { EmbeddingsParams, Embeddings } from "langchain/embeddings/base"; import { sliceIntoChunks } from "./utils/util.js"; - type DocumentOrString = Document | string; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -18,20 +20,22 @@ class Embedder { async init(modelName: string) { const config = await AutoConfig.from_pretrained(modelName); - this.pipe = await pipeline( - "embeddings", - modelName, - { - quantized: false, - config - } - ); + this.pipe = await pipeline("embeddings", modelName, { + quantized: false, + config, + }); } // Embeds a text and returns the embedding - async embed(text: string, metadata?: RecordMetadata): Promise { + async embed( + text: string, + metadata?: RecordMetadata + ): Promise { try { - const result = await this.pipe(text, { pooling: 'mean', normalize: true }); + const result = await this.pipe(text, { + pooling: "mean", + normalize: true, + }); const id = (metadata?.id as string) || randomUUID(); return { id, @@ -58,7 +62,10 @@ class Embedder { batch.map((documentOrString) => isString(documentOrString) ? this.embed(documentOrString) - : this.embed(documentOrString.pageContent, documentOrString.metadata) + : this.embed( + documentOrString.pageContent, + documentOrString.metadata + ) ) ); await onDoneBatch(embeddings); @@ -71,7 +78,10 @@ interface TransformersJSEmbeddingParams extends EmbeddingsParams { onEmbeddingDone?: (embeddings: PineconeRecord[]) => void; } -class TransformersJSEmbedding extends Embeddings implements TransformersJSEmbeddingParams { +class TransformersJSEmbedding + extends Embeddings + implements TransformersJSEmbeddingParams +{ modelName: string; pipe: Pipeline | null = null; @@ -82,26 +92,21 @@ class TransformersJSEmbedding extends Embeddings implements TransformersJSEmbedd } async embedDocuments(texts: string[]): Promise { - this.pipe = this.pipe || await pipeline( - "embeddings", - this.modelName - ); + this.pipe = this.pipe || (await pipeline("embeddings", this.modelName)); - const embeddings = await Promise.all(texts.map(async (text) => this.embedQuery(text))); + const embeddings = await Promise.all( + texts.map(async (text) => this.embedQuery(text)) + ); return embeddings; } async embedQuery(text: string): Promise { - this.pipe = this.pipe || await pipeline( - "embeddings", - this.modelName - ); + this.pipe = this.pipe || (await pipeline("embeddings", this.modelName)); const result = await this.pipe(text); return Array.from(result.data) as number[]; } } - const embedder = new Embedder(); export { embedder, TransformersJSEmbedding }; diff --git a/src/index.ts b/src/index.ts index 978c4f6..a91617e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,26 +1,34 @@ /* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable dot-notation */ import * as dotenv from "dotenv"; -import { Pinecone, type PineconeRecord } from '@pinecone-database/pinecone'; -import { getEnv } from "utils/util.ts"; +import { Pinecone, type PineconeRecord } from "@pinecone-database/pinecone"; +import { getEnv, validateEnvironmentVariables } from "utils/util.ts"; import cliProgress from "cli-progress"; -import { Document } from 'langchain/document'; +import { Document } from "langchain/document"; import * as dfd from "danfojs-node"; import { embedder } from "embeddings.ts"; import loadCSVFile from "utils/csvLoader.ts"; import splitFile from "utils/fileSplitter.ts"; -import { chunkedUpsert } from './utils/chunkedUpsert.ts'; import type { ArticleRecord } from "types.ts"; +import { chunkedUpsert } from "./utils/chunkedUpsert.ts"; dotenv.config(); +validateEnvironmentVariables(); -const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); +const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic +); // Index setup const indexName = getEnv("PINECONE_INDEX"); const pinecone = new Pinecone(); -async function getChunk(df: dfd.DataFrame, start: number, size: number): Promise { +async function getChunk( + df: dfd.DataFrame, + start: number, + size: number +): Promise { // eslint-disable-next-line no-return-await return await df.head(start + size).tail(size); } @@ -48,19 +56,27 @@ async function* processInChunks( } async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { - const chunkGenerator = processInChunks( + const chunkGenerator = processInChunks< + ArticleRecord, + "section" | "url" | "title" | "publication" | "author" | "article", + "article" + >( dataFrame, 100, - ['section', 'url', 'title', 'publication', 'author', 'article'], - 'article' + ["section", "url", "title", "publication", "author", "article"], + "article" ); const index = pinecone.index(indexName); for await (const documents of chunkGenerator) { - await embedder.embedBatch(documents, chunkSize, async (embeddings: PineconeRecord[]) => { - await chunkedUpsert(index, embeddings, "default"); - progressBar.increment(embeddings.length); - }); + await embedder.embedBatch( + documents, + chunkSize, + async (embeddings: PineconeRecord[]) => { + await chunkedUpsert(index, embeddings, "default"); + progressBar.increment(embeddings.length); + } + ); } } @@ -75,16 +91,22 @@ try { // Create the index if it doesn't already exist const indexList = await pinecone.listIndexes(); - if (indexList.indexOf({ name: indexName }) === -1) { - await pinecone.createIndex({ name: indexName, dimension: 384, waitUntilReady: true }) + if (!indexList.indexes?.some((index) => index.name === indexName)) { + await pinecone.createIndex({ + name: indexName, + dimension: 384, + spec: { serverless: { region: "us-west-2", cloud: "aws" } }, + waitUntilReady: true, + }); } progressBar.start(clean.shape[0], 0); await embedder.init("Xenova/all-MiniLM-L6-v2"); await embedAndUpsert(clean, 1); progressBar.stop(); - console.log(`Inserted ${progressBar.getTotal()} documents into index ${indexName}`); - + console.log( + `Inserted ${progressBar.getTotal()} documents into index ${indexName}` + ); } catch (error) { console.error(error); } diff --git a/src/recommend.ts b/src/recommend.ts index 78ab98a..7a6e6ae 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -1,11 +1,13 @@ /* eslint-disable import/no-extraneous-dependencies */ -import { getEnv, getQueryingCommandLineArguments } from "utils/util.ts"; +import { getEnv, getQueryingCommandLineArguments, validateEnvironmentVariables } from "utils/util.ts"; import { embedder } from "embeddings.ts"; -import { Table } from 'console-table-printer'; +import { Table } from "console-table-printer"; import { Pinecone } from "@pinecone-database/pinecone"; import type { ScoredPineconeRecord } from "@pinecone-database/pinecone"; import type { ArticleRecord } from "types.ts"; +validateEnvironmentVariables(); + const indexName = getEnv("PINECONE_INDEX"); const pinecone = new Pinecone(); @@ -13,41 +15,48 @@ const pinecone = new Pinecone(); try { const description = await pinecone.describeIndex(indexName); if (!description.status?.ready) { - throw `Index not ready, description was ${JSON.stringify(description)}` + throw new Error( + `Index not ready, description was ${JSON.stringify(description)}` + ); } } catch (e) { - console.log('An error occurred. Run "npm run index" to load data into the index before querying.') + console.log( + 'An error occurred. Run "npm run index" to load data into the index before querying.' + ); throw e; } -const index = pinecone.index(indexName).namespace('default'); +const index = pinecone.index(indexName).namespace("default"); await embedder.init("Xenova/all-MiniLM-L6-v2"); const { query, section } = getQueryingCommandLineArguments(); // We create a simulated user with an interest given a query and a specific section -const queryEmbedding = await embedder.embed(query) +const queryEmbedding = await embedder.embed(query); const queryResult = await index.query({ - vector: queryEmbedding.values, - includeMetadata: true, - includeValues: true, - filter: { - section: { "$eq": section } - }, - topK: 10 + vector: queryEmbedding.values, + includeMetadata: true, + includeValues: true, + filter: { + section: { $eq: section }, + }, + topK: 10, }); // We extract the vectors of the results -const userVectors = queryResult?.matches?.map((result: ScoredPineconeRecord) => result.values); +const userVectors = queryResult?.matches?.map( + (result: ScoredPineconeRecord) => result.values +); // A couple of functions to calculate mean vector -const mean = (arr: number[]): number => arr.reduce((a, b) => a + b, 0) / arr.length; +const mean = (arr: number[]): number => + arr.reduce((a, b) => a + b, 0) / arr.length; const meanVector = (vectors: number[][]): number[] => { const { length } = vectors[0]; return Array.from({ length }).map((_, i) => - mean(vectors.map(vec => vec[i])) + mean(vectors.map((vec) => vec[i])) ); }; @@ -57,10 +66,10 @@ const meanVec = meanVector(userVectors!); // We query the index with the mean vector to get recommendations for the user const recommendations = await index.query({ - vector: meanVec, - includeMetadata: true, - includeValues: true, - topK: 10 + vector: meanVec, + includeMetadata: true, + includeValues: true, + topK: 10, }); const userPreferences = new Table({ @@ -68,7 +77,7 @@ const userPreferences = new Table({ { name: "title", alignment: "left" }, { name: "author", alignment: "left" }, { name: "section", alignment: "left" }, - ] + ], }); const userRecommendations = new Table({ @@ -76,7 +85,7 @@ const userRecommendations = new Table({ { name: "title", alignment: "left" }, { name: "author", alignment: "left" }, { name: "section", alignment: "left" }, - ] + ], }); queryResult?.matches?.slice(0, 10).forEach((result: any) => { @@ -85,7 +94,7 @@ queryResult?.matches?.slice(0, 10).forEach((result: any) => { title, article: `${article.slice(0, 70)}...`, publication, - section + section, }); }); @@ -98,7 +107,7 @@ recommendations?.matches?.slice(0, 10).forEach((result: any) => { title, article: `${article.slice(0, 70)}...`, publication, - section + section, }); }); console.log("=========== Recommendations =========="); diff --git a/src/types.ts b/src/types.ts index 67857bc..a180623 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,10 +1,9 @@ export type ArticleRecord = { - index: number, - title: string; - article: string; - publication: string; - url: string; - author: string; - section: string; - } - \ No newline at end of file + index: number; + title: string; + article: string; + publication: string; + url: string; + author: string; + section: string; +}; diff --git a/src/utils/chunkedUpsert.ts b/src/utils/chunkedUpsert.ts index 5a43309..fc5c3db 100644 --- a/src/utils/chunkedUpsert.ts +++ b/src/utils/chunkedUpsert.ts @@ -1,34 +1,29 @@ -import type { Index, PineconeRecord } from '@pinecone-database/pinecone'; +import type { Index, PineconeRecord } from "@pinecone-database/pinecone"; +import { sliceIntoChunks } from "./util.ts"; -const sliceIntoChunks = (arr: T[], chunkSize: number) => { - return Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => - arr.slice(i * chunkSize, (i + 1) * chunkSize) +export const chunkedUpsert = async ( + index: Index, + vectors: Array, + namespace: string, + chunkSize = 10 +) => { + // Split the vectors into chunks + const chunks = sliceIntoChunks(vectors, chunkSize); + + try { + // Upsert each chunk of vectors into the index + await Promise.allSettled( + chunks.map(async (chunk) => { + try { + await index.namespace(namespace).upsert(chunk); + } catch (e) { + console.log("Error upserting chunk", e); + } + }) ); - }; -export const chunkedUpsert = async ( - index: Index, - vectors: Array, - namespace: string, - chunkSize = 10 - ) => { - // Split the vectors into chunks - const chunks = sliceIntoChunks(vectors, chunkSize); - - try { - // Upsert each chunk of vectors into the index - await Promise.allSettled( - chunks.map(async (chunk) => { - try { - await index.namespace(namespace).upsert(vectors); - } catch (e) { - console.log('Error upserting chunk', e); - } - }) - ); - - return true; - } catch (e) { - throw new Error(`Error upserting vectors into index: ${e}`); - } - }; \ No newline at end of file + return true; + } catch (e) { + throw new Error(`Error upserting vectors into index: ${e}`); + } +}; diff --git a/src/utils/csvLoader.ts b/src/utils/csvLoader.ts index fc6b099..4a38dd4 100644 --- a/src/utils/csvLoader.ts +++ b/src/utils/csvLoader.ts @@ -1,13 +1,13 @@ import fs from "fs/promises"; import * as dfd from "danfojs-node"; -async function loadCSVFile( - filePath: string -): Promise { +async function loadCSVFile(filePath: string): Promise { try { // Get csv file absolute path const csvAbsolutePath = await fs.realpath(filePath); - const df: dfd.DataFrame = (await dfd.readCSV(csvAbsolutePath)) as dfd.DataFrame; + const df: dfd.DataFrame = (await dfd.readCSV( + csvAbsolutePath + )) as dfd.DataFrame; // Create a readable stream from the CSV file return df; diff --git a/src/utils/dataLoader.ts b/src/utils/dataLoader.ts index d76c7a8..7161aa7 100644 --- a/src/utils/dataLoader.ts +++ b/src/utils/dataLoader.ts @@ -15,7 +15,10 @@ const jsonToCSV = async ( const { data } = await response.json(); const topLevelData = data; - const transforms = [unwind({ paths: [...unwindFieldsPaths] }), flatten({ objects: true, arrays: true })]; + const transforms = [ + unwind({ paths: [...unwindFieldsPaths] }), + flatten({ objects: true, arrays: true }), + ]; const json2csvParser = new Parser({ fields, transforms }); const csv = json2csvParser.parse(topLevelData); @@ -52,7 +55,10 @@ const dataFrameFromURL = async ( return df; }; -const dropDuplicates = (df: dfd.DataFrame, columnName: string): dfd.DataFrame => { +const dropDuplicates = ( + df: dfd.DataFrame, + columnName: string +): dfd.DataFrame => { // Get the column as a series const series = df[columnName]; diff --git a/src/utils/fileSplitter.ts b/src/utils/fileSplitter.ts index eeadc40..e3b5efc 100644 --- a/src/utils/fileSplitter.ts +++ b/src/utils/fileSplitter.ts @@ -1,12 +1,15 @@ /* eslint-disable no-plusplus */ -import fs from 'fs'; -import readline from 'readline'; +import fs from "fs"; +import readline from "readline"; -async function splitFile(filePath: string, numLinesPerPart: number): Promise { +async function splitFile( + filePath: string, + numLinesPerPart: number +): Promise { const fileStream = fs.createReadStream(filePath); const rl = readline.createInterface({ input: fileStream, - crlfDelay: Infinity + crlfDelay: Infinity, }); let partIndex = 1; @@ -33,5 +36,4 @@ async function splitFile(filePath: string, numLinesPerPart: number): Promise(arr: T[], chunkSize: number) => Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => - arr.slice(i * chunkSize, (i + 1) * chunkSize) -); +const sliceIntoChunks = (arr: T[], chunkSize: number) => + Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => + arr.slice(i * chunkSize, (i + 1) * chunkSize) + ); const getQueryingCommandLineArguments = () => { const argv = yargs(hideBin(process.argv)) @@ -41,12 +42,11 @@ export const getEnv = (key: string): string => { const validateEnvironmentVariables = () => { getEnv("PINECONE_API_KEY"); - getEnv("PINECONE_ENVIRONMENT"); getEnv("PINECONE_INDEX"); }; export { getQueryingCommandLineArguments, sliceIntoChunks, - validateEnvironmentVariables + validateEnvironmentVariables, }; From c39c545dd1178c66a6402090d6f32feefa3480da Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Wed, 20 Dec 2023 14:44:47 -0500 Subject: [PATCH 2/4] update README code snippets, add .DS_Store to gitignore, update splitFile in index.ts to use 100000 --- .gitignore | 3 ++- README.md | 75 ++++++++++++++++++++++++++++++++-------------------- src/index.ts | 2 +- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index ece1dfb..38088e4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ dist .turbo data/ all-the-news-2-1.zip -all-the-news-2-1.csv \ No newline at end of file +all-the-news-2-1.csv +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 48bc206..4622b56 100644 --- a/README.md +++ b/README.md @@ -96,19 +96,27 @@ Next we'll create a function that will generate the embeddings and upsert them i ```typescript async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { - const chunkGenerator = processInChunks( + const chunkGenerator = processInChunks< + ArticleRecord, + "section" | "url" | "title" | "publication" | "author" | "article", + "article" + >( dataFrame, 100, - ['section', 'url', 'title', 'publication', 'author', 'article'], - 'article' + ["section", "url", "title", "publication", "author", "article"], + "article" ); const index = pinecone.index(indexName); for await (const documents of chunkGenerator) { - await embedder.embedBatch(documents, chunkSize, async (embeddings: PineconeRecord[]) => { - await chunkedUpsert(index, embeddings, "default"); - progressBar.increment(embeddings.length); - }); + await embedder.embedBatch( + documents, + chunkSize, + async (embeddings: PineconeRecord[]) => { + await chunkedUpsert(index, embeddings, "default"); + progressBar.increment(embeddings.length); + } + ); } } ``` @@ -116,7 +124,7 @@ async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { We'll use the `splitFile` utility function to split the CSV file we downloaded into chunks of 100k parts each. For the purposes of this example, we'll only use the first 100k records. ```typescript -const fileParts = await splitFile("./data/all-the-news-2-1.csv", 1000000); +const fileParts = await splitFile("./data/all-the-news-2-1.csv", 100000); const firstFile = fileParts[0]; ``` @@ -132,8 +140,13 @@ Now we'll create the Pinecone index and kick off the embedding and upserting pro ```typescript // Create the index if it doesn't already exist const indexList = await pinecone.listIndexes(); -if (indexList.indexOf({ name: indexName }) === -1) { - await pinecone.createIndex({ name: indexName, dimension: 384, waitUntilReady: true }) +if (!indexList.indexes?.some((index) => index.name === indexName)) { + await pinecone.createIndex({ + name: indexName, + dimension: 384, + spec: { serverless: { region: "us-west-2", cloud: "aws" } }, + waitUntilReady: true, + }); } progressBar.start(clean.shape[0], 0); @@ -154,10 +167,14 @@ const pinecone = new Pinecone(); try { const description = await pinecone.describeIndex(indexName); if (!description.status?.ready) { - throw `Index not ready, description was ${JSON.stringify(description)}` + throw new Error( + `Index not ready, description was ${JSON.stringify(description)}` + ); } } catch (e) { - console.log('An error occurred. Run "npm run index" to load data into the index before querying.') + console.log( + 'An error occurred. Run "npm run index" to load data into the index before querying.' + ); throw e; } @@ -174,7 +191,7 @@ const queryResult = await index.query({ includeMetadata: true, includeValues: true, filter: { - section: { "$eq": section } + section: { $eq: section } }, topK: 10 }); @@ -184,15 +201,18 @@ We'll calculate the **mean** vector given the results of the query. The mean vec ```typescript // We extract the vectors of the results -const userVectors = queryResult?.matches?.map((result: ScoredPineconeRecord) => result.values); +const userVectors = queryResult?.matches?.map( + (result: ScoredPineconeRecord) => result.values +); // A couple of functions to calculate mean vector -const mean = (arr: number[]): number => arr.reduce((a, b) => a + b, 0) / arr.length; +const mean = (arr: number[]): number => + arr.reduce((a, b) => a + b, 0) / arr.length; const meanVector = (vectors: number[][]): number[] => { const { length } = vectors[0]; return Array.from({ length }).map((_, i) => - mean(vectors.map(vec => vec[i])) + mean(vectors.map((vec) => vec[i])) ); }; @@ -204,14 +224,11 @@ const meanVec = meanVector(userVectors!); To resolve the recommendations, we'll query the index with the mean vector and filter out the articles that the user has already read. ```typescript -const recommendations = await pineconeIndex.query({ - queryRequest: { - vector: meanVec, - includeMetadata: true, - includeValues: true, - namespace: "default", - topK: 10, - }, +const recommendations = await index.query({ + vector: meanVec, + includeMetadata: true, + includeValues: true, + topK: 10, }); ``` @@ -235,10 +252,11 @@ const userRecommendations = new Table({ }); queryResult?.matches?.slice(0, 10).forEach((result: any) => { - const { title, author, section } = result.metadata; + const { title, article, publication, section } = result.metadata; userPreferences.addRow({ title, - author, + article: `${article.slice(0, 70)}...`, + publication, section, }); }); @@ -247,10 +265,11 @@ console.log("========== User Preferences =========="); userPreferences.printTable(); recommendations?.matches?.slice(0, 10).forEach((result: any) => { - const { title, author, section } = result.metadata; + const { title, article, publication, section } = result.metadata; userRecommendations.addRow({ title, - author, + article: `${article.slice(0, 70)}...`, + publication, section, }); }); diff --git a/src/index.ts b/src/index.ts index a91617e..ab5d138 100644 --- a/src/index.ts +++ b/src/index.ts @@ -81,7 +81,7 @@ async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { } try { - const fileParts = await splitFile("./data/all-the-news-2-1.csv", 500000); + const fileParts = await splitFile("./data/all-the-news-2-1.csv", 100000); const firstFile = fileParts[0]; // For this example, we will use the first file part to create the index From ec063c03cb2f6dba2420c8fae1527a353a1f579d Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Mon, 15 Jan 2024 18:26:07 -0500 Subject: [PATCH 3/4] add cloud and region to environment variables --- .env.example | 4 ++- README.md | 28 ++++++++++-------- package-lock.json | 73 ++++++----------------------------------------- package.json | 2 +- src/index.ts | 10 +++++-- src/recommend.ts | 6 +++- src/utils/util.ts | 2 ++ 7 files changed, 44 insertions(+), 81 deletions(-) diff --git a/.env.example b/.env.example index aefa63d..d0f9450 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,4 @@ PINECONE_API_KEY= -PINECONE_INDEX= \ No newline at end of file +PINECONE_INDEX= +PINECONE_CLOUD="aws" +PINECONE_REGION="us-west-2" \ No newline at end of file diff --git a/README.md b/README.md index 4622b56..e8d593a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ npm install ## Required configuration -In order to run this example, you have to supply the Pinecone credentials needed to interact with the Pinecone API. You can find these credentials in the [Pinecone web console](https://app.pinecone.io) under **API Keys**. This project uses `dotenv` to easily load values from the `.env` file into the environment when executing. +In order to run this example, you have to supply the Pinecone credentials needed to interact with the Pinecone API. You can find these credentials in the [Pinecone web console](https://app.pinecone.io) under **API Keys**. This project uses `dotenv` to easily load values from the `.env` file into the environment when executing. Copy the template file: @@ -22,11 +22,15 @@ And fill in your API key and index name: ```sh PINECONE_API_KEY= -PINECONE_INDEX=article-recommendations +PINECONE_INDEX="article-recommendations" +PINECONE_CLOUD="aws" +PINECONE_REGION="us-west-2" ``` `PINECONE_INDEX` is the name of the index where this demo will store and query embeddings. You can change `PINECONE_INDEX` to any name you like, but make sure the name not going to collide with any indexes you are already using. +`PINECONE_CLOUD` and `PINECONE_REGION` define where the index should be deployed. Currently, this is the only available cloud and region combination (`aws` and `us-west-2`), so it's recommended to leave them defaulted. + ## Data preparation Next, we will prepare data for the Pinecone vector index, and insert it in batches. @@ -144,7 +148,7 @@ if (!indexList.indexes?.some((index) => index.name === indexName)) { await pinecone.createIndex({ name: indexName, dimension: 384, - spec: { serverless: { region: "us-west-2", cloud: "aws" } }, + spec: { serverless: { region: indexRegion, cloud: indexCloud } }, waitUntilReady: true, }); } @@ -178,22 +182,22 @@ try { throw e; } -const index = pinecone.index(indexName).namespace('default'); +const index = pinecone.index(indexName).namespace("default"); await embedder.init("Xenova/all-MiniLM-L6-v2"); const { query, section } = getQueryingCommandLineArguments(); // We create a simulated user with an interest given a query and a specific section -const queryEmbedding = await embedder.embed(query) +const queryEmbedding = await embedder.embed(query); const queryResult = await index.query({ - vector: queryEmbedding.values, - includeMetadata: true, - includeValues: true, - filter: { - section: { $eq: section } - }, - topK: 10 + vector: queryEmbedding.values, + includeMetadata: true, + includeValues: true, + filter: { + section: { $eq: section }, + }, + topK: 10, }); ``` diff --git a/package-lock.json b/package-lock.json index 8feef59..23a7628 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^1.1.2-spruceDev.20231219000826", + "@pinecone-database/pinecone": "^1.1.3-spruceDev.20240115214739", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", @@ -63,25 +63,6 @@ "node": ">=6.9.0" } }, - "node_modules/@edge-runtime/primitives": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@edge-runtime/primitives/-/primitives-4.0.5.tgz", - "integrity": "sha512-t7QiN5d/KpXgCvIfSt6Nm9Hj3WVdNgc5CpOD73jasY+9EvTI7Ngdj5cXvjcHrPcmYWJZMySPgeEeoL/1N/Llag==", - "engines": { - "node": ">=16" - } - }, - "node_modules/@edge-runtime/types": { - "version": "2.2.7", - "resolved": "https://registry.npmjs.org/@edge-runtime/types/-/types-2.2.7.tgz", - "integrity": "sha512-9MTwGooICP7+ZsX9BTy6YCRzOr4tP6RFRymsc8CaKORfvuAHgLZUQaLwILfQ94tddufVXcBwq637VfEd3ZXbWA==", - "dependencies": { - "@edge-runtime/primitives": "4.0.5" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/@esbuild-kit/cjs-loader": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@esbuild-kit/cjs-loader/-/cjs-loader-2.4.2.tgz", @@ -750,17 +731,14 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "1.1.2-spruceDev.20231219000826", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.2-spruceDev.20231219000826.tgz", - "integrity": "sha512-W7S3J20TyN6eABWn6/bmBggMsvImNvCEfJh885Qu7XSGtzirpPKDLyiqm9fqqBpM2O20SI6ngjfI0sYURR72HQ==", + "version": "1.1.3-spruceDev.20240115214739", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.3-spruceDev.20240115214739.tgz", + "integrity": "sha512-R3VCHgf5Qu2qbfoGGqTTWdVU6Y1Fo66sQD6lrv001hoaEHSIDn96WrQxuaOzgekCkT5xMZsat2p18gtTvBcigw==", "dependencies": { - "@edge-runtime/types": "^2.2.3", "@sinclair/typebox": "^0.29.0", - "@types/node": "^18.11.17", "ajv": "^8.12.0", "cross-fetch": "^3.1.5", - "encoding": "^0.1.13", - "typescript": "^4.9.4" + "encoding": "^0.1.13" }, "engines": { "node": ">=14.0.0" @@ -786,18 +764,6 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, - "node_modules/@pinecone-database/pinecone/node_modules/typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -7050,19 +7016,6 @@ "regenerator-runtime": "^0.13.11" } }, - "@edge-runtime/primitives": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@edge-runtime/primitives/-/primitives-4.0.5.tgz", - "integrity": "sha512-t7QiN5d/KpXgCvIfSt6Nm9Hj3WVdNgc5CpOD73jasY+9EvTI7Ngdj5cXvjcHrPcmYWJZMySPgeEeoL/1N/Llag==" - }, - "@edge-runtime/types": { - "version": "2.2.7", - "resolved": "https://registry.npmjs.org/@edge-runtime/types/-/types-2.2.7.tgz", - "integrity": "sha512-9MTwGooICP7+ZsX9BTy6YCRzOr4tP6RFRymsc8CaKORfvuAHgLZUQaLwILfQ94tddufVXcBwq637VfEd3ZXbWA==", - "requires": { - "@edge-runtime/primitives": "4.0.5" - } - }, "@esbuild-kit/cjs-loader": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@esbuild-kit/cjs-loader/-/cjs-loader-2.4.2.tgz", @@ -7456,17 +7409,14 @@ } }, "@pinecone-database/pinecone": { - "version": "1.1.2-spruceDev.20231219000826", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.2-spruceDev.20231219000826.tgz", - "integrity": "sha512-W7S3J20TyN6eABWn6/bmBggMsvImNvCEfJh885Qu7XSGtzirpPKDLyiqm9fqqBpM2O20SI6ngjfI0sYURR72HQ==", + "version": "1.1.3-spruceDev.20240115214739", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.3-spruceDev.20240115214739.tgz", + "integrity": "sha512-R3VCHgf5Qu2qbfoGGqTTWdVU6Y1Fo66sQD6lrv001hoaEHSIDn96WrQxuaOzgekCkT5xMZsat2p18gtTvBcigw==", "requires": { - "@edge-runtime/types": "^2.2.3", "@sinclair/typebox": "^0.29.0", - "@types/node": "^18.11.17", "ajv": "^8.12.0", "cross-fetch": "^3.1.5", - "encoding": "^0.1.13", - "typescript": "^4.9.4" + "encoding": "^0.1.13" }, "dependencies": { "ajv": { @@ -7484,11 +7434,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, - "typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==" } } }, diff --git a/package.json b/package.json index 51a1506..e08dc4a 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "author": "", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^1.1.2-spruceDev.20231219000826", + "@pinecone-database/pinecone": "^1.1.3-spruceDev.20240115214739", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", diff --git a/src/index.ts b/src/index.ts index ab5d138..b9155a6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,11 @@ /* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable dot-notation */ import * as dotenv from "dotenv"; -import { Pinecone, type PineconeRecord } from "@pinecone-database/pinecone"; +import { + Pinecone, + type PineconeRecord, + type ServerlessSpecCloudEnum, +} from "@pinecone-database/pinecone"; import { getEnv, validateEnvironmentVariables } from "utils/util.ts"; import cliProgress from "cli-progress"; import { Document } from "langchain/document"; @@ -22,6 +26,8 @@ const progressBar = new cliProgress.SingleBar( // Index setup const indexName = getEnv("PINECONE_INDEX"); +const indexCloud = getEnv("PINECONE_CLOUD") as ServerlessSpecCloudEnum; +const indexRegion = getEnv("PINECONE_REGION"); const pinecone = new Pinecone(); async function getChunk( @@ -95,7 +101,7 @@ try { await pinecone.createIndex({ name: indexName, dimension: 384, - spec: { serverless: { region: "us-west-2", cloud: "aws" } }, + spec: { serverless: { region: indexRegion, cloud: indexCloud } }, waitUntilReady: true, }); } diff --git a/src/recommend.ts b/src/recommend.ts index 7a6e6ae..c936960 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -1,5 +1,9 @@ /* eslint-disable import/no-extraneous-dependencies */ -import { getEnv, getQueryingCommandLineArguments, validateEnvironmentVariables } from "utils/util.ts"; +import { + getEnv, + getQueryingCommandLineArguments, + validateEnvironmentVariables, +} from "utils/util.ts"; import { embedder } from "embeddings.ts"; import { Table } from "console-table-printer"; import { Pinecone } from "@pinecone-database/pinecone"; diff --git a/src/utils/util.ts b/src/utils/util.ts index d679098..e57a872 100644 --- a/src/utils/util.ts +++ b/src/utils/util.ts @@ -43,6 +43,8 @@ export const getEnv = (key: string): string => { const validateEnvironmentVariables = () => { getEnv("PINECONE_API_KEY"); getEnv("PINECONE_INDEX"); + getEnv("PINECONE_CLOUD"); + getEnv("PINECONE_REGION"); }; export { From c33bf22437db6a22496756b3b9e718d55c5df9a5 Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Tue, 16 Jan 2024 08:31:11 -0500 Subject: [PATCH 4/4] bump pinecone-database to v2 --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 23a7628..f71ee8c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^1.1.3-spruceDev.20240115214739", + "@pinecone-database/pinecone": "^2.0.0", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", @@ -731,9 +731,9 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "1.1.3-spruceDev.20240115214739", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.3-spruceDev.20240115214739.tgz", - "integrity": "sha512-R3VCHgf5Qu2qbfoGGqTTWdVU6Y1Fo66sQD6lrv001hoaEHSIDn96WrQxuaOzgekCkT5xMZsat2p18gtTvBcigw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-2.0.0.tgz", + "integrity": "sha512-deDcyJNqqgbww+nt3asqkdsTTASYOdVwfqlRmumKKT+v9760+tupRoUVPWqZ1+3LoVyvC+03DfhulQQHz6rC3Q==", "dependencies": { "@sinclair/typebox": "^0.29.0", "ajv": "^8.12.0", @@ -7409,9 +7409,9 @@ } }, "@pinecone-database/pinecone": { - "version": "1.1.3-spruceDev.20240115214739", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.1.3-spruceDev.20240115214739.tgz", - "integrity": "sha512-R3VCHgf5Qu2qbfoGGqTTWdVU6Y1Fo66sQD6lrv001hoaEHSIDn96WrQxuaOzgekCkT5xMZsat2p18gtTvBcigw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-2.0.0.tgz", + "integrity": "sha512-deDcyJNqqgbww+nt3asqkdsTTASYOdVwfqlRmumKKT+v9760+tupRoUVPWqZ1+3LoVyvC+03DfhulQQHz6rC3Q==", "requires": { "@sinclair/typebox": "^0.29.0", "ajv": "^8.12.0", diff --git a/package.json b/package.json index e08dc4a..94f52db 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "author": "", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^1.1.3-spruceDev.20240115214739", + "@pinecone-database/pinecone": "^2.0.0", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0",