From 5e0ea11b84e9eb83e9d26bcc3b8f4203d04da6a3 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 7 Sep 2023 06:09:17 -0400 Subject: [PATCH 1/4] Updates --- README.md | 9 +++-- package-lock.json | 78 ++++++++++++++++++++++++++++++++++---- package.json | 2 +- src/embeddings.ts | 13 +++---- src/index.ts | 25 ++++++------ src/recommend.ts | 32 ++++++++-------- src/utils/chunkedUpsert.ts | 34 +++++++++++++++++ src/utils/pinecone.ts | 24 ------------ 8 files changed, 147 insertions(+), 70 deletions(-) create mode 100644 src/utils/chunkedUpsert.ts delete mode 100644 src/utils/pinecone.ts diff --git a/README.md b/README.md index 97c9530..439d3a8 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,13 @@ npm install ## Importing the Libraries -We'll start by importing the necessary libraries. We'll be using the `@pinecone-database/pinecone` library to interact with Pinecone. We'll also be using the `danfojs-node` library to load the data into an easy to manipulate dataframe. We'll use the `Document` type from Langchain to keep the data structure consistent across the indexing process and retrieval agent. +We'll start by importing the necessary libraries. -We'll be using the `Embedder` class found in `embeddings.ts` to embed the data We'll also be using the `cli-progress` library to display a progress bar. +- We'll use the `@pinecone-database/pinecone` library to interact with Pinecone. +- We'll use the `danfojs-node` library to load the data into an easy to manipulate dataframe. +- We'll use the `Document` type from Langchain to keep the data structure consistent across the indexing process and retrieval agent. +- We'll use the `Embedder` class found in `embeddings.ts` to embed the data. +- We'll use the `cli-progress` library to display a progress bar. To load the dataset used in the example, we'll be using a utility called `squadLoader.js`. @@ -24,7 +28,6 @@ import cliProgress from "cli-progress"; import { Document } from "langchain/document"; import * as dfd from "danfojs-node"; import { embedder } from "embeddings.ts"; -import { SquadRecord, loadSquad } from "./utils/squadLoader.js"; ``` ## Upload articles diff --git a/package-lock.json b/package-lock.json index 25fbe0f..929b0eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^0.1.6", + "@pinecone-database/pinecone": "^0.1.6-dev.20230907100643", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", @@ -731,16 +731,39 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.6.tgz", - "integrity": "sha512-tCnVc28udecthhgSBTdcMhYEW+xsR++AdZasp+ZE/AvUD1hOR2IR3edjk9m0sDxZyvXbno2KeqUbLIOZr7sCTw==", + "version": "0.1.6-dev.20230907100643", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.6-dev.20230907100643.tgz", + "integrity": "sha512-EQvF3xivlSyx7y2XKZ3YWs8iPFTM4ydjSO1paNqnf56yh/1bBShR8vPcbdA/FaCM+KZgStTUFz1pLJHZFz7aQw==", "dependencies": { + "@sinclair/typebox": "^0.28.15", + "@types/web": "^0.0.99", + "ajv": "^8.12.0", "cross-fetch": "^3.1.5" }, "engines": { "node": ">=14.0.0" } }, + "node_modules/@pinecone-database/pinecone/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@pinecone-database/pinecone/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -795,6 +818,11 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@sinclair/typebox": { + "version": "0.28.20", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.28.20.tgz", + "integrity": "sha512-QCF3BGfacwD+3CKhGsMeixnwOmX4AWgm61nKkNdRStyLVu0mpVFYlDSY8gVBOOED1oSwzbJauIWl/+REj8K5+w==" + }, "node_modules/@streamparser/json": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@streamparser/json/-/json-0.0.6.tgz", @@ -1094,6 +1122,11 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "node_modules/@types/web": { + "version": "0.0.99", + "resolved": "https://registry.npmjs.org/@types/web/-/web-0.0.99.tgz", + "integrity": "sha512-xMz3tOvtkZzc7RpQrDNiLe5sfMmP+fz8bOxHIZ/U8qXyvzDX4L4Ss1HCjor/O9DSelba+1iXK1VM7lruX28hiQ==" + }, "node_modules/@types/webgl-ext": { "version": "0.0.30", "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", @@ -7362,11 +7395,32 @@ } }, "@pinecone-database/pinecone": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.6.tgz", - "integrity": "sha512-tCnVc28udecthhgSBTdcMhYEW+xsR++AdZasp+ZE/AvUD1hOR2IR3edjk9m0sDxZyvXbno2KeqUbLIOZr7sCTw==", + "version": "0.1.6-dev.20230907100643", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.6-dev.20230907100643.tgz", + "integrity": "sha512-EQvF3xivlSyx7y2XKZ3YWs8iPFTM4ydjSO1paNqnf56yh/1bBShR8vPcbdA/FaCM+KZgStTUFz1pLJHZFz7aQw==", "requires": { + "@sinclair/typebox": "^0.28.15", + "@types/web": "^0.0.99", + "ajv": "^8.12.0", "cross-fetch": "^3.1.5" + }, + "dependencies": { + "ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + } } }, "@protobufjs/aspromise": { @@ -7423,6 +7477,11 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "@sinclair/typebox": { + "version": "0.28.20", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.28.20.tgz", + "integrity": "sha512-QCF3BGfacwD+3CKhGsMeixnwOmX4AWgm61nKkNdRStyLVu0mpVFYlDSY8gVBOOED1oSwzbJauIWl/+REj8K5+w==" + }, "@streamparser/json": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@streamparser/json/-/json-0.0.6.tgz", @@ -7677,6 +7736,11 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "@types/web": { + "version": "0.0.99", + "resolved": "https://registry.npmjs.org/@types/web/-/web-0.0.99.tgz", + "integrity": "sha512-xMz3tOvtkZzc7RpQrDNiLe5sfMmP+fz8bOxHIZ/U8qXyvzDX4L4Ss1HCjor/O9DSelba+1iXK1VM7lruX28hiQ==" + }, "@types/webgl-ext": { "version": "0.0.30", "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", diff --git a/package.json b/package.json index 70c5a52..7fa28f9 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "author": "", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^0.1.6", + "@pinecone-database/pinecone": "^0.1.6-dev.20230907100643", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", diff --git a/src/embeddings.ts b/src/embeddings.ts index d358360..3f5fa39 100644 --- a/src/embeddings.ts +++ b/src/embeddings.ts @@ -1,7 +1,7 @@ import { randomUUID } from "crypto"; import { Pipeline, pipeline, AutoConfig } from "@xenova/transformers"; -import { Vector } from "@pinecone-database/pinecone"; -import { Document } from 'langchain/document'; +import type { PineconeRecord, RecordMetadata } from "@pinecone-database/pinecone"; +import type { Document } from 'langchain/document'; import { EmbeddingsParams, Embeddings } from "langchain/embeddings/base"; import { sliceIntoChunks } from "./utils/util.js"; @@ -24,13 +24,12 @@ class Embedder { { quantized: false, config - }, - + } ); } // Embeds a text and returns the embedding - async embed(text: string, metadata?: Record): Promise { + async embed(text: string, metadata?: RecordMetadata): Promise { try { const result = await this.pipe(text, { pooling: 'mean', normalize: true }); const id = (metadata?.id as string) || randomUUID(); @@ -51,7 +50,7 @@ class Embedder { async embedBatch( documents: DocumentOrString[], batchSize: number, - onDoneBatch: (embeddings: Vector[]) => void + onDoneBatch: (embeddings: PineconeRecord[]) => void ) { const batches = sliceIntoChunks(documents, batchSize); for (const batch of batches) { @@ -69,7 +68,7 @@ class Embedder { interface TransformersJSEmbeddingParams extends EmbeddingsParams { modelName: string; - onEmbeddingDone?: (embeddings: Vector[]) => void; + onEmbeddingDone?: (embeddings: PineconeRecord[]) => void; } class TransformersJSEmbedding extends Embeddings implements TransformersJSEmbeddingParams { diff --git a/src/index.ts b/src/index.ts index 1d98f40..dd78e59 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,17 +1,17 @@ /* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable dot-notation */ import * as dotenv from "dotenv"; -import { Vector, utils } from '@pinecone-database/pinecone'; +import { Pinecone, type PineconeRecord } from '@pinecone-database/pinecone'; import { getEnv } from "utils/util.ts"; -import { getPineconeClient } from "utils/pinecone.ts"; import cliProgress from "cli-progress"; import { Document } from 'langchain/document'; import * as dfd from "danfojs-node"; import { embedder } from "embeddings.ts"; import loadCSVFile from "utils/csvLoader.ts"; import splitFile from "utils/fileSplitter.ts"; +import { chunkedUpsert } from './utils/chunkedUpsert.ts'; -interface ArticleRecord { +type ArticleRecord = { index: number, title: string; article: string; @@ -21,24 +21,19 @@ interface ArticleRecord { section: string; } - - dotenv.config(); -const { createIndexIfNotExists, chunkedUpsert } = utils; const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); // Index setup const indexName = getEnv("PINECONE_INDEX"); -const pineconeClient = await getPineconeClient(); - +const pinecone = new Pinecone(); async function getChunk(df: dfd.DataFrame, start: number, size: number): Promise { // eslint-disable-next-line no-return-await return await df.head(start + size).tail(size); } - async function* processInChunks( dataFrame: dfd.DataFrame, chunkSize: number, @@ -68,10 +63,10 @@ async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { ['section', 'url', 'title', 'publication', 'author', 'article'], 'article' ); - const index = pineconeClient.Index(indexName); + const index = pinecone.index(indexName); for await (const documents of chunkGenerator) { - await embedder.embedBatch(documents, chunkSize, async (embeddings: Vector[]) => { + await embedder.embedBatch(documents, chunkSize, async (embeddings: PineconeRecord[]) => { await chunkedUpsert(index, embeddings, "default"); progressBar.increment(embeddings.length); }); @@ -86,7 +81,13 @@ try { const data = await loadCSVFile(firstFile); const clean = data.dropNa() as dfd.DataFrame; clean.head().print(); - await createIndexIfNotExists(pineconeClient, indexName, 384); + + // Create the index if it doesn't already exist + const indexList = await pinecone.listIndexes(); + if (indexList.indexOf({ name: indexName }) === -1) { + await pinecone.createIndex({ name: indexName, dimension: 384, waitUntilReady: true }) + } + progressBar.start(clean.shape[0], 0); await embedder.init("Xenova/all-MiniLM-L6-v2"); await embedAndUpsert(clean, 1); diff --git a/src/recommend.ts b/src/recommend.ts index c0c0ef7..b3ac72f 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -1,37 +1,41 @@ /* eslint-disable import/no-extraneous-dependencies */ import { getEnv, getQueryingCommandLineArguments } from "utils/util.ts"; -import { getPineconeClient } from "utils/pinecone.ts"; import { embedder } from "embeddings.ts"; import { Table } from 'console-table-printer'; -import { ScoredVector } from "@pinecone-database/pinecone"; - +import { Pinecone } from "@pinecone-database/pinecone"; +import type { ScoredPineconeRecord } from "@pinecone-database/pinecone"; const indexName = getEnv("PINECONE_INDEX"); -const pineconeClient = await getPineconeClient(); -const pineconeIndex = pineconeClient.Index(indexName); +const pinecone = new Pinecone(); + +type DocumentMetadata = { + title?: string, + author?: string, + section?: string, + publication?: string, +} + +const index = pinecone.index(indexName); await embedder.init("Xenova/all-MiniLM-L6-v2"); const { query, section } = getQueryingCommandLineArguments(); -// We create a simulated a user with an interest given a query and a specific section +// We create a simulated user with an interest given a query and a specific section const queryEmbedding = await embedder.embed(query); -const queryResult = await pineconeIndex.query({ - queryRequest: { +const queryResult = await index.query({ vector: queryEmbedding.values, includeMetadata: true, includeValues: true, - namespace: "default", filter: { section: { "$eq": section } }, topK: 10 - } }); // We extract the vectors of the results -const userVectors = queryResult?.matches?.map((result: ScoredVector) => result.values as number[]); +const userVectors = queryResult?.matches?.map((result: ScoredPineconeRecord) => result.values); // A couple of functions to calculate mean vector const mean = (arr: number[]): number => arr.reduce((a, b) => a + b, 0) / arr.length; @@ -48,17 +52,13 @@ const meanVector = (vectors: number[][]): number[] => { const meanVec = meanVector(userVectors!); // We query the index with the mean vector to get recommendations for the user -const recommendations = await pineconeIndex.query({ - queryRequest: { +const recommendations = await index.query({ vector: meanVec, includeMetadata: true, includeValues: true, - namespace: "default", topK: 10 - } }); - const userPreferences = new Table({ columns: [ { name: "title", alignment: "left" }, diff --git a/src/utils/chunkedUpsert.ts b/src/utils/chunkedUpsert.ts new file mode 100644 index 0000000..5a43309 --- /dev/null +++ b/src/utils/chunkedUpsert.ts @@ -0,0 +1,34 @@ +import type { Index, PineconeRecord } from '@pinecone-database/pinecone'; + +const sliceIntoChunks = (arr: T[], chunkSize: number) => { + return Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => + arr.slice(i * chunkSize, (i + 1) * chunkSize) + ); + }; + +export const chunkedUpsert = async ( + index: Index, + vectors: Array, + namespace: string, + chunkSize = 10 + ) => { + // Split the vectors into chunks + const chunks = sliceIntoChunks(vectors, chunkSize); + + try { + // Upsert each chunk of vectors into the index + await Promise.allSettled( + chunks.map(async (chunk) => { + try { + await index.namespace(namespace).upsert(vectors); + } catch (e) { + console.log('Error upserting chunk', e); + } + }) + ); + + return true; + } catch (e) { + throw new Error(`Error upserting vectors into index: ${e}`); + } + }; \ No newline at end of file diff --git a/src/utils/pinecone.ts b/src/utils/pinecone.ts deleted file mode 100644 index 3b8e1bb..0000000 --- a/src/utils/pinecone.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { PineconeClient } from "@pinecone-database/pinecone"; -import { config } from "dotenv"; -import { getEnv, validateEnvironmentVariables } from "./util.js"; - -config(); - -let pineconeClient: PineconeClient | null = null; - -// Returns a PineconeClient instance -export const getPineconeClient: () => Promise = async () => { - validateEnvironmentVariables(); - - if (pineconeClient) { - return pineconeClient; - } - pineconeClient = new PineconeClient(); - - await pineconeClient.init({ - apiKey: getEnv("PINECONE_API_KEY"), - environment: getEnv("PINECONE_ENVIRONMENT"), - }); - - return pineconeClient; -}; \ No newline at end of file From d53b8ee88838ccdb2531a28a3a970b503f917278 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 7 Sep 2023 07:33:47 -0400 Subject: [PATCH 2/4] Updates --- .gitignore | 4 ++- README.md | 72 +++++++++++++++++++++++------------------------- src/index.ts | 11 +------- src/recommend.ts | 22 +++++++++------ src/types.ts | 10 +++++++ 5 files changed, 62 insertions(+), 57 deletions(-) create mode 100644 src/types.ts diff --git a/.gitignore b/.gitignore index 927a776..ece1dfb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ node_modules .env dist .turbo -data/ \ No newline at end of file +data/ +all-the-news-2-1.zip +all-the-news-2-1.csv \ No newline at end of file diff --git a/README.md b/README.md index 439d3a8..4642c49 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The goal is to create a recommendation engine that retrieves the best article re npm install ``` -## Importing the Libraries +## Dependencies We'll start by importing the necessary libraries. @@ -20,16 +20,6 @@ We'll start by importing the necessary libraries. To load the dataset used in the example, we'll be using a utility called `squadLoader.js`. -```typescript -import { Vector, utils } from "@pinecone-database/pinecone"; -import { getEnv } from "utils/util.ts"; -import { getPineconeClient } from "utils/pinecone.ts"; -import cliProgress from "cli-progress"; -import { Document } from "langchain/document"; -import * as dfd from "danfojs-node"; -import { embedder } from "embeddings.ts"; -``` - ## Upload articles Next, we will prepare data for the Pinecone vector index, and insert it in batches. @@ -97,18 +87,19 @@ Next we'll create a function that will generate the embeddings and upsert them i ```typescript async function embedAndUpsert(dataFrame: dfd.DataFrame, chunkSize: number) { - const chunkGenerator = processInChunks(dataFrame, chunkSize); - const index = pineconeClient.Index(indexName); + const chunkGenerator = processInChunks( + dataFrame, + 100, + ['section', 'url', 'title', 'publication', 'author', 'article'], + 'article' + ); + const index = pinecone.index(indexName); for await (const documents of chunkGenerator) { - await embedder.embedBatch( - documents, - chunkSize, - async (embeddings: Vector[]) => { - await chunkedUpsert(index, embeddings, "default"); - progressBar.increment(embeddings.length); - } - ); + await embedder.embedBatch(documents, chunkSize, async (embeddings: PineconeRecord[]) => { + await chunkedUpsert(index, embeddings, "default"); + progressBar.increment(embeddings.length); + }); } } ``` @@ -143,26 +134,35 @@ We will query the index for the an imagined user. We'll simulate a set of the ar ```typescript const indexName = getEnv("PINECONE_INDEX"); -const pineconeClient = await getPineconeClient(); -const pineconeIndex = pineconeClient.Index(indexName); +const pinecone = new Pinecone(); + +// Ensure the index exists +try { + const description = await pinecone.describeIndex(indexName); + if (!description.status?.ready) { + throw `Index not ready, description was ${JSON.stringify(description)}` + } +} catch (e) { + console.log('An error occurred. Run "npm run index" to load data into the index before querying.') + throw e; +} + +const index = pinecone.index(indexName).namespace('default'); await embedder.init("Xenova/all-MiniLM-L6-v2"); -// We create a simulated a user with an interest given a query and a specific section const { query, section } = getQueryingCommandLineArguments(); -const queryEmbedding = await embedder.embed(query); -const queryResult = await pineconeIndex.query({ - queryRequest: { +// We create a simulated user with an interest given a query and a specific section +const queryEmbedding = await embedder.embed(query) +const queryResult = await index.query({ vector: queryEmbedding.values, includeMetadata: true, includeValues: true, - namespace: "default", filter: { - section: { $eq: section }, + section: { "$eq": section } }, - topK: 10, - }, + topK: 10 }); ``` @@ -170,22 +170,20 @@ We'll calculate the **mean** vector given the results of the query. The mean vec ```typescript // We extract the vectors of the results -const userVectors = queryResult?.matches?.map( - (result: ScoredVector) => result.values as number[] -); +const userVectors = queryResult?.matches?.map((result: ScoredPineconeRecord) => result.values); // A couple of functions to calculate mean vector -const mean = (arr: number[]): number => - arr.reduce((a, b) => a + b, 0) / arr.length; +const mean = (arr: number[]): number => arr.reduce((a, b) => a + b, 0) / arr.length; const meanVector = (vectors: number[][]): number[] => { const { length } = vectors[0]; return Array.from({ length }).map((_, i) => - mean(vectors.map((vec) => vec[i])) + mean(vectors.map(vec => vec[i])) ); }; // We calculate the mean vector of the results +// eslint-disable-next-line @typescript-eslint/no-non-null-assertion const meanVec = meanVector(userVectors!); ``` diff --git a/src/index.ts b/src/index.ts index dd78e59..978c4f6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,16 +10,7 @@ import { embedder } from "embeddings.ts"; import loadCSVFile from "utils/csvLoader.ts"; import splitFile from "utils/fileSplitter.ts"; import { chunkedUpsert } from './utils/chunkedUpsert.ts'; - -type ArticleRecord = { - index: number, - title: string; - article: string; - publication: string; - url: string; - author: string; - section: string; -} +import type { ArticleRecord } from "types.ts"; dotenv.config(); diff --git a/src/recommend.ts b/src/recommend.ts index b3ac72f..78ab98a 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -4,26 +4,30 @@ import { embedder } from "embeddings.ts"; import { Table } from 'console-table-printer'; import { Pinecone } from "@pinecone-database/pinecone"; import type { ScoredPineconeRecord } from "@pinecone-database/pinecone"; +import type { ArticleRecord } from "types.ts"; const indexName = getEnv("PINECONE_INDEX"); const pinecone = new Pinecone(); -type DocumentMetadata = { - title?: string, - author?: string, - section?: string, - publication?: string, +// Ensure the index exists +try { + const description = await pinecone.describeIndex(indexName); + if (!description.status?.ready) { + throw `Index not ready, description was ${JSON.stringify(description)}` + } +} catch (e) { + console.log('An error occurred. Run "npm run index" to load data into the index before querying.') + throw e; } -const index = pinecone.index(indexName); +const index = pinecone.index(indexName).namespace('default'); await embedder.init("Xenova/all-MiniLM-L6-v2"); const { query, section } = getQueryingCommandLineArguments(); // We create a simulated user with an interest given a query and a specific section -const queryEmbedding = await embedder.embed(query); - +const queryEmbedding = await embedder.embed(query) const queryResult = await index.query({ vector: queryEmbedding.values, includeMetadata: true, @@ -35,7 +39,7 @@ const queryResult = await index.query({ }); // We extract the vectors of the results -const userVectors = queryResult?.matches?.map((result: ScoredPineconeRecord) => result.values); +const userVectors = queryResult?.matches?.map((result: ScoredPineconeRecord) => result.values); // A couple of functions to calculate mean vector const mean = (arr: number[]): number => arr.reduce((a, b) => a + b, 0) / arr.length; diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..67857bc --- /dev/null +++ b/src/types.ts @@ -0,0 +1,10 @@ +export type ArticleRecord = { + index: number, + title: string; + article: string; + publication: string; + url: string; + author: string; + section: string; + } + \ No newline at end of file From 46bb634c822b1b1bbb3968f310f12a1e8b279a21 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 7 Sep 2023 07:46:21 -0400 Subject: [PATCH 3/4] Add missing configuration instructions --- .env.example | 1 - README.md | 39 +++++++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index a644d33..4cbd9e5 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,3 @@ -OPENAI_API_KEY= PINECONE_API_KEY= PINECONE_ENVIRONMENT= PINECONE_INDEX= \ No newline at end of file diff --git a/README.md b/README.md index 4642c49..23617cd 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,27 @@ The goal is to create a recommendation engine that retrieves the best article re npm install ``` -## Dependencies +## Required configuration -We'll start by importing the necessary libraries. +In order to run this example, you have to supply the Pinecone credentials needed to interact with the Pinecone API. You can find these credentials in the [Pinecone web console](https://app.pinecone.io) under **API Keys**. This project uses `dotenv` to easily load values from the `.env` file into the environment when executing. -- We'll use the `@pinecone-database/pinecone` library to interact with Pinecone. -- We'll use the `danfojs-node` library to load the data into an easy to manipulate dataframe. -- We'll use the `Document` type from Langchain to keep the data structure consistent across the indexing process and retrieval agent. -- We'll use the `Embedder` class found in `embeddings.ts` to embed the data. -- We'll use the `cli-progress` library to display a progress bar. +Copy the template file: -To load the dataset used in the example, we'll be using a utility called `squadLoader.js`. +```sh +cp .env.example .env +``` + +And fill in your API key and environment details: + +```sh +PINECONE_API_KEY= +PINECONE_ENVIRONMENT= +PINECONE_INDEX=article-recommendations +``` -## Upload articles +`PINECONE_INDEX` is the name of the index where this demo will store and query embeddings. You can change `PINECONE_INDEX` to any name you like, but make sure the name not going to collide with any indexes you are already using. + +## Data preparation Next, we will prepare data for the Pinecone vector index, and insert it in batches. @@ -37,7 +45,9 @@ mv all-the-news-2-1.csv data/. ## Create Vector embeddings -Since the dataset could be pretty big, we'll use a generator function that will yield chunks of data to be processed. +To load data into our index, we need to create embeddings and upsert records into Pinecone. Run `npm run index` to do that in this project. + +Since the dataset could be pretty big, this project uses a generator function that will yield chunks of data to be processed. ```typescript async function* processInChunks( @@ -72,7 +82,7 @@ Here are the parameters the function accepts: - `metadataFields`: This is an array of field names (which are keys of `T`) to be included in the metadata of each `Document`. - `pageContentField`: This is the field name (which is a key of `T`) to be used for the page content of each `Document`. -Here's what it the function does: +Here's what the function does: 1. It loops over the DataFrame in chunks of size `chunkSize`. 2. For each chunk, it converts the chunk to JSON to get an array of records (of type `T`). @@ -121,7 +131,12 @@ const clean = data.dropNa() as dfd.DataFrame; Now we'll create the Pinecone index and kick off the embedding and upserting process. ```typescript -await createIndexIfNotExists(pineconeClient, indexName, 384); +// Create the index if it doesn't already exist +const indexList = await pinecone.listIndexes(); +if (indexList.indexOf({ name: indexName }) === -1) { + await pinecone.createIndex({ name: indexName, dimension: 384, waitUntilReady: true }) +} + progressBar.start(clean.shape[0], 0); await embedder.init("Xenova/all-MiniLM-L6-v2"); await embedAndUpsert(clean, 1); From 820461d87232d95f0d2e0e697428915bf227cff9 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Thu, 7 Sep 2023 11:09:35 -0400 Subject: [PATCH 4/4] Consume v1 release --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 929b0eb..8053c59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^0.1.6-dev.20230907100643", + "@pinecone-database/pinecone": "^1.0.0", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", @@ -731,9 +731,9 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "0.1.6-dev.20230907100643", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.6-dev.20230907100643.tgz", - "integrity": "sha512-EQvF3xivlSyx7y2XKZ3YWs8iPFTM4ydjSO1paNqnf56yh/1bBShR8vPcbdA/FaCM+KZgStTUFz1pLJHZFz7aQw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.0.0.tgz", + "integrity": "sha512-CtsfbK4qTDjnS56FVH64FEWNVnhwOyheBlLe3e9T6o9Gaxc00f/079JWUUiZ1lrkc3K/YkmlYYOXbdGyKP2z3A==", "dependencies": { "@sinclair/typebox": "^0.28.15", "@types/web": "^0.0.99", @@ -7395,9 +7395,9 @@ } }, "@pinecone-database/pinecone": { - "version": "0.1.6-dev.20230907100643", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.6-dev.20230907100643.tgz", - "integrity": "sha512-EQvF3xivlSyx7y2XKZ3YWs8iPFTM4ydjSO1paNqnf56yh/1bBShR8vPcbdA/FaCM+KZgStTUFz1pLJHZFz7aQw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-1.0.0.tgz", + "integrity": "sha512-CtsfbK4qTDjnS56FVH64FEWNVnhwOyheBlLe3e9T6o9Gaxc00f/079JWUUiZ1lrkc3K/YkmlYYOXbdGyKP2z3A==", "requires": { "@sinclair/typebox": "^0.28.15", "@types/web": "^0.0.99", diff --git a/package.json b/package.json index 7fa28f9..bb71ac6 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "author": "", "license": "MIT", "dependencies": { - "@pinecone-database/pinecone": "^0.1.6-dev.20230907100643", + "@pinecone-database/pinecone": "^1.0.0", "@xenova/transformers": "^2.2.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0",