From dc88ae98f3f563d2aa0d579901d59a1c8e3c6342 Mon Sep 17 00:00:00 2001 From: davidatbraintrust <138083560+davidatbraintrust@users.noreply.github.com> Date: Fri, 27 Oct 2023 16:51:24 -0700 Subject: [PATCH 1/4] Add braintrust integration tutorial --- docs/integrations.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/integrations.md b/docs/integrations.md index ddb363f..9b0923c 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -41,6 +41,14 @@ import TabItem from '@theme/TabItem'; +## Braintrust + +Braintrust is the enterprise-grade stack for building AI products. From evaluations, to prompt playground, to data management, +we take uncertainty and tedium out of incorporating AI into +your business. + +- [Evaluate a Chroma RAG app using Braintrust](https://www.braintrustdata.com/docs/examples/rag) + ## 🦜️🔗 LangchainJS Here is an [example in LangChainJS](https://github.com/hwchase17/langchainjs/blob/main/examples/src/chains/chat_vector_db_chroma.ts) From 92381c8e6f5b960e198a607db0c2d3eba35d5e49 Mon Sep 17 00:00:00 2001 From: davidatbraintrust <140105352+davidateg@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:02:11 -0800 Subject: [PATCH 2/4] Updated --- docs/integrations/braintrust.md | 12 ++++++++++++ docs/integrations/index.md | 3 ++- sidebars.js | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 docs/integrations/braintrust.md diff --git a/docs/integrations/braintrust.md b/docs/integrations/braintrust.md new file mode 100644 index 0000000..746393b --- /dev/null +++ b/docs/integrations/braintrust.md @@ -0,0 +1,12 @@ +--- +slug: /integrations/braintrust +title: Braintrust +--- + +[Braintrust](braintrustdata.com) is the enterprise-grade stack for building AI products. They provide tools including: evaluations, prompt playground, dataset management, tracing, etc. + +It's easy to use Braintrust to evaluate AI apps built with Chroma. Braintrust provides a Typescript and Python library to run and log evaluations. + +- [Tutorial: Evaluate Chroma Retrieval app w/ Braintrust](https://www.braintrustdata.com/docs/examples/rag) + +Learn more on their [docs](https://www.braintrustdata.com/docs). diff --git a/docs/integrations/index.md b/docs/integrations/index.md index 3f2f272..d080fc3 100644 --- a/docs/integrations/index.md +++ b/docs/integrations/index.md @@ -16,8 +16,9 @@ We welcome pull requests to add new Integrations to the community. |--------------|-----------|---------------| | [🦜️🔗 Langchain](/integrations/langchain) | ✅ | ✅ | | [🦙 LlamaIndex](/integrations/llama-index) | ✅ | :soon: | +| [Braintrust](/integrations/braintrust) | ✅ | ✅ | -*Coming soon* - integrations with LangSmith, JinaAI, Braintrust and more. +*Coming soon* - integrations with LangSmith, JinaAI, and more. *** diff --git a/sidebars.js b/sidebars.js index 57130ed..630af73 100644 --- a/sidebars.js +++ b/sidebars.js @@ -65,6 +65,7 @@ const sidebars = { items: [ 'integrations/langchain', 'integrations/llama-index', + 'integrations/braintrust', ], }, ], From 58549d8cf77923e4bc1fd51c4206c1d38e86e6ab Mon Sep 17 00:00:00 2001 From: davidatbraintrust <140105352+davidateg@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:37:26 -0800 Subject: [PATCH 3/4] Add code example --- docs/integrations/braintrust.md | 51 ++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/integrations/braintrust.md b/docs/integrations/braintrust.md index 746393b..cb2938d 100644 --- a/docs/integrations/braintrust.md +++ b/docs/integrations/braintrust.md @@ -5,8 +5,57 @@ title: Braintrust [Braintrust](braintrustdata.com) is the enterprise-grade stack for building AI products. They provide tools including: evaluations, prompt playground, dataset management, tracing, etc. -It's easy to use Braintrust to evaluate AI apps built with Chroma. Braintrust provides a Typescript and Python library to run and log evaluations. +It's easy to use Braintrust to evaluate AI retrieval apps built with Chroma. Braintrust provides a Typescript and Python library to run and log evaluations. - [Tutorial: Evaluate Chroma Retrieval app w/ Braintrust](https://www.braintrustdata.com/docs/examples/rag) +Example evaluation script in Python: +(refer to the tutorial above to get the full implementation) +```python +from autoevals.llm import * +from braintrust import Eval + +PROJECT_NAME="Chroma_Eval" + +from openai import OpenAI + +client = OpenAI() +leven_evaluator = LevenshteinScorer() + +async def pipeline_a(input, hooks=None): + # Get a relevant fact from Chroma + relevant = collection.query( + query_texts=[input], + n_results=1, + ) + relevant_text = ','.join(relevant["documents"][0]) + prompt = """ + You are an assistant called BT. Help the user. + Relevant information: {relevant} + Question: {question} + Answer: + """.format(question=input, relevant=relevant_text) + messages = [{"role": "system", "content": prompt}] + response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=messages, + temperature=0, + max_tokens=100, + ) + + result = response.choices[0].message.content + return result + +# Run an evaluation and log to Braintrust +await Eval( + PROJECT_NAME, + # define your test cases + data = lambda:[{"input": "What is my eye color?", "expected": "Brown"}], + # define your retrieval pipeline w/ Chroma above + task = pipeline_a, + # use a prebuilt scoring function or define your own :) + scores=[leven_evaluator], +) +``` + Learn more on their [docs](https://www.braintrustdata.com/docs). From 31ea625a31b141469389bcbcd2d512bb14548eaa Mon Sep 17 00:00:00 2001 From: Jeffrey Huber Date: Tue, 19 Dec 2023 09:27:07 -0800 Subject: [PATCH 4/4] tune language --- docs/integrations/braintrust.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/integrations/braintrust.md b/docs/integrations/braintrust.md index cb2938d..e33e979 100644 --- a/docs/integrations/braintrust.md +++ b/docs/integrations/braintrust.md @@ -3,9 +3,9 @@ slug: /integrations/braintrust title: Braintrust --- -[Braintrust](braintrustdata.com) is the enterprise-grade stack for building AI products. They provide tools including: evaluations, prompt playground, dataset management, tracing, etc. +[Braintrust](braintrustdata.com) is an enterprise-grade stack for building AI products including: evaluations, prompt playground, dataset management, tracing, etc. -It's easy to use Braintrust to evaluate AI retrieval apps built with Chroma. Braintrust provides a Typescript and Python library to run and log evaluations. +Braintrust provides a Typescript and Python library to run and log evaluations and integrates well with Chroma. - [Tutorial: Evaluate Chroma Retrieval app w/ Braintrust](https://www.braintrustdata.com/docs/examples/rag) @@ -58,4 +58,4 @@ await Eval( ) ``` -Learn more on their [docs](https://www.braintrustdata.com/docs). +Learn more: [docs](https://www.braintrustdata.com/docs).