From 0e745ec215caf837a9cad71a11a737d69b166bd4 Mon Sep 17 00:00:00 2001 From: Ashok Gelal Date: Sat, 2 Mar 2024 17:32:24 -0500 Subject: [PATCH 1/6] add Msty provider --- CONTRIBUTING.md | 2 +- core/config/types.ts | 3 +- core/index.d.ts | 3 +- core/llm/autodetect.ts | 2 + core/llm/llms/Msty.ts | 12 +++++ core/llm/llms/index.ts | 2 + docs/docs/config-file-migration.md | 14 +++++ docs/docs/model-setup/overview.md | 2 +- docs/docs/model-setup/select-provider.md | 3 +- docs/docs/reference/Model Providers/msty.md | 49 +++++++++++++++++ docs/docs/walkthroughs/codellama.md | 22 +++++++- .../walkthroughs/config-file-migration.md | 14 +++++ docs/static/schemas/config.json | 54 +++++++++++++++++-- gui/src/pages/models.tsx | 2 +- gui/src/util/modelData.ts | 45 ++++++++++++---- schema/json/SerializedContinueConfig.json | 3 +- 16 files changed, 211 insertions(+), 21 deletions(-) create mode 100644 core/llm/llms/Msty.ts create mode 100644 docs/docs/reference/Model Providers/msty.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 036e5734bdf..90264968bf4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,7 +113,7 @@ After you've written your context provider, make sure to complete the following: ### Adding an LLM Provider -Continue has support for more than a dozen different LLM "providers", making it easy to use models running on OpenAI, Ollama, Together, LM Studio, and more. You can find all of the existing providers [here](https://github.com/continuedev/continue/tree/main/core/llm/llms), and if you see one missing, you can add it with the following steps: +Continue has support for more than a dozen different LLM "providers", making it easy to use models running on OpenAI, Ollama, Together, LM Studio, Msty, and more. You can find all of the existing providers [here](https://github.com/continuedev/continue/tree/main/core/llm/llms), and if you see one missing, you can add it with the following steps: 1. Create a new file in the `core/llm/llms` directory. The name of the file should be the name of the provider, and it should export a class that extends `BaseLLM`. This class should contain the following minimal implementation. We recommend viewing pre-existing providers for more details. The [LlamaCpp Provider](./core/llm/llms/LlamaCpp.ts) is a good simple example. diff --git a/core/config/types.ts b/core/config/types.ts index abd2aeb3128..5e9f4251255 100644 --- a/core/config/types.ts +++ b/core/config/types.ts @@ -419,7 +419,8 @@ declare global { | "mistral" | "bedrock" | "deepinfra" - | "flowise"; + | "flowise" + | "msty"; export type ModelName = // OpenAI diff --git a/core/index.d.ts b/core/index.d.ts index f66df8b477e..08f38a159f9 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -451,7 +451,8 @@ type ModelProvider = | "mistral" | "bedrock" | "deepinfra" - | "flowise"; + | "flowise" + | "msty"; export type ModelName = | "AUTODETECT" diff --git a/core/llm/autodetect.ts b/core/llm/autodetect.ts index f68e22b115a..bad1a1e8f44 100644 --- a/core/llm/autodetect.ts +++ b/core/llm/autodetect.ts @@ -34,6 +34,7 @@ const PROVIDER_HANDLES_TEMPLATING: ModelProvider[] = [ "openai", "ollama", "together", + "msty", ]; const PROVIDER_SUPPORTS_IMAGES: ModelProvider[] = [ @@ -41,6 +42,7 @@ const PROVIDER_SUPPORTS_IMAGES: ModelProvider[] = [ "ollama", "google-palm", "free-trial", + "msty" ]; function modelSupportsImages(provider: ModelProvider, model: string): boolean { diff --git a/core/llm/llms/Msty.ts b/core/llm/llms/Msty.ts new file mode 100644 index 00000000000..2f126ad1391 --- /dev/null +++ b/core/llm/llms/Msty.ts @@ -0,0 +1,12 @@ +import Ollama from "./Ollama"; +import {LLMOptions, ModelProvider} from "../../index"; + +class Msty extends Ollama { + static providerName: ModelProvider = "msty"; + static defaultOptions: Partial = { + apiBase: "http://localhost:10000", + model: "codellama-7b", + }; +} + +export default Msty; diff --git a/core/llm/llms/index.ts b/core/llm/llms/index.ts index 9da9ec2c050..69375c88676 100644 --- a/core/llm/llms/index.ts +++ b/core/llm/llms/index.ts @@ -26,6 +26,7 @@ import OpenAIFreeTrial from "./OpenAIFreeTrial"; import Replicate from "./Replicate"; import TextGenWebUI from "./TextGenWebUI"; import Together from "./Together"; +import Msty from "./Msty"; function convertToLetter(num: number): string { let result = ""; @@ -94,6 +95,7 @@ const LLMs = [ DeepInfra, OpenAIFreeTrial, Flowise, + Msty ]; export async function llmFromDescription( diff --git a/docs/docs/config-file-migration.md b/docs/docs/config-file-migration.md index d38c7232798..b755aa02b3a 100644 --- a/docs/docs/config-file-migration.md +++ b/docs/docs/config-file-migration.md @@ -197,6 +197,20 @@ After the "Full example" these examples will only show the relevant portion of t } ``` +### Msty with CodeLlama 13B + +```json +{ + "models": [ + { + "title": "Msty", + "provider": "msty", + "model": "codellama-13b" + } + ] +} +``` + ### OpenAI-compatible API This is an example of serving a model using an OpenAI-compatible API on http://localhost:8000. diff --git a/docs/docs/model-setup/overview.md b/docs/docs/model-setup/overview.md index 88ac00c1adf..cc9797c0eb1 100644 --- a/docs/docs/model-setup/overview.md +++ b/docs/docs/model-setup/overview.md @@ -10,7 +10,7 @@ If you are unsure what model or provider to use, here is our current rule of thu - Use GPT-4 via OpenAI if you want the best possible model overall - Use DeepSeek Coder 33B via the Together API if you want the best open-source model -- Use DeepSeek Coder 6.7B with Ollama if you want to run a model locally +- Use DeepSeek Coder 6.7B with Ollama or Msty if you want to run a model locally Learn more: diff --git a/docs/docs/model-setup/select-provider.md b/docs/docs/model-setup/select-provider.md index 1f0cc1e08f0..6c7c7fae72b 100644 --- a/docs/docs/model-setup/select-provider.md +++ b/docs/docs/model-setup/select-provider.md @@ -1,7 +1,7 @@ --- title: Select a provider description: Swap out different LLM providers -keywords: [openai, anthropic, PaLM, ollama, ggml] +keywords: [openai, anthropic, PaLM, ollama, ggml, msty] --- # Select a model provider @@ -22,6 +22,7 @@ You can run a model on your local computer using: - [FastChat](../reference/Model%20Providers/openai.md) (OpenAI compatible server) - [llama-cpp-python](../reference/Model%20Providers/openai.md) (OpenAI compatible server) - [TensorRT-LLM](https://github.com/NVIDIA/trt-llm-as-openai-windows?tab=readme-ov-file#examples) (OpenAI compatible server) +- [Msty](../reference/Model%20Providers/msty.md) Once you have it running, you will need to configure it in the GUI or manually add it to your `config.json`. diff --git a/docs/docs/reference/Model Providers/msty.md b/docs/docs/reference/Model Providers/msty.md new file mode 100644 index 00000000000..584176b9b71 --- /dev/null +++ b/docs/docs/reference/Model Providers/msty.md @@ -0,0 +1,49 @@ +# Msty + +[Msty](https://msty.app/) is an application for Windows, Mac, and Linux that makes it really easy to run online as well as local open-source models, including Llama-2, DeepSeek Coder, etc. No need to fidget with your terminal, run a command, or anything. Just download the app from the website, click a button, and you are up and running. Continue can then be configured to use the `Msty` LLM class: + +```json title="~/.continue/config.json" +{ + "models": [ + { + "title": "Msty", + "provider": "msty", + "model": "deepseek-coder:6.7b", + "completionOptions": {} + } + ] +} +``` + +## Completion Options + +In addition to the model type, you can also configure some of the parameters that Msty uses to run the model. + +- temperature: options.temperature - This is a parameter that controls the randomness of the generated text. Higher values result in more creative but potentially less coherent outputs, while lower values lead to more predictable and focused outputs. +- top_p: options.topP - This sets a threshold (between 0 and 1) to control how diverse the predicted tokens should be. The model generates tokens that are likely according to their probability distribution, but also considers the top-k most probable tokens. +- top_k: options.topK - This parameter limits the number of unique tokens to consider when generating the next token in the sequence. Higher values increase the variety of generated sequences, while lower values lead to more focused outputs. +- num_predict: options.maxTokens - This determines the maximum number of tokens (words or characters) to generate for the given input prompt. +- num_thread: options.numThreads - This is the multi-threading configuration option that controls how many threads the model uses for parallel processing. Higher values may lead to faster generation times but could also increase memory usage and complexity. Set this to one or two lower than the number of threads your CPU can handle to leave some for your GUI when running the model locally. + +## Authentication + +If you need to send custom headers for authentication, you may use the `requestOptions.headers` property like this: + +```json title="~/.continue/config.json" +{ + "models": [ + { + "title": "Msty", + "provider": "msty", + "model": "deepseek-coder:6.7b", + "requestOptions": { + "headers": { + "Authorization": "Bearer xxx" + } + } + } + ] +} +``` + +[View the source](https://github.com/continuedev/continue/blob/main/core/llm/llms/Msty.ts) diff --git a/docs/docs/walkthroughs/codellama.md b/docs/docs/walkthroughs/codellama.md index 89177abf95b..ab4586779db 100644 --- a/docs/docs/walkthroughs/codellama.md +++ b/docs/docs/walkthroughs/codellama.md @@ -1,12 +1,12 @@ --- title: Using Code Llama with Continue description: How to use Code Llama with Continue -keywords: [code llama, meta, togetherai, ollama, replciate, fastchat] +keywords: [code llama, meta, togetherai, ollama, replciate, fastchat, msty] --- # Using Code Llama with Continue -With Continue, you can use Code Llama as a drop-in replacement for GPT-4, either by running locally with Ollama or GGML or through Replicate. +With Continue, you can use Code Llama as a drop-in replacement for GPT-4, either by running locally with Ollama, Msty, or GGML or through Replicate. If you haven't already installed Continue, you can do that [here](https://marketplace.visualstudio.com/items?itemName=Continue.continue). For more general information on customizing Continue, read [our customization docs](../customization/overview.md). @@ -83,3 +83,21 @@ If you haven't already installed Continue, you can do that [here](https://market ] } ``` + +## Msty + +1. Download Msty [here](https://msty.app/) for your platform (Windows, Mac, or Linux) +2. Open the app and click "Setup Local AI". Optionally, download any model you want with click of a button from Text Module page. +3. Change your Continue config file like this: + +```json title="~/.continue/config.json" +{ + "models": [ + { + "title": "Code Llama", + "provider": "msty", + "model": "codellama:7b" + } + ] +} +``` diff --git a/docs/docs/walkthroughs/config-file-migration.md b/docs/docs/walkthroughs/config-file-migration.md index b8e83d2f812..52ba61271b2 100644 --- a/docs/docs/walkthroughs/config-file-migration.md +++ b/docs/docs/walkthroughs/config-file-migration.md @@ -196,6 +196,20 @@ After the "Full example" these examples will only show the relevant portion of t } ``` +### Msty with CodeLlama 13B + +```json +{ + "models": [ + { + "title": "Msty", + "provider": "msty", + "model": "codellama-13b" + } + ] +} +``` + ### OpenAI-compatible API This is an example of serving a model using an OpenAI-compatible API on http://localhost:8000. diff --git a/docs/static/schemas/config.json b/docs/static/schemas/config.json index 51686f00b65..6f7480413c5 100644 --- a/docs/static/schemas/config.json +++ b/docs/static/schemas/config.json @@ -33,7 +33,7 @@ }, "mirostat": { "title": "Mirostat", - "description": "Enable Mirostat sampling, controlling perplexity during text generation (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0). Only available for Ollama, LM Studio, and llama.cpp providers", + "description": "Enable Mirostat sampling, controlling perplexity during text generation (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0). Only available for Ollama, LM Studio, Msty, and llama.cpp providers", "type": "number" }, "stop": { @@ -117,7 +117,8 @@ "llamafile", "mistral", "deepinfra", - "flowise" + "flowise", + "msty" ], "markdownEnumDescriptions": [ "### OpenAI\nUse gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.\n\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/openai)", @@ -133,7 +134,8 @@ "### LMStudio\nLMStudio provides a professional and well-designed GUI for exploring, configuring, and serving LLMs. It is available on both Mac and Windows. To get started:\n1. Download from [lmstudio.ai](https://lmstudio.ai/) and open the application\n2. Search for and download the desired model from the home screen of LMStudio.\n3. In the left-bar, click the '<->' icon to open the Local Inference Server and press 'Start Server'.\n4. Once your model is loaded and the server has started, you can begin using Continue.\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/lmstudio)", "### Llamafile\nTo get started with llamafiles, find and download a binary on their [GitHub repo](https://github.com/Mozilla-Ocho/llamafile#binary-instructions). Then run it with the following command:\n\n```shell\nchmod +x ./llamafile\n./llamafile\n```\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/llamafile)", "### Mistral API\n\nTo get access to the Mistral API, obtain your API key from the [Mistral platform](https://docs.mistral.ai/)", - "### DeepInfra\n\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/deepinfra)" + "### DeepInfra\n\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/deepinfra)", + "### Msty\nMsty is the simplest way to get started with online or local LLMs on all desktop platforms - Windows, Mac, and Linux. No fussing around, one-click and you are up and running. To get started, follow these steps:\n1. Download from [Msty.app](https://msty.app/) and open the application, and click 'Setup Local AI'\n2. Go to Local AI Module page and download a model of your choice.\n3. Once the model has finished downloading, you can start asking questions through Continue.\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/Msty)" ], "type": "string" }, @@ -637,6 +639,52 @@ } } }, + { + "if": { + "properties": { + "provider": { + "enum": ["msty"] + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "model": { + "anyOf": [ + { + "enum": [ + "mistral-7b", + "llama2-7b", + "llama2-13b", + "codellama-7b", + "codellama-13b", + "codellama-34b", + "codellama-70b", + "phi-2", + "phind-codellama-34b", + "wizardcoder-7b", + "wizardcoder-13b", + "wizardcoder-34b", + "zephyr-7b", + "codeup-13b", + "deepseek-7b", + "deepseek-33b", + "neural-chat-7b", + "deepseek-1b", + "stable-code-3b", + "starcoder-1b", + "starcoder-3b", + "AUTODETECT" + ] + }, + { "type": "string" } + ], + "markdownDescription": "Select a pre-defined option, or find the exact model tag for a model from Local AI Module page in Msty." + } + } + } + }, { "if": { "properties": { diff --git a/gui/src/pages/models.tsx b/gui/src/pages/models.tsx index 782a4298415..147fae5afcb 100644 --- a/gui/src/pages/models.tsx +++ b/gui/src/pages/models.tsx @@ -69,7 +69,7 @@ function Models() { To set up an LLM you will choose