diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bb1fafc2..b7a44af9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,7 +16,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "python3 -m venv /workspaces/data-formulator/venv && . /workspaces/data-formulator/venv/bin/activate && pip install -r /workspaces/data-formulator/requirements.txt --verbose && yarn install && yarn build" + "postCreateCommand": "python3 -m venv /workspaces/data-formulator/venv && . /workspaces/data-formulator/venv/bin/activate && pip install https://github.com/user-attachments/files/17319752/data_formulator-0.1.0.tar.gz --verbose && data_formulator" // Configure tool-specific properties. // "customizations": {}, diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 85a8e538..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,34 +0,0 @@ -# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs - -name: Node.js CI - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [18.x] - # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'yarn' - - - name: Install dependencies - run: yarn install - - - name: Run build - run: yarn build \ No newline at end of file diff --git a/.github/workflows/python-build.yml b/.github/workflows/python-build.yml new file mode 100644 index 00000000..46d6652a --- /dev/null +++ b/.github/workflows/python-build.yml @@ -0,0 +1,62 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: build + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: Set Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'yarn' + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install node dependencies + run: yarn install + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install build + - name: Build frontend + run: yarn build + - name: Build python artifact + run: python -m build + - name: Archive production artifacts + uses: actions/upload-artifact@v4 + with: + name: release-dist + path: dist + + pypi-publish: + runs-on: ubuntu-latest + needs: + - build + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') # only publish when push with tag + environment: + name: pypi + url: https://pypi.org/p/data-formulator + permissions: + id-token: write + steps: + - name: Retrieve release distributions + uses: actions/download-artifact@v4 + with: + name: release-dist + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 83c3c1b9..b6982ac0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ + + +*openai-keys.env **/*.ipynb_checkpoints/ .DS_Store diff --git a/CODESPACES.md b/CODESPACES.md index bf43f167..2545a261 100644 --- a/CODESPACES.md +++ b/CODESPACES.md @@ -15,12 +15,11 @@ You will need a GitHub account and to be logged in to use Codespaces. ### Step 2: Run the app The codespace is a VSCode development environment in the cloud. Once the Codespace is created, start Data Formuator with the following steps: -* Press **F5** to run. Or if you prefer, click the **Run and Debug** tab on the left, and the **Start Debugging** button. * A toast about port forwarding will appear, click the **Open in Browser** button. * You will see the Data Formulator app! - image + image diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 6ba8c1bc..80a6dfd4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -9,15 +9,15 @@ How to set up your local machine. ## Backend (Python) - **Create a Virtual Environment** -```bash -python -m venv venv -.\venv\Scripts\activate -``` + ```bash + python -m venv venv + .\venv\Scripts\activate + ``` - **Install Dependencies** -```bash -pip install -r requirements.txt -``` + ```bash + pip install -r requirements.txt + ``` - **Run** - **Windows** @@ -33,9 +33,10 @@ pip install -r requirements.txt ## Frontend (TypeScript) - **Install NPM packages** -```bash -yarn -``` + + ```bash + yarn + ``` - **Development mode** @@ -46,14 +47,43 @@ yarn Open [http://localhost:3000](http://localhost:3000) to view it in the browser. The page will reload if you make edits. You will also see any lint errors in the console. -- **Build for Production** +## Build for Production + +- **Build the frontend and then the backend** + Compile the TypeScript files and bundle the project: ```bash yarn build ``` - This builds the app for production to the `dist` folder. + This builds the app for production to the `py-src/data_formulator/dist` folder. + + Then, build python package: + + ```bash + pip install build + python -m build + ``` + This will create a python wheel in the `dist/` folder. The name would be `data_formulator--py3-none-any.whl` + +- **Test the artifact** + + You can then install the build result wheel (testing in a virtual environment is recommended): + ```bash + # replace with the actual build version. + pip install dist/data_formulator--py3-none-any.whl + ``` + + Once installed, you can run Data Formulator with: + ```bash + data_formulator + ``` + or + ```bash + python -m data_formulator + ``` Open [http://localhost:5000](http://localhost:5000) to view it in the browser. + ## Usage See the [Usage section on the README.md page](README.md#usage). diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..8e281551 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include py-src/data_formulator/dist/* +include py-src/data_formulator/dist/assets/* \ No newline at end of file diff --git a/README.md b/README.md index c4ad623a..afc330c1 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ [![arxiv](https://img.shields.io/badge/Paper-arXiv:2408.16119-b31b1b.svg)](https://arxiv.org/abs/2408.16119)  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)  +[![YouTube](https://img.shields.io/badge/YouTube-white?logo=youtube&logoColor=%23FF0000)](https://youtu.be/3ndlwt0Wi3c)  +[![build](https://github.com/microsoft/data-formulator/actions/workflows/python-build.yml/badge.svg)](https://github.com/microsoft/data-formulator/actions/workflows/python-build.yml) @@ -13,36 +15,58 @@ Transform data and create rich visualizations iteratively with AI 🪄. Try Data [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/microsoft/data-formulator?quickstart=1) - + +## News 🔥🔥🔥 + +- [10-11-2024] Data Formulator python package released! + - You can now install Data Formulator using Python and run it locally, easily. [[check it out]](#get-started). + - Our Codespace configuration is also updated for fast start up ⚡️. [[try it now!]](https://codespaces.new/microsoft/data-formulator?quickstart=1) + - New exprimental feature: load an image or a messy text, and ask AI parsing and cleaning it for you(!). [[demo]](https://github.com/microsoft/data-formulator/pull/31#issuecomment-2403652717) + +- [10-01-2024] Initial release of Data Formulator, check out our [[blog]](https://www.microsoft.com/en-us/research/blog/data-formulator-exploring-how-ai-can-help-analysts-create-rich-data-visualizations/) and [[video]](https://youtu.be/3ndlwt0Wi3c)! + + + ## Overview **Data Formulator** is an application from Microsoft Research that uses large language models to transform data, expediting the practice of data visualization. -To create rich visualizations, data analysts often need to iterate back and forth among data processing and chart specification to achieve their goals. To achieve this, analysts need proficiency in data transformation and visualization tools, and they also spend effort managing the iteration history. This can be challenging! +Data Formulator is an AI-powered tool for analysts to iteratively create rich visualizations. Unlike most chat-based AI tools where users need to describe everything in natural language, Data Formulator combines *user interface interactions (UI)* and *natural language (NL) inputs* for easier interaction. This blended approach makes it easier for users to describe their chart designs while delegating data transformation to AI. -Data Formulator is an AI-powered tool for analysts to iteratively create rich visualizations. Unlike most chat-based AI tools where users need to describe everything in natural language, Data Formulator combines user interface interactions (UI) with natural language (NL) inputs. This blended approach makes it easier for users to describe their chart designs while delegating data transformation to AI. +## Get Started -Check out these cool Data Formulator features that can help you create impressive visualizations! -* Using the **blended UI and NL inputs** to describe the chart. -* Utilizing **data threads** to navigate the history and reuse previous results to create new ones instead of starting from scratch every time. +Play with Data Formulator with one of the following options: -## Get Started +- **Option 1: Install via Python PIP** + + Use Python PIP for an easy setup experience, running locally (recommend: install it in a virtual environment). + + ```bash + # install data_formulator + pip install data_formulator -Choose one of the following options to set up Data Formulator: + # start data_formulator + data_formulator + + # alternatively, you can run data formualtor with this command + python -m data_formulator + ``` -- **Option 1: Codespaces** + Data Formulator will be automatically opened in the browser at [http://localhost:5000](http://localhost:5000). + +- **Option 2: Codespaces (5 minutes)** - Use Codespaces for an easy setup experience, as everything is preconfigured to get you up and running quickly. For more details, see [CODESPACES.md](CODESPACES.md). + You can also run Data Formualtor in codespace, we have everything pre-configured. For more details, see [CODESPACES.md](CODESPACES.md). [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/microsoft/data-formulator?quickstart=1) -- **Option 2: Local Installation** +- **Option 3: Working in the developer mode** - Opt for a local installation if you prefer full control over your development environment and the ability to customize the setup to your specific needs. For detailed instructions, refer to [DEVELOPMENT.md](DEVELOPMENT.md). + You can build Data Formulator locally if you prefer full control over your development environment and the ability to customize the setup to your specific needs. For detailed instructions, refer to [DEVELOPMENT.md](DEVELOPMENT.md). ## Using Data Formulator @@ -50,19 +74,22 @@ Choose one of the following options to set up Data Formulator: Once you’ve completed the setup using either option, follow these steps to start using Data Formulator: ### The basics of data visualization -* Provide OpenAI keys and select a model (GPT-4o suggested) and choose a dataset -* Choose a visualization type -* Drag and drop data fields to the encoding shelf to create visualization - +* Provide OpenAI keys and select a model (GPT-4o suggested) and choose a dataset. +* Choose a chart type, and then drag-and-drop data fields to chart properties (x, y, color, ...) to specify visual encodings. https://github.com/user-attachments/assets/0fbea012-1d2d-46c3-a923-b1fc5eb5e5b8 ### Create visualization beyond the initial dataset (powered by 🤖) -* Add new field names in the encoding shelf, describe the chart intent -* Click the **Formulate** button -* Inspect the code behind the concept -* Follow up the chart to create new ones +* You can type names of **fields that do not exist in current data** in the encoding shelf: + - this tells Data Formulator that you want to create visualizions that require computation or transformation from existing data, + - you can optionally provide a natural language prompt to explain your intent to clarify your intent (not necessary when field names are self-explanatory). +* Click the **Formulate** button. + - Data Formulator will transform data and instantiate the visualization based on the encoding and prompt. +* Inspect the data, chart and code. +* To create a new chart based on existing ones, follow up in natural language: + - provide a follow up prompt (e.g., *``show only top 5!''*), + - you may also update visual encodings for the new chart. https://github.com/user-attachments/assets/160c69d2-f42d-435c-9ff3-b1229b5bddba @@ -70,11 +97,10 @@ https://github.com/user-attachments/assets/c93b3e84-8ca8-49ae-80ea-f91ceef34acb Repeat this process as needed to explore and understand your data. Your explorations are trackable in the **Data Threads** panel. -## Developers +## Developers' Guide Follow the [developers' instructions](DEVELOPMENT.md) to build your new data analysis tools on top of Data Formulator. - ## Research Papers * [Data Formulator 2: Iteratively Creating Rich Visualizations with AI](https://arxiv.org/abs/2408.16119) diff --git a/local_server.bat b/local_server.bat index 4a3fe91a..c8e24a8e 100644 --- a/local_server.bat +++ b/local_server.bat @@ -2,7 +2,7 @@ :: Licensed under the MIT License. @echo off -set FLASK_APP=app.py +set FLASK_APP=py-src/data_formulator/app.py set FLASK_RUN_PORT=5000 set FLASK_RUN_HOST=0.0.0.0 flask run diff --git a/local_server.sh b/local_server.sh index d51b5775..5fdb2a5a 100644 --- a/local_server.sh +++ b/local_server.sh @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -env FLASK_APP=app.py FLASK_RUN_PORT=5000 FLASK_RUN_HOST=0.0.0.0 flask run \ No newline at end of file +env FLASK_APP=py-src/data_formulator/app.py FLASK_RUN_PORT=5000 FLASK_RUN_HOST=0.0.0.0 flask run \ No newline at end of file diff --git a/package.json b/package.json index f6edbba3..58faa2ed 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "ag-grid-enterprise": "^32.0.2", "ag-grid-react": "^32.0.2", "d3": "^7.3.0", + "dompurify": "^3.1.7", "localforage": "^1.10.0", "lodash": "^4.17.21", "markdown-to-jsx": "^7.1.8", @@ -24,6 +25,7 @@ "react": "^18.2.0", "react-animate-height": "^3.0.4", "react-animate-on-change": "^2.2.0", + "react-diff-viewer": "^3.1.1", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", "react-dom": "^18.2.0", @@ -37,6 +39,7 @@ "redux": "^4.2.0", "redux-persist": "^6.0.0", "typescript": "^4.9.5", + "validator": "^13.12.0", "vega": "^5.23.0", "vega-embed": "^6.21.0", "vega-lite": "^5.5.0", diff --git a/py-src/data_formulator/__init__.py b/py-src/data_formulator/__init__.py new file mode 100644 index 00000000..9b558668 --- /dev/null +++ b/py-src/data_formulator/__init__.py @@ -0,0 +1,5 @@ +from .app import run_app + +__all__ = [ + "run_app", +] \ No newline at end of file diff --git a/py-src/data_formulator/__main__.py b/py-src/data_formulator/__main__.py new file mode 100644 index 00000000..ebc97f9a --- /dev/null +++ b/py-src/data_formulator/__main__.py @@ -0,0 +1,4 @@ +from .app import run_app + +if __name__ == "__main__": + run_app() \ No newline at end of file diff --git a/py-src/data_formulator/agents/__init__.py b/py-src/data_formulator/agents/__init__.py new file mode 100644 index 00000000..766dd2fd --- /dev/null +++ b/py-src/data_formulator/agents/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from data_formulator.agents.agent_concept_derive import ConceptDeriveAgent +from data_formulator.agents.agent_py_concept_derive import PyConceptDeriveAgent +from data_formulator.agents.agent_data_transformation import DataTransformationAgent +from data_formulator.agents.agent_data_transform_v2 import DataTransformationAgentV2 +from data_formulator.agents.agent_data_load import DataLoadAgent +from data_formulator.agents.agent_sort_data import SortDataAgent +from data_formulator.agents.agent_data_clean import DataCleanAgent +from data_formulator.agents.agent_data_rec import DataRecAgent + +__all__ = [ + "ConceptDeriveAgent", + "PyConceptDeriveAgent", + "DataTransformationAgent", + "DataTransformationAgentV2", + "DataRecAgent", + "DataLoadAgent", + "SortDataAgent", + "DataCleanAgent" +] \ No newline at end of file diff --git a/server/agents/agent_code_explanation.py b/py-src/data_formulator/agents/agent_code_explanation.py similarity index 96% rename from server/agents/agent_code_explanation.py rename to py-src/data_formulator/agents/agent_code_explanation.py index 504db710..0053c69e 100644 --- a/server/agents/agent_code_explanation.py +++ b/py-src/data_formulator/agents/agent_code_explanation.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import pandas as pd -from agents.agent_utils import generate_data_summary, extract_code_from_gpt_response +from data_formulator.agents.agent_utils import generate_data_summary, extract_code_from_gpt_response import logging diff --git a/server/agents/agent_concept_derive.py b/py-src/data_formulator/agents/agent_concept_derive.py similarity index 97% rename from server/agents/agent_concept_derive.py rename to py-src/data_formulator/agents/agent_concept_derive.py index fabeb996..69873b47 100644 --- a/server/agents/agent_concept_derive.py +++ b/py-src/data_formulator/agents/agent_concept_derive.py @@ -8,7 +8,7 @@ APP_ROOT = os.path.abspath('..') sys.path.append(os.path.abspath(APP_ROOT)) -from agents.agent_utils import generate_data_summary, field_name_to_ts_variable_name, extract_code_from_gpt_response, infer_ts_datatype +from data_formulator.agents.agent_utils import generate_data_summary, field_name_to_ts_variable_name, extract_code_from_gpt_response, infer_ts_datatype import logging diff --git a/py-src/data_formulator/agents/agent_data_clean.py b/py-src/data_formulator/agents/agent_data_clean.py new file mode 100644 index 00000000..8674d302 --- /dev/null +++ b/py-src/data_formulator/agents/agent_data_clean.py @@ -0,0 +1,150 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +import pandas as pd + +from data_formulator.agents.agent_utils import extract_json_objects, generate_data_summary, extract_code_from_gpt_response, field_name_to_ts_variable_name, infer_ts_datatype + +import logging + +logger = logging.getLogger(__name__) + + +SYSTEM_PROMPT = '''You are a data scientist to help user to generate or clean the raw input into a *csv block* (or tsv if that's the original format). +The output csv format should be readable into a python pandas dataframe directly. + +Create [OUTPUT] based on [RAW DATA] provided. The output should have two components: + +1. a csv codeblock that represents the cleaned data, as follows: + +```csv +..... +``` + +2. a json object that explains the mode and cleaning rationale (wrap in a json block): + +```json +{ + "mode": ..., // one of "data generation" or "data cleaning" based on the provided task + "reason": ... // explain the cleaning reason here +} +``` + +**Important:** +- NEVER make assumptions or judgments about a person's gender, biological sex, sexuality, religion, race, nationality, ethnicity, political stance, socioeconomic status, mental health, invisible disabilities, medical conditions, personality type, social impressions, emotional state, and cognitive state. +- NEVER create formulas that could be used to discriminate based on age. Ageism of any form (explicit and implicit) is strictly prohibited. +- If above issue occurs, just copy the original data and return in the block + +The cleaning process must follow instructions below: +* the output should be a structured csv table: + - if the raw data is unstructured, structure it into a csv table. If the table is in other formats, transform it into a csv table. + - if the raw data contain other informations other than the table, remove surrounding texts that does not belong to the table. + - if the raw data contains multiple levels of header, make it a flat table. It's ok to combine multiple levels of headers to form the new header to not lose information. + - if the table has footer or summary row, remove them, since they would not be compatible with the csv table format. + - the csv table should have the same number of cells for each line, according to the title. If there are some rows with missing values, patch them with empty cells. + - if the raw data has some rows that do not belong to the table, also remove them (e.g., subtitles in between rows) + - if the header row misses some columns, add their corresponding column names. E.g., when the header doesn't have an index column, but every row has an index value, add the missing column header. +* clean up columns with messy information + - if a column is number but some cells has annotations like "*" "?" or brackets, clean them up. + - if a column is number but has units like ($, %, s), convert them to number (make sure unit conversion is correct when multiple units exist like minute and second) and include unit in the header. + - you don't need to convert format of the cell. +* if the user asks about generating synthetic data: + - NEVER generate data that has implicit bias as noted above, if that happens, return a dummy data consisting of dummy columns with 'a, b, c' and numbers. + - NEVER generate data contain people's names, use "A" , "B", "C"... instead. + - If the user doesn't indicate how many rows to be generated, plan in generating a dataset with 10-20 rows depending on the content. +''' + + + +EXAMPLE = ''' +[RAW DATA] + +Rank NOC Gold Silver Bronze Total +1 South Korea 5 1 1 7 +2 France* 0 1 1 2 + United States 0 1 1 2 +4 China 0 1 0 1 + Germany 0 1 0 1 +6 Mexico 0 0 1 1 + Turkey 0 0 1 1 +Totals (7 entries) 5 5 5 15 + +[OUTPUT] + +''' + +class DataCleanAgent(object): + + def __init__(self, client, model): + self.model = model + self.client = client + + def run(self, content_type, raw_data): + """derive a new concept based on the raw input data + """ + + if content_type == "text": + user_prompt = { + "role": "user", + "content": [{ + 'type': 'text', + 'text': f"[DATA]\n\n{raw_data}\n\n[OUTPUT]\n" + }] + } + elif content_type == "image": + user_prompt = { + 'role': 'user', + 'content': [ { + 'type': 'text', + 'text': '''[RAW_DATA]\n\n'''}, + { + 'type': 'image_url', + 'image_url': { + "url": raw_data, + "detail": "high" + } + }, + { + 'type': 'text', + 'text': '''[OUTPUT]\n\n''' + }, + ] + } + + logger.info(user_prompt) + + system_message = { + 'role': 'system', + 'content': [ {'type': 'text', 'text': SYSTEM_PROMPT}]} + + messages = [system_message, user_prompt] + + ###### the part that calls open_ai + response = self.client.chat.completions.create( + model=self.model, messages = messages, temperature=0.7, max_tokens=1200, + top_p=0.95, n=1, frequency_penalty=0, presence_penalty=0, stop=None) + + candidates = [] + for choice in response.choices: + + logger.info("\n=== Python Data Clean Agent ===>\n") + logger.info(choice.message.content + "\n") + + code_blocks = extract_code_from_gpt_response(choice.message.content + "\n", "csv") + reason_blocks = extract_json_objects(choice.message.content + "\n") + + if len(code_blocks) > 0: + result = { + 'status': 'ok', + 'content': code_blocks[-1], + 'info': reason_blocks[-1] if len(reason_blocks) > 0 else {"reason": "no reason presented", "mode": "data cleaning"} + } + else: + result = {'status': 'other error', 'content': 'unable to extract code from response'} + + result['dialog'] = [*messages, {"role": choice.message.role, "content": choice.message.content}] + result['agent'] = 'DataCleanAgent' + candidates.append(result) + + return candidates \ No newline at end of file diff --git a/server/agents/agent_data_filter.py b/py-src/data_formulator/agents/agent_data_filter.py similarity index 97% rename from server/agents/agent_data_filter.py rename to py-src/data_formulator/agents/agent_data_filter.py index 5edf7fcd..895f7663 100644 --- a/server/agents/agent_data_filter.py +++ b/py-src/data_formulator/agents/agent_data_filter.py @@ -3,8 +3,8 @@ import json -from agents.agent_utils import generate_data_summary, extract_code_from_gpt_response -import py_sandbox +from data_formulator.agents.agent_utils import generate_data_summary, extract_code_from_gpt_response +import data_formulator.py_sandbox as py_sandbox import logging diff --git a/py-src/data_formulator/agents/agent_data_load.py b/py-src/data_formulator/agents/agent_data_load.py new file mode 100644 index 00000000..86fb367d --- /dev/null +++ b/py-src/data_formulator/agents/agent_data_load.py @@ -0,0 +1,173 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json + +from data_formulator.agents.agent_utils import extract_json_objects, generate_data_summary +import logging + +logger = logging.getLogger(__name__) + + +SYSTEM_PROMPT = '''You are a data scientist to help user infer data types based off the table provided by the user. +Given a dataset provided by the user, identify their type and semantic type, and provide a very short summary of the dataset. + +Types to consider include: string, number, date +Semantic types to consider include: Location, Year, Month, Day, Date, Time, DateTime, Range, Duration, Name, Percentage, String, Number + +Furthermore, if the field is string type and is ordinal (especially for english month name, week name, range), provide the natural sort order of the fields here. +Otherwise, put sort_order as null (for example, Name should not be sorted). + +Special cases: +* sometimes, column name is year like "2020", "2021" but its content is not actually year (e.g., sales), in these cases, the semantic type of the column would not be Year! + +Create a json object function based off the [DATA] provided. + +output should be in the format of: + +```json +{ + "fields": { + "field1": {"type": ..., "semantic_type": ..., "sort_order": [...]}, // replace field1 field2 with actual field names, if the field is string type and is ordinal, provide the natural sort order of the fields here + "field2": {"type": ..., "semantic_type": ..., "sort_order": null}, + ... + }, + "data summary": ... // a short summary of the data +} +``` +''' + +EXAMPLES = ''' +[DATA] + +Here are our datasets, here are their field summaries and samples: + +table_0 (income_json) fields: + name -- type: object, values: Alabama, Alaska, Arizona, Arkansas, California, Colorado, Connecticut, Delaware, District of Columbia, Florida, ..., South Dakota, Tennessee, Texas, Utah, Vermont, Virginia, Washington, West Virginia, Wisconsin, Wyoming + region -- type: object, values: midwest, northeast, other, south, west + state_id -- type: int64, values: 1, 2, 4, 5, 6, 8, 9, 10, 11, 12, ..., 47, 48, 49, 50, 51, 53, 54, 55, 56, 72 + pct -- type: float64, values: 0.006, 0.008, 0.02, 0.021, 0.022, 0.024, 0.025, 0.026000000000000002, 0.027, 0.028, ..., 0.192, 0.193, 0.194, 0.196, 0.197, 0.199, 0.2, 0.201, 0.213, 0.289 + total -- type: int64, values: 222679, 250875, 256563, 268015, 291468, 326086, 337245, 405504, 410347, 449296, ..., 3522934, 3721358, 3815532, 4551497, 4763457, 4945140, 7168502, 7214163, 8965352, 12581722 + group -- type: object, values: 10000 to 14999, 100000 to 149999, 15000 to 24999, 150000 to 199999, 200000+, 25000 to 34999, 35000 to 49999, 50000 to 74999, 75000 to 99999, <10000 + +table_0 (income_json) sample: + +``` +|name|region|state_id|pct|total|group +0|Alabama|south|1|0.10200000000000001|1837292|<10000 +1|Alabama|south|1|0.07200000000000001|1837292|10000 to 14999 +2|Alabama|south|1|0.13|1837292|15000 to 24999 +3|Alabama|south|1|0.115|1837292|25000 to 34999 +4|Alabama|south|1|0.14300000000000002|1837292|35000 to 49999 +...... +``` + +[OUTPUT] + +```json +{ + "fields": { + "name": {"type": "string", "semantic_type": "Location", "sort_order": null}, + "region": {"type": "string", "semantic_type": "String", "sort_order": ["northeast", "midwest", "south", "west", "other"]}, + "state_id": {"type": "number", "semantic_type": "Number", "sort_order": null}, + "pct": {"type": "number", "semantic_type": "Percentage", "sort_order": null}, + "total": {"type": "number", "semantic_type": "Number", "sort_order": null}, + "group": {"type": "string", "semantic_type": "Range", "sort_order": ["<10000", "10000 to 14999", "15000 to 24999", "25000 to 34999", "35000 to 49999", "50000 to 74999", "75000 to 99999", "100000 to 149999", "150000 to 199999", "200000+"]} + }, + "data summary": "The dataset contains information about income distribution across different states in the USA. It includes fields for state names, regions, state IDs, percentage of total income, total income, and income groups." +} +``` + +[DATA] + +Here are our datasets, here are their field summaries and samples: + +table_0 (weather_seattle_atlanta) fields: + Date -- type: object, values: 1/1/2020, 1/10/2020, 1/11/2020, ..., 9/6/2020, 9/7/2020, 9/8/2020, 9/9/2020 + City -- type: object, values: Atlanta, Seattle + Temperature -- type: int64, values: 30, 31, 32, ..., 83, 84, 85, 86 + +table_0 (weather_seattle_atlanta) sample: +``` +|Date|City|Temperature +0|1/1/2020|Seattle|51 +1|1/1/2020|Atlanta|45 +2|1/2/2020|Seattle|45 +3|1/2/2020|Atlanta|47 +4|1/3/2020|Seattle|48 +...... +``` + +[OUTPUT] + +``` +{ + "fields": { + "Date": { + "type": "string", + "semantic_type": "Date", + "sort_order": null + }, + "City": { + "type": "string", + "semantic_type": "Location", + "sort_order": null + }, + "Temperature": { + "type": "number", + "semantic_type": "Number", + "sort_order": null + } + }, + "data_summary": "This dataset contains weather information for the cities of Seattle and Atlanta. The fields include the date, city name, and temperature readings. The 'Date' field represents dates in a string format, the 'City' field represents city names, and the 'Temperature' field represents temperature values in integer format." +}```''' + +class DataLoadAgent(object): + + def __init__(self, client, model): + self.client = client + self.model = model + + def run(self, input_data, n=1): + + data_summary = generate_data_summary([input_data], include_data_samples=True, field_sample_size=30) + + user_query = f"[DATA]\n\n{data_summary}\n\n[OUTPUT]" + + logger.info(user_query) + + messages = [{"role":"system", "content": SYSTEM_PROMPT}, + {"role":"user","content": user_query}] + + ###### the part that calls open_ai + response = self.client.chat.completions.create( + model=self.model, messages=messages, temperature=0.2, max_tokens=4096, + top_p=0.95, n=n, frequency_penalty=0, presence_penalty=0, stop=None) + + #log = {'messages': messages, 'response': response.model_dump(mode='json')} + + candidates = [] + for choice in response.choices: + + logger.info("\n=== Data load result ===>\n") + logger.info(choice.message.content + "\n") + + json_blocks = extract_json_objects(choice.message.content + "\n") + logger.info(json_blocks) + + if len(json_blocks) > 0: + result = {'status': 'ok', 'content': json_blocks[0]} + else: + try: + json_block = json.loads(choice.message.content + "\n") + result = {'status': 'ok', 'content': json_block} + except: + result = {'status': 'other error', 'content': 'unable to extract VegaLite script from response'} + + # individual dialog for the agent + result['dialog'] = [*messages, {"role": choice.message.role, "content": choice.message.content}] + result['agent'] = 'DataLoadAgent' + + candidates.append(result) + + return candidates \ No newline at end of file diff --git a/server/agents/agent_data_rec.py b/py-src/data_formulator/agents/agent_data_rec.py similarity index 97% rename from server/agents/agent_data_rec.py rename to py-src/data_formulator/agents/agent_data_rec.py index c67c38b1..56c8b3d0 100644 --- a/server/agents/agent_data_rec.py +++ b/py-src/data_formulator/agents/agent_data_rec.py @@ -3,11 +3,13 @@ import json -from agents.agent_utils import extract_json_objects, generate_data_summary, extract_code_from_gpt_response -import py_sandbox +from data_formulator.agents.agent_utils import extract_json_objects, generate_data_summary, extract_code_from_gpt_response +from data_formulator.agents.agent_data_transform_v2 import completion_response_wrapper + +import data_formulator.py_sandbox as py_sandbox + import traceback -from agents.agent_data_transform_v2 import completion_response_wrapper import logging diff --git a/server/agents/agent_data_transform_v2.py b/py-src/data_formulator/agents/agent_data_transform_v2.py similarity index 98% rename from server/agents/agent_data_transform_v2.py rename to py-src/data_formulator/agents/agent_data_transform_v2.py index abc1644a..fc71489f 100644 --- a/server/agents/agent_data_transform_v2.py +++ b/py-src/data_formulator/agents/agent_data_transform_v2.py @@ -3,8 +3,9 @@ import json -from agents.agent_utils import extract_json_objects, generate_data_summary, extract_code_from_gpt_response -import py_sandbox +from data_formulator.agents.agent_utils import extract_json_objects, generate_data_summary, extract_code_from_gpt_response +import data_formulator.py_sandbox as py_sandbox + import traceback import logging diff --git a/server/agents/agent_data_transformation.py b/py-src/data_formulator/agents/agent_data_transformation.py similarity index 98% rename from server/agents/agent_data_transformation.py rename to py-src/data_formulator/agents/agent_data_transformation.py index c46e9922..37b337fb 100644 --- a/server/agents/agent_data_transformation.py +++ b/py-src/data_formulator/agents/agent_data_transformation.py @@ -3,8 +3,9 @@ import json -from agents.agent_utils import generate_data_summary, extract_code_from_gpt_response -import py_sandbox +from data_formulator.agents.agent_utils import generate_data_summary, extract_code_from_gpt_response +import data_formulator.py_sandbox as py_sandbox + import traceback import logging diff --git a/server/agents/agent_generic_py_concept.py b/py-src/data_formulator/agents/agent_generic_py_concept.py similarity index 98% rename from server/agents/agent_generic_py_concept.py rename to py-src/data_formulator/agents/agent_generic_py_concept.py index cac835a0..6ec2bab5 100644 --- a/server/agents/agent_generic_py_concept.py +++ b/py-src/data_formulator/agents/agent_generic_py_concept.py @@ -3,8 +3,8 @@ import json -from agents.agent_utils import generate_data_summary, extract_code_from_gpt_response -import py_sandbox +from data_formulator.agents.agent_utils import generate_data_summary, extract_code_from_gpt_response +import data_formulator.py_sandbox as py_sandbox import traceback diff --git a/server/agents/agent_py_concept_derive.py b/py-src/data_formulator/agents/agent_py_concept_derive.py similarity index 96% rename from server/agents/agent_py_concept_derive.py rename to py-src/data_formulator/agents/agent_py_concept_derive.py index fe410b3f..57299347 100644 --- a/server/agents/agent_py_concept_derive.py +++ b/py-src/data_formulator/agents/agent_py_concept_derive.py @@ -4,8 +4,9 @@ import json import pandas as pd -from agents.agent_utils import generate_data_summary, extract_code_from_gpt_response, field_name_to_ts_variable_name, infer_ts_datatype -import py_sandbox +from data_formulator.agents.agent_utils import generate_data_summary, extract_code_from_gpt_response, field_name_to_ts_variable_name, infer_ts_datatype +import data_formulator.py_sandbox as py_sandbox + import traceback import logging diff --git a/server/agents/agent_sort_data.py b/py-src/data_formulator/agents/agent_sort_data.py similarity index 97% rename from server/agents/agent_sort_data.py rename to py-src/data_formulator/agents/agent_sort_data.py index 17b053e6..aa13ed35 100644 --- a/server/agents/agent_sort_data.py +++ b/py-src/data_formulator/agents/agent_sort_data.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import json -from agents.agent_utils import extract_json_objects +from data_formulator.agents.agent_utils import extract_json_objects import logging @@ -96,7 +96,7 @@ def run(self, name, values, n=1): logger.info("\n=== Sort data agent ===>\n") logger.info(choice.message.content + "\n") - json_blocks = extract_json_objects(choice.message.content + "\n", "json") + json_blocks = extract_json_objects(choice.message.content + "\n") if len(json_blocks) > 0: result = {'status': 'ok', 'content': json_blocks[0]} diff --git a/server/agents/agent_utils.py b/py-src/data_formulator/agents/agent_utils.py similarity index 97% rename from server/agents/agent_utils.py rename to py-src/data_formulator/agents/agent_utils.py index 829ff722..802112f3 100644 --- a/server/agents/agent_utils.py +++ b/py-src/data_formulator/agents/agent_utils.py @@ -180,7 +180,7 @@ def dedup_data_transform_candidates(candidates): return [items[0] for _, items in candidate_groups.items()] -def get_field_summary(field_name, df): +def get_field_summary(field_name, df, field_sample_size): try: values = sorted([x for x in list(set(df[field_name].values)) if x != None]) except: @@ -188,7 +188,7 @@ def get_field_summary(field_name, df): val_sample = "" - sample_size = 7 + sample_size = field_sample_size if len(values) <= sample_size: val_sample = values @@ -199,7 +199,7 @@ def get_field_summary(field_name, df): return f"{field_name} -- type: {df[field_name].dtype}, values: {val_str}" -def generate_data_summary(input_tables, include_data_samples=True): +def generate_data_summary(input_tables, include_data_samples=True, field_sample_size=7): input_table_names = [f'{string_to_py_varname(t["name"])}' for t in input_tables] @@ -208,7 +208,7 @@ def generate_data_summary(input_tables, include_data_samples=True): field_summaries = [] for input_data in input_tables: df = pd.DataFrame(input_data['rows']) - s = '\n\t'.join([get_field_summary(fname, df) for fname in list(df.columns.values)]) + s = '\n\t'.join([get_field_summary(fname, df, field_sample_size) for fname in list(df.columns.values)]) field_summaries.append(s) table_field_summaries = [f'table_{i} ({input_table_names[i]}) fields:\n\t{s}' for i, s in enumerate(field_summaries)] diff --git a/server/agents/client_utils.py b/py-src/data_formulator/agents/client_utils.py similarity index 100% rename from server/agents/client_utils.py rename to py-src/data_formulator/agents/client_utils.py diff --git a/app.py b/py-src/data_formulator/app.py similarity index 80% rename from app.py rename to py-src/data_formulator/app.py index 35eee8b6..f6b2679c 100644 --- a/app.py +++ b/py-src/data_formulator/app.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import random import sys import os @@ -10,6 +11,9 @@ import html import pandas as pd +import webbrowser +import threading + from flask_cors import CORS import json @@ -18,33 +22,31 @@ from vega_datasets import data as vega_data -APP_ROOT = Path(os.path.join(Path(__file__).parent, 'server')).absolute() -sys.path.append(os.path.abspath(APP_ROOT)) - -from agents.agent_concept_derive import ConceptDeriveAgent -from agents.agent_data_transformation import DataTransformationAgent -from agents.agent_data_transform_v2 import DataTransformationAgentV2 -from agents.agent_data_rec import DataRecAgent - -from agents.agent_sort_data import SortDataAgent -from agents.agent_data_load import DataLoadAgent -from agents.agent_data_filter import DataFilterAgent -from agents.agent_generic_py_concept import GenericPyConceptDeriveAgent -from agents.agent_code_explanation import CodeExplanationAgent +from data_formulator.agents.agent_concept_derive import ConceptDeriveAgent +from data_formulator.agents.agent_data_transform_v2 import DataTransformationAgentV2 +from data_formulator.agents.agent_data_rec import DataRecAgent -from agents.client_utils import get_client +from data_formulator.agents.agent_sort_data import SortDataAgent +from data_formulator.agents.agent_data_load import DataLoadAgent +from data_formulator.agents.agent_data_clean import DataCleanAgent +from data_formulator.agents.agent_code_explanation import CodeExplanationAgent -import pathlib +from data_formulator.agents.client_utils import get_client from dotenv import load_dotenv +APP_ROOT = Path(os.path.join(Path(__file__).parent)).absolute() -APP_DIR = pathlib.Path(__file__).parent.resolve() -load_dotenv(os.path.join(APP_DIR, 'openai-keys.env')) +print(APP_ROOT) + +# try to look for stored openAI keys information from the ROOT dir, +# this file might be in one of the two locations +load_dotenv(os.path.join(APP_ROOT, "..", "..", 'openai-keys.env')) +load_dotenv(os.path.join(APP_ROOT, 'openai-keys.env')) import os -app = Flask(__name__, static_url_path='', static_folder=os.path.join(APP_ROOT, "..", "dist")) +app = Flask(__name__, static_url_path='', static_folder=os.path.join(APP_ROOT, "dist")) CORS(app) @app.route('/vega-datasets') @@ -61,14 +63,6 @@ def get_example_dataset_list(): except: pass - # this is a dataset we use for demoing the system - try: - with open('global-energy.json', 'r') as f: - info_obj = {'name': 'global-energy.csv', 'snapshot': json.dumps(json.load(f))} - dataset_info.append(info_obj) - except: - pass - response = flask.jsonify(dataset_info) response.headers.add('Access-Control-Allow-Origin', '*') return response @@ -76,14 +70,9 @@ def get_example_dataset_list(): @app.route('/vega-dataset/') def get_datasets(path): try: - # this is a dataset we use for demoing the system - if path == "global-energy.csv": - with open('global-energy.json', 'r') as f: - data_object = json.dumps(json.load(f)) - else: - df = vega_data(path) - # to_json is necessary for handle NaN issues - data_object = df.to_json(None, 'records') + df = vega_data(path) + # to_json is necessary for handle NaN issues + data_object = df.to_json(None, 'records') except Exception as err: print(path) print(err) @@ -286,6 +275,34 @@ def derive_concept_request(): response.headers.add('Access-Control-Allow-Origin', '*') return response + +@app.route('/clean-data', methods=['GET', 'POST']) +def clean_data_request(): + + if request.is_json: + app.logger.info("# data clean request") + content = request.get_json() + token = content["token"] + + client = get_client(content['model']['endpoint'], content['model']['key']) + model = content['model']['model'] + + app.logger.info(f" model: {content['model']}") + + agent = DataCleanAgent(client=client, model=model) + + candidates = agent.run(content['content_type'], content["raw_data"]) + + candidates = [c for c in candidates if c['status'] == 'ok'] + + response = flask.jsonify({ "status": "ok", "token": token, "result": candidates }) + else: + response = flask.jsonify({ "token": -1, "status": "error", "result": [] }) + + response.headers.add('Access-Control-Allow-Origin', '*') + return response + + @app.route('/codex-sort-request', methods=['GET', 'POST']) def sort_data_request(): @@ -407,29 +424,21 @@ def refine_data(): print("previous dialog") print(dialog[0]['content']) - prev_system_prompt = dialog[0]['content'] - - if prev_system_prompt.startswith("You are a data scientist to help user to filter data based on user description."): - agent = DataFilterAgent(client, model=model) - results = agent.followup(input_tables[0], dialog, new_instruction) - elif prev_system_prompt.startswith("You are a data scientist to help user to derive new column based on existing columns in a dataset."): - agent = GenericPyConceptDeriveAgent(client, model=model) - new_field_name = [field['name'] for field in output_fields if field['name'] not in input_tables[0][0].keys()][0] - results = agent.followup(input_tables[0], new_field_name, dialog, new_instruction) - else: - agent = DataTransformationAgentV2(client, model=model) - results = agent.followup(input_tables, dialog, [field['name'] for field in output_fields], new_instruction) - repair_attempts = 0 - while results[0]['status'] == 'error' and repair_attempts < 2: - error_message = results[0]['content'] - new_instruction = f"We run into the following problem executing the code, please fix it:\n\n{error_message}\n\nPlease think step by step, reflect why the error happens and fix the code so that no more errors would occur." + # always resort to the data transform agent + agent = DataTransformationAgentV2(client, model=model) + results = agent.followup(input_tables, dialog, [field['name'] for field in output_fields], new_instruction) - response_message = dialog['response']['choices'][0]['message'] - prev_dialog = [*dialog['messages'], {"role": response_message['role'], 'content': response_message['content']}] + repair_attempts = 0 + while results[0]['status'] == 'error' and repair_attempts < 2: + error_message = results[0]['content'] + new_instruction = f"We run into the following problem executing the code, please fix it:\n\n{error_message}\n\nPlease think step by step, reflect why the error happens and fix the code so that no more errors would occur." - results = agent.followup(input_tables, prev_dialog, [field['name'] for field in output_fields], new_instruction) - repair_attempts += 1 + response_message = dialog['response']['choices'][0]['message'] + prev_dialog = [*dialog['messages'], {"role": response_message['role'], 'content': response_message['content']}] + + results = agent.followup(input_tables, prev_dialog, [field['name'] for field in output_fields], new_instruction) + repair_attempts += 1 response = flask.jsonify({ "status": "ok", "token": token, "results": results}) else: @@ -438,8 +447,15 @@ def refine_data(): response.headers.add('Access-Control-Allow-Origin', '*') return response +def run_app(): + port = 5000 #+ random.randint(0, 999) + url = "http://localhost:{0}".format(port) + threading.Timer(2, lambda: webbrowser.open(url, new=2) ).start() + + app.run(host='0.0.0.0', port=port, threaded=True) + if __name__ == '__main__': #app.run(debug=True, host='127.0.0.1', port=5000) #use 0.0.0.0 for public - app.run(host='0.0.0.0', port=5000, threaded=True) \ No newline at end of file + run_app() \ No newline at end of file diff --git a/server/py_sandbox.py b/py-src/data_formulator/py_sandbox.py similarity index 100% rename from server/py_sandbox.py rename to py-src/data_formulator/py_sandbox.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0df6d4de --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = [ "setuptools >= 75.0" ] +build-backend = "setuptools.build_meta" + +[project] +name = "data_formulator" +version = "0.1.2" + +requires-python = ">=3.9" +authors = [ + {name = "Chenglong Wang", email = "chenglong.wang@microsoft.com"}, + {name = "Dan Marshall", email = "danmar@microsoft.com"}, +] +readme = "README.md" +license = {file = "LICENSE"} +description = "Data Formulator is research protoype data visualization tool powered by AI." +keywords = ["data visualization", "LLM", "AI"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python" +] + +dependencies = [ + "autopep8", + "jupyter", + "pandas", + "docker", + "namedlist", + "matplotlib", + "flask", + "flask-cors", + "openai", + "azure-identity", + "azure-keyvault-secrets", + "python-dotenv", + "vega_datasets" +] + +[project.urls] +Homepage = "https://github.com/microsoft/data-formulator" +Repository = "https://github.com/microsoft/data-formulator.git" +"Bug Tracker" = "https://github.com/microsoft/data-formulator/issues" + +[tool.setuptools] +package-dir = {"" = "py-src"} +include-package-data = true + +[project.scripts] +data_formulator = "data_formulator:run_app" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1a6ca681..8ba373ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ azure-identity azure-keyvault-secrets python-dotenv vega_datasets +-e . #also need to install data formulator itself \ No newline at end of file diff --git a/server/agents/__init__.py b/server/agents/__init__.py deleted file mode 100644 index 78fa41a6..00000000 --- a/server/agents/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -from .agent_concept_derive import ConceptDeriveAgent -from .agent_py_concept_derive import PyConceptDeriveAgent -from .agent_data_transformation import DataTransformationAgent -from .agent_data_transform_v2 import DataTransformationAgentV2 -from .agent_data_load import DataLoadAgent -from .agent_sort_data import SortDataAgent -from agents.agent_data_rec import DataRecAgent - -__all__ = [ - "ConceptDeriveAgent", - "PyConceptDeriveAgent", - "DataTransformationAgent", - "DataTransformationAgentV2", - "DataRecAgent", - "DataLoadAgent", - "SortDataAgent", -] \ No newline at end of file diff --git a/server/agents/agent_data_load.py b/server/agents/agent_data_load.py deleted file mode 100644 index b234bd4c..00000000 --- a/server/agents/agent_data_load.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -import json - -from agents.agent_utils import extract_json_objects, generate_data_summary -import logging - -logger = logging.getLogger(__name__) - - -SYSTEM_PROMPT = '''You are a data scientist to help user infer data types based off the table provided by the user. -Given a dataset provided by the user, identify their type and semantic type, and provide a very short summary of the dataset. - -Types to consider include: string, number, date -Semantic types to consider include: Location, Year, Month, Day, Date, Time, DateTime, Duration, Name, Percentage, String, Number - -Create a json object function based off the [DATA] provided. - -[DATA] - -Here are our datasets, here are their field summaries and samples: - -table_0 (us_covid_cases) fields: - Date -- type: object, values: 1/1/2021, 1/1/2022, 1/1/2023, ..., 9/8/2022, 9/9/2020, 9/9/2021, 9/9/2022 - Cases -- type: int64, values: -23999, -14195, -6940, ..., 1018935, 1032159, 1178403, 1433977 - -table_0 (us_covid_cases) sample: -``` -|Date|Cases -0|1/21/2020|1 -1|1/22/2020|0 -2|1/23/2020|0 -3|1/24/2020|1 -4|1/25/2020|1 -...... -``` - -[OUTPUT] - -{ - "fields": { - "Date": {"type": "date", "semantic_type": "Date"}, - "Cases": {"type": "number", "semantic_type": "Number"} - }, - "data summary": "US covid 19 data from 2020 to 2022" -} - -[DATA] - -Here are our datasets, here are their field summaries and samples: - -table_0 (weather_seattle_atlanta) fields: - Date -- type: object, values: 1/1/2020, 1/10/2020, 1/11/2020, ..., 9/6/2020, 9/7/2020, 9/8/2020, 9/9/2020 - City -- type: object, values: Atlanta, Seattle - Temperature -- type: int64, values: 30, 31, 32, ..., 83, 84, 85, 86 - -table_0 (weather_seattle_atlanta) sample: -``` -|Date|City|Temperature -0|1/1/2020|Seattle|51 -1|1/1/2020|Atlanta|45 -2|1/2/2020|Seattle|45 -3|1/2/2020|Atlanta|47 -4|1/3/2020|Seattle|48 -...... -``` - -[OUTPUT] - -{ - "fields": { - "Date": {"type": "date", "semantic_type": "Date"}, - "City": {"type": "string", "semantic_type": "Location"}, - "Temperature": {"type": "number", "semantic_type": "Number"} - }, - "data summary": "Seattle and Atlanta temperature in 2020" -} -''' - -class DataLoadAgent(object): - - def __init__(self, client, model): - self.client = client - self.model = model - - def run(self, input_data, n=1): - - data_summary = generate_data_summary([input_data], include_data_samples=True) - - user_query = f"[DATA]\n\n{data_summary}\n\n[OUTPUT]" - - logger.info(user_query) - - messages = [{"role":"system", "content": SYSTEM_PROMPT}, - {"role":"user","content": user_query}] - - ###### the part that calls open_ai - response = self.client.chat.completions.create( - model=self.model, messages=messages, temperature=0.2, max_tokens=2400, - top_p=0.95, n=n, frequency_penalty=0, presence_penalty=0, stop=None) - - #log = {'messages': messages, 'response': response.model_dump(mode='json')} - - candidates = [] - for choice in response.choices: - - logger.info("\n=== Data load result ===>\n") - logger.info(choice.message.content + "\n") - - json_blocks = extract_json_objects(choice.message.content + "\n") - logger.info(json_blocks) - - if len(json_blocks) > 0: - result = {'status': 'ok', 'content': json_blocks[0]} - else: - try: - json_block = json.loads(choice.message.content + "\n") - result = {'status': 'ok', 'content': json_block} - except: - result = {'status': 'other error', 'content': 'unable to extract VegaLite script from response'} - - # individual dialog for the agent - result['dialog'] = [*messages, {"role": choice.message.role, "content": choice.message.content}] - result['agent'] = 'DataLoadAgent' - - candidates.append(result) - - return candidates \ No newline at end of file diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 2ba7bebd..c2f8020b 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -216,7 +216,7 @@ export const dataFormulatorSlice = createSlice({ //state.table = undefined; // avoid resetting inputted models - //state.oaiModels = []; + // state.oaiModels = state.oaiModels.filter((m: any) => m.endpoint != 'default'); state.selectedModel = state.oaiModels.length > 0 ? state.oaiModels[0] : undefined; state.testedModels = []; @@ -428,6 +428,12 @@ export const dataFormulatorSlice = createSlice({ let encoding = (state.charts.find(chart => chart.id == chartId) as Chart).encodingMap[channel]; if (prop == 'fieldID') { encoding.fieldID = value; + + // automatcially fetch the auto-sort order from the field + let field = state.conceptShelfItems.find(f => f.id == value); + if (field?.levels) { + encoding.sortBy = JSON.stringify(field.levels); + } } else if (prop == 'bin') { encoding.bin = value; } else if (prop == 'aggregate') { @@ -453,8 +459,8 @@ export const dataFormulatorSlice = createSlice({ let enc1 = chart.encodingMap[channel1]; let enc2 = chart.encodingMap[channel2]; - chart.encodingMap[channel1] = { fieldID: enc2.fieldID, aggregate: enc2.aggregate, bin: enc2.bin }; - chart.encodingMap[channel2] = { fieldID: enc1.fieldID, aggregate: enc1.aggregate, bin: enc1.bin }; + chart.encodingMap[channel1] = { fieldID: enc2.fieldID, aggregate: enc2.aggregate, bin: enc2.bin, sortBy: enc2.sortBy }; + chart.encodingMap[channel2] = { fieldID: enc1.fieldID, aggregate: enc1.aggregate, bin: enc1.bin, sortBy: enc1.sortBy }; } }, addConceptItems: (state, action: PayloadAction) => { @@ -610,6 +616,9 @@ export const dataFormulatorSlice = createSlice({ if (((field.source == "original" && field.tableRef == tableId ) || field.source == "custom") && Object.keys(typeMap).includes(field.name)) { field.semanticType = typeMap[field.name]['semantic_type']; field.type = typeMap[field.name]['type'] as Type; + if (typeMap[field.name]['sort_order']) { + field.levels = { "values": typeMap[field.name]['sort_order'], "reason": "natural sort order"} + } return field; } else { return field; diff --git a/src/app/utils.tsx b/src/app/utils.tsx index c2a41ea6..a815fa01 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -37,8 +37,8 @@ export function getUrls() { // these functions involves openai models DERIVE_CONCEPT_URL: `${appConfig.serverUrl}/derive-concept-request`, SORT_DATA_URL: `${appConfig.serverUrl}/codex-sort-request`, + CLEAN_DATA_URL: `${appConfig.serverUrl}/clean-data`, SERVER_DERIVE_DATA_URL: `${appConfig.serverUrl}/derive-data`, - SERVER_DERIVE_DATA_V2_URL: `${appConfig.serverUrl}/dispatch-and-derive-data`, SERVER_REFINE_DATA_URL: `${appConfig.serverUrl}/refine-data`, CODE_EXPL_URL: `${appConfig.serverUrl}/code-expl`, SERVER_PROCESS_DATA_ON_LOAD: `${appConfig.serverUrl}/process-data-on-load`, @@ -423,7 +423,7 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i // use post processor to handle smart chart instantiation if (chartTemplate.postProcessor) { - vgObj = chartTemplate.postProcessor(vgObj); + vgObj = chartTemplate.postProcessor(vgObj, workingTable); } // console.log(JSON.stringify(vgObj)) diff --git a/src/assets/example-image-table.png b/src/assets/example-image-table.png new file mode 100644 index 00000000..be686135 Binary files /dev/null and b/src/assets/example-image-table.png differ diff --git a/src/components/ChartTemplates.tsx b/src/components/ChartTemplates.tsx index 31ee503d..a3b16d8d 100644 --- a/src/components/ChartTemplates.tsx +++ b/src/components/ChartTemplates.tsx @@ -158,7 +158,7 @@ const scatterPlots: ChartTemplate[] = [ "y": ["encoding", "y"], // a object can have multiple destinations "color": ["layer", 1, "encoding", "color"] }, - "postProcessor": (vgSpec: any) => { + "postProcessor": (vgSpec: any, table: any[]) => { if (vgSpec.encoding.y?.type == "nominal") { vgSpec['layer'][0]['encoding']['detail'] = JSON.parse(JSON.stringify(vgSpec['encoding']['y'])) } else if (vgSpec.encoding.x?.type == "nominal") { @@ -178,7 +178,7 @@ const scatterPlots: ChartTemplate[] = [ }, "channels": ["x", "y", "color", "opacity", "column", "row"], "paths": Object.fromEntries(["x", "y", "color", "opacity", "column", "row"].map(channel => [channel, ["encoding", channel]])), - "postProcessor": (vgSpec: any) => { + "postProcessor": (vgSpec: any, table: any[]) => { if (vgSpec.encoding.x && vgSpec.encoding.x.type != "nominal") { vgSpec.encoding.x.type = "nominal"; } @@ -204,6 +204,62 @@ const barCharts: ChartTemplate[] = [ "row": ["encoding", "row"] } }, + { + "chart": "Pyramid Chart", + "icon": chartIconColumn, + "template": { + "spacing": 0, + + "resolve": {"scale": {"y": "shared"}}, + "hconcat": [{ + "mark": "bar", + "encoding": { + "y": { }, + "x": { "scale": {"reverse": true}, "stack": null}, + "color": { "legend": null }, + "opacity": {"value": 0.9} + } + }, { + "mark": "bar", + "encoding": { + "y": {"axis": null}, + "x": {"stack": null}, + "color": { "legend": null}, + "opacity": {"value": 0.9}, + } + }], + "config": { + "view": {"stroke": null}, + "axis": {"grid": false} + }, + }, + "channels": ["x", "y", "color"], + "paths": { + "x": [["hconcat", 0, "encoding", "x"], ["hconcat", 1, "encoding", "x"]], + "y": [["hconcat", 0, "encoding", "y"], ["hconcat", 1, "encoding", "y"]], + "color": [["hconcat", 0, "encoding", "color"], ["hconcat", 1, "encoding", "color"]], + }, + "postProcessor": (vgSpec: any, table: any[]) => { + try { + if (table) { + let colorField = vgSpec['hconcat'][0]['encoding']['color']['field']; + let colorValues = [...new Set(table.map(r => r[colorField]))] ; + vgSpec.hconcat[0].transform = [{"filter": `datum[\"${colorField}\"] == \"${colorValues[0]}\"`}] + vgSpec.hconcat[0].title = colorValues[0] + vgSpec.hconcat[1].transform = [{"filter": `datum[\"${colorField}\"] == \"${colorValues[1]}\"`}] + vgSpec.hconcat[1].title = colorValues[1] + let xField = vgSpec['hconcat'][0]['encoding']['x']['field']; + let xValues = [...new Set(table.filter(r => r[colorField] == colorValues[0] || r[colorField] == colorValues[1]).map(r => r[xField]))]; + let domain = [Math.min(...xValues, 0), Math.max(...xValues)] + vgSpec.hconcat[0]['encoding']['x']['scale']['domain'] = domain; + vgSpec.hconcat[1]['encoding']['x']['scale'] = {domain: domain}; + } + } catch { + + } + return vgSpec; + } + }, { "chart": "Grouped Bar Chart", "icon": chartIconColumnGrouped, @@ -357,7 +413,7 @@ let tableCharts : ChartTemplate[] = [ }, "channels": ["x", "y", "color", "column", "row"], "paths": Object.fromEntries(["x", "y", "color", "column", "row"].map(channel => [channel, ["encoding", channel]])), - "postProcessor": (vgSpec: any) => { + "postProcessor": (vgSpec: any, table: any[]) => { if (vgSpec.encoding.y && vgSpec.encoding.y.type != "nominal") { vgSpec.encoding.y.type = "nominal"; } diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 66f83e07..a0a69953 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -128,7 +128,7 @@ export type ChartTemplate = { template: any, channels: string[], paths: { [key: string]: (string | number)[] | (string | number)[][]; }, - postProcessor?: (vgSpec: any) => any + postProcessor?: (vgSpec: any, table: any[]) => any } export const AGGR_OP_LIST = ["count", "sum", "average"] as const diff --git a/src/scss/EncodingShelf.scss b/src/scss/EncodingShelf.scss index 163203e6..c20f32e8 100644 --- a/src/scss/EncodingShelf.scss +++ b/src/scss/EncodingShelf.scss @@ -46,19 +46,17 @@ } } - .auto-sort-option-label { width: calc(100% - 24px); overflow: hidden; display: -webkit-box; - -webkit-line-clamp: 1; + line-clamp: 2; + -webkit-line-clamp: 2; -webkit-box-orient: vertical; background: #fff; text-align: left; - font-size: smaller; white-space: normal; - font-size: smaller !important; - font-style: italic; + font-size: xx-small !important; } .channel-shelf-box { diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 1241a851..186ed99d 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -17,6 +17,7 @@ import { Typography, Box, + Tooltip, } from '@mui/material'; @@ -31,12 +32,13 @@ import { DndProvider } from 'react-dnd' import { HTML5Backend } from 'react-dnd-html5-backend' import { SelectableGroup } from 'react-selectable-fast'; -import { TableSelectionDialog, TableURLDialog } from './TableSelectionView'; +import { TableCopyDialogV2, TableSelectionDialog, TableURLDialog } from './TableSelectionView'; import { TableCopyDialog, TableUploadDialog } from './TableSelectionView'; import { toolName } from '../app/App'; import { DataThread } from './DataThread'; import dfLogo from '../assets/df-logo.png'; +import exampleImageTable from "../assets/example-image-table.png"; import { ModelSelectionButton } from './ModelSelectionDialog'; const MainSplitPane = styled(SplitPane)(({ theme }) => ({ @@ -119,6 +121,17 @@ export const DataFormulatorFC = ({ }) => { ); + let exampleMessyText=`Rank NOC Gold Silver Bronze Total +1 South Korea 5 1 1 7 +2 France* 0 1 1 2 + United States 0 1 1 2 +4 China 0 1 0 1 + Germany 0 1 0 1 +6 Mexico 0 0 1 1 + Turkey 0 0 1 1 +Totals (7 entries) 5 5 5 15 +` + let dataUploadRequestBox = @@ -128,9 +141,13 @@ export const DataFormulatorFC = ({ }) => { Load data from - , , , or + , , or + + + Besides formatted data (csv, tsv or json), you can copy-paste  + Example of a messy text block: {exampleMessyText}}>a text block or  + Example of a table in image format: }>an image that contain data into clipboard to get started. - Select or upload datasets (in csv, tsv or json records format) to get started. ; diff --git a/src/views/EncodingBox.tsx b/src/views/EncodingBox.tsx index 59f4ea65..0d944c86 100644 --- a/src/views/EncodingBox.tsx +++ b/src/views/EncodingBox.tsx @@ -154,8 +154,13 @@ export const EncodingBox: FC = function EncodingBox({ channel, dispatch(dfActions.swapChartEncoding({chartId, channel1, channel2})) } - let handleUpdateEncoding = (channel: Channel, encoding: EncodingItem) => { - dispatch(dfActions.updateChartEncoding({chartId, channel, encoding})); + let handleResetEncoding = () => { + dispatch(dfActions.updateChartEncoding({chartId, channel, encoding: {bin: false}})); + } + + // updating a property of the encoding + let updateEncProp = (prop: keyof EncodingItem, value: any) => { + dispatch(dfActions.updateChartEncodingProp({chartId, channel, prop: prop as string, value})); } const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); @@ -167,7 +172,12 @@ export const EncodingBox: FC = function EncodingBox({ channel, const dispatch = useDispatch(); - useEffect(() => { setAutoSortResult(field?.levels) }, [encoding.fieldID, field]) + useEffect(() => { + setAutoSortResult(field?.levels); + if (field?.levels) { + updateEncProp('sortBy', JSON.stringify(field?.levels)); + } + }, [encoding.fieldID, field]) // make this a drop element for concepts const [{ canDrop, isOver }, drop] = useDrop(() => ({ @@ -175,7 +185,10 @@ export const EncodingBox: FC = function EncodingBox({ channel, drop: (item: any): EncodingDropResult => { if (item.type === "concept-card") { if (item.source === "conceptShelf") { - handleUpdateEncoding(channel, { 'fieldID': item.fieldID, bin: false }); + handleResetEncoding(); + updateEncProp('fieldID', item.fieldID); + updateEncProp('bin', false); + //handleUpdateEncoding(channel, { 'fieldID': item.fieldID, bin: false }); } else if (item.source === "encodingShelf") { handleSwapEncodingField(channel, item.channel); } else { @@ -204,10 +217,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, // items that control the editor panel popover const [editMode, setEditMode] = React.useState(false); - // updating a property of the encoding - let updateEncProp = (prop: keyof EncodingItem, value: any) => { - dispatch(dfActions.updateChartEncodingProp({chartId, channel, prop: prop as string, value})); - } const isActive = canDrop && isOver; let backgroundColor = ''; @@ -219,7 +228,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, let fieldComponent = field === undefined ? "" : ( { - handleUpdateEncoding(channel, { 'bin': false }); + handleResetEncoding(); }} /> ) @@ -231,31 +240,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, value={value} control={} label={label} /> } - let aggrOpt = [ - Aggregate, - - { updateEncProp("aggregate", event.target.value == "none" ? undefined : event.target.value as AggrOp); }} - > - {radioLabel("none", "none", `aggr--1`)} - {AGGR_OP_LIST.map((t, i) => radioLabel(t, t, `aggr-${i}`))} - - - ] - let stackOpt = (chart.chartType == "bar" || chart.chartType == "area") && (channel == "x" || channel == "y") ? [ Stack, = function EncodingBox({ channel, ] : []; - let binOpt = [ - Bin, - - { updateEncProp("bin", event.target.value == "on"); }} - > - {radioLabel("off", "off", `bin-radio-off`)} - {radioLabel("on", "on", `bin-radio-on`)} - - - ] - let domainItems = (field?.source == "custom" || field?.source == "original") ? getDomains(field as FieldItem, tables)[0] : []; if (field?.source == "derived") { domainItems = deriveTransformExamplesV2( @@ -374,129 +335,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, }); } - let autoSortBtn = - - let sortOptions = [radioLabel("↑ asc", "ascending", `sort-ascending`), radioLabel("↓ desc", "descending", `sort-descending`)] - let extraSortOptions = []; - - // TODO: check sort options - if (channel == "x" && (field?.type == "string" || field?.type == "auto")) { - - extraSortOptions.push(radioLabel("y↑ asc", "y", `sort-x-y-ascending`, 90)); - extraSortOptions.push(radioLabel("y↓ desc", "-y", `sort-x-y-descending`, 90)) - } - if (channel == "y" && (field?.type == "string" || field?.type == "auto")) { - extraSortOptions.push(radioLabel("x↑", "x", `sort-y-x-ascending`, 90)); - extraSortOptions.push(radioLabel("x↓", "-x", `sort-y-x-descending`, 90)) - } - if (extraSortOptions.length > 0) { - sortOptions = [ - radioLabel("↑ asc", "ascending", `sort-ascending`, 90), - radioLabel("↓ desc", "descending", `sort-descending`, 90), - ...extraSortOptions]; - } - - // if (autoSortEnabled) { - // if (autoSortResult != undefined && autoSortResult.length > 0) { - // let autoSortOpt = - // - // {autoSortResult.map(x => x ? x.toString() : 'null').join(", ")} - // ; - - // let autoSortOptReversed = - // - // {[...autoSortResult].reverse().map(x =>x ? x.toString() : 'null' ).join(", ")} - // ; - - // sortOptions = [ - // ...sortOptions, - // } - // label={autoSortOpt} />, - // } - // label={autoSortOptReversed} />, - // } - // label={autoSortBtn} /> - // ] - // } else { - // sortOptions = [ - // ...sortOptions, - // } - // label={autoSortBtn} /> - // ] - // } - // } - - let sortByFieldInputBox = { - console.log(`change: ${value}`) - }} - // value={tempValue} - filterOptions={(options, params) => { - const filtered = filter(options, params); - const { inputValue } = params; - // Suggest the creation of a new value - const isExisting = options.some((option) => inputValue === option); - if (inputValue !== '' && !isExisting) { - return [...filtered, `${inputValue}`, ] - } else { - return [...filtered]; - } - }} - sx={{ flexGrow: 1, flexShrink: 1, width: '120px', "& .MuiInput-input": { padding: "0px 8px !important"}}} - fullWidth - selectOnFocus - clearOnBlur - handleHomeEndKeys - autoHighlight - id="free-solo-with-text-demo" - options={conceptShelfItems.map(f => f.name).filter(name => name != "")} - getOptionLabel={(option) => { - // Value selected with enter, right from the input - return option; - }} - groupBy={(option) => { - let groupItem = conceptGroups.find(item => item.field.name == option); - if (groupItem && groupItem.field.name != "") { - return `from ${groupItem.group}`; - } else { - return "create a new concept" - } - }} - renderGroup={(params) => ( -
  • - {params.group} - {params.children} -
    • - )} - renderOption={(props, option) => { - let renderOption = (conceptShelfItems.map(f => f.name).includes(option) || option == "...") ? option : `"${option}"`; - let otherStyle = option == `...` ? {color: "darkgray"} : {} - - return { - //handleSelectOption(option); - }} sx={{fontSize: "small", ...otherStyle}}>{renderOption} - }} - freeSolo - renderInput={(params) => ( - - )} - /> - let sortByFieldID = encoding.fieldID - let sortByOptions = [ radioLabel("default", "default", `sort-by-default`) ] @@ -508,14 +346,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, sortByOptions.push(radioLabel("x values", "x", `sort-y-by-x-ascending`, 90)); } - // ***** sort by field option ***** - // sortByOptions = [ - // ...sortByOptions, - // } - // label={sortByFieldInputBox} /> - // ] - if (autoSortEnabled) { if (autoSortInferRunning) { sortByOptions = [ @@ -528,7 +358,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, } else { if (autoSortResult != undefined) { - let autoSortOptTitle = + let autoSortOptTitle = Sort Order: {autoSortResult.values.map(x => x ? x.toString() : 'null').join(", ")} @@ -576,7 +406,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, value={JSON.stringify(autoSortResult)} control={} label={} /> + onClick={autoSortFunction}>infer smart sort order} /> ] } } @@ -594,7 +424,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, row aria-labelledby="sort-option-radio-buttons-group" name="sort-option-radio-buttons-group" - value={encoding.sortBy || 'default'} + value={encoding.sortBy || 'default'} sx={{ width: 180 }} onChange={(event) => { updateEncProp("sortBy", event.target.value) }} > @@ -625,28 +455,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, ] - - - // let sortOpt = [ - // Sort By, - // - // { updateEncProp("sort", event.target.value) }} - // > - // {sortOptions} - // - // , - // ] let colorSchemeList = [ "category10", @@ -688,48 +496,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, margin: '0px 12px', padding: "6px", fontSize: "12px" }} > - {/* :not(style)': { margin: "0px", }, }} - noValidate - autoComplete="off"> - - Data Field - - - */} - {/* :not(style)': { margin: "4px", }, }} - noValidate - autoComplete="off"> - - Aggregate - - - */} - {/* {aggrOpt} - {binOpt} */} {stackOpt} {sortByOpt} {sortOrderOpt} diff --git a/src/views/EncodingShelfCard.tsx b/src/views/EncodingShelfCard.tsx index b31ebb09..cb238de2 100644 --- a/src/views/EncodingShelfCard.tsx +++ b/src/views/EncodingShelfCard.tsx @@ -275,7 +275,7 @@ export const EncodingShelfCard: FC = function ({ chartId extra_prompt: instruction, model: activeModel }) - let engine = betaMode ? getUrls().SERVER_DERIVE_DATA_V2_URL : getUrls().SERVER_DERIVE_DATA_URL; + let engine = getUrls().SERVER_DERIVE_DATA_URL; if (mode == "formulate" && currentTable.derive?.dialog) { messageBody = JSON.stringify({ diff --git a/src/views/TableSelectionView.tsx b/src/views/TableSelectionView.tsx index 20ae0ebf..e9a23bbf 100644 --- a/src/views/TableSelectionView.tsx +++ b/src/views/TableSelectionView.tsx @@ -2,12 +2,16 @@ // Licensed under the MIT License. import * as React from 'react'; +import validator from 'validator'; +import DOMPurify from 'dompurify'; import Tabs from '@mui/material/Tabs'; import Tab from '@mui/material/Tab'; import Typography from '@mui/material/Typography'; import Box from '@mui/material/Box'; -import { Button, Dialog, DialogActions, DialogContent, DialogTitle, IconButton, Input, Paper, TextField } from '@mui/material'; +import { alpha, Button, Collapse, Dialog, DialogActions, DialogContent, DialogTitle, Divider, + IconButton, Input, CircularProgress, LinearProgress, Paper, TextField, useTheme, + Card} from '@mui/material'; import { CustomReactTable } from './ReactTable'; import { DictTable } from "../components/ComponentType"; @@ -16,8 +20,17 @@ import { getUrls } from '../app/utils'; import { createTableFromFromObjectArray, createTableFromText, loadDataWrapper } from '../data/utils'; import CloseIcon from '@mui/icons-material/Close'; -import { dfActions, fetchFieldSemanticType } from '../app/dfSlice'; -import { useDispatch } from 'react-redux'; +import KeyboardReturnIcon from '@mui/icons-material/KeyboardReturn'; +import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import AutoFixNormalIcon from '@mui/icons-material/AutoFixNormal'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import CancelIcon from '@mui/icons-material/Cancel'; + +import ReactDiffViewer from 'react-diff-viewer' + +import { dfActions, dfSelectors, fetchFieldSemanticType } from '../app/dfSlice'; +import { useDispatch, useSelector } from 'react-redux'; import { useState } from 'react'; import { AppDispatch } from '../app/store'; @@ -393,4 +406,315 @@ export const TableURLDialog: React.FC = ({ buttonElement, d {dialog} ; -} \ No newline at end of file +} + + +export const TableCopyDialogV2: React.FC = ({ buttonElement, disabled }) => { + + let activeModel = useSelector(dfSelectors.getActiveModel); + + const [dialogOpen, setDialogOpen] = useState(false); + const [tableName, setTableName] = useState(""); + + const [tableContent, setTableContent] = useState(""); + const [tableContentType, setTableContentType] = useState<'text' | 'image'>('text'); + + const [cleaningInProgress, setCleaningInProgress] = useState(false); + const [cleanTableContent, setCleanTableContent] = useState<{content: string, reason: string, mode: string} | undefined>(undefined); + + let viewTable = cleanTableContent == undefined ? undefined : createTableFromText(tableName || "clean-table", cleanTableContent.content) + + + const [loadFromURL, setLoadFromURL] = useState(false); + const [url, setURL] = useState(""); + + let theme = useTheme() + + const dispatch = useDispatch(); + + let handleSubmitContent = (tableStr: string): void => { + let table : undefined | DictTable = undefined; + try { + let content = JSON.parse(tableStr); + table = createTableFromFromObjectArray(tableName || 'data-0', content); + } catch (error) { + table = createTableFromText(tableName || 'data-0', tableStr); + } + if (table) { + dispatch(dfActions.addTable(table)); + dispatch(fetchFieldSemanticType(table)); + } + }; + + let handleLoadURL = () => { + console.log("hello hello") + setLoadFromURL(!loadFromURL); + + let parts = url.split('/'); + + // Get the last part of the URL, which should be the file name with extension + const tableName = parts[parts.length - 1]; + + fetch(url) + .then(res => res.text()) + .then(content => { + setTableName(tableName); + setTableContent(content); + setTableContentType("text"); + }) + } + + let handleCleanData = () => { + //setCleanTableContent("hehehao\n\n" + tableContent); + let token = String(Date.now()); + setCleaningInProgress(true); + setCleanTableContent(undefined); + let message = { + method: 'POST', + headers: { 'Content-Type': 'application/json', }, + body: JSON.stringify({ + token: token, + content_type: tableContentType, + raw_data: tableContent, + model: activeModel + }), + }; + + fetch(getUrls().CLEAN_DATA_URL, message) + .then((response) => response.json()) + .then((data) => { + setCleaningInProgress(false); + console.log(data); + console.log(token); + + if (data["status"] == "ok") { + if (data["token"] == token) { + let candidate = data["result"][0]; + console.log(candidate) + + let cleanContent = candidate['content']; + let info = candidate['info']; + + setCleanTableContent({content: cleanContent.trim(), reason: info['reason'], mode: info['mode']}); + console.log(`data cleaning reason:`) + console.log(info); + } + } else { + // TODO: add warnings to show the user + dispatch(dfActions.addMessages({ + "timestamp": Date.now(), + "type": "error", + "value": "unable to perform auto-sort." + })); + setCleanTableContent(undefined); + } + }).catch((error) => { + setCleaningInProgress(false); + setCleanTableContent(undefined); + + dispatch(dfActions.addMessages({ + "timestamp": Date.now(), + "type": "error", + "value": "unable to perform clean data due to server issue." + })); + }); + } + + const newStyles = { + variables: { }, + line: { + '&:hover': { + background: alpha(theme.palette.primary.main, 0.2), + }, + }, + titleBlock: { + padding: '4px 8px', + borderBottom: 'none' + }, + marker: { + width: 'fit-content' + }, + content: { + fontSize: 12, + width: 'fit-content', + maxWidth: "50%", + minWidth: 300, + }, + diffContainer: { + "pre": { lineHeight: 1.2, fontFamily: 'sans-serif' } + }, + contentText: { + + }, + gutter: { + minWidth: '12px', + fontSize: 12, + padding: '0 8px', + } + }; + + let renderLines = (str: string) => ( + {str} + ); + + let dialog = {setDialogOpen(false)}} open={dialogOpen} + sx={{ '& .MuiDialog-paper': { maxWidth: '80%', maxHeight: 800, minWidth: 800 } }} + > + Paste & Upload Data + { setDialogOpen(false); }} + aria-label="close" + > + + + + + + { setTableName(event.target.value); }} + autoComplete='off' id="outlined-basic" label="dataset name" variant="outlined" /> + + + + + { setURL(event.target.value); }} + onKeyDown={(event)=> { + if(event.key == 'Enter'){ + handleLoadURL(); + event.preventDefault(); + } + }} + autoComplete='off' id="outlined-basic" label="url" variant="outlined" /> + + + + + + {cleaningInProgress && tableContentType == "text" ? : ""} + {viewTable ? + <> + {/* */} + { + return { id: name, label: name, minWidth: 60, align: undefined, format: (v: any) => v} + })} + /> + {/* {cleanTableContent.reason} */} + + : ( tableContentType == "text" ? + 1000 ? 12 : 14, lineHeight: 1.2 }}} + id="upload content" value={tableContent} maxRows={30} + onChange={(event) => { + setTableContent(event.target.value); + }} + InputLabelProps={{ shrink: true }} + placeholder="Paste data (in csv, tsv, or json format), or a text snippet / an image that contains data to get started." + onPasteCapture={(e) => { + console.log(e.clipboardData.files); + if (e.clipboardData.files.length > 0) { + let file = e.clipboardData.files[0]; + let read = new FileReader(); + + read.readAsDataURL(file); + + read.onloadend = function(){ + let res = read.result; + console.log(res); + if (res) { + setTableContent(res as string); + setTableContentType("image"); + } + } + } + }} + autoComplete='off' + label="data content" variant="outlined" multiline minRows={15} + /> + : + + {cleaningInProgress ? : ""} + { + setTableContent(""); + setTableContentType("text"); + }} + > + + + {validator.isURL(tableContent) || validator.isDataURI(tableContent) ? ( + the image is corrupted, please try again. + ) : ( + Invalid image data + )} + ) + } + + + + { cleanTableContent != undefined ? + + + + : } + {/* + + */} + + + + ; + + return <> + + {dialog} + ; +} diff --git a/vite.config.ts b/vite.config.ts index 281f9458..f4886165 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ }, }, build: { + outDir: path.join(__dirname, 'py-src', 'data_formulator', "dist"), rollupOptions: { output: { entryFileNames: `DataFormulator.js`, // specific name for the main JS bundle diff --git a/yarn.lock b/yarn.lock index 3277978c..12ed524a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9,6 +9,32 @@ dependencies: "@babel/highlight" "^7.22.5" +"@babel/code-frame@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" + integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA== + dependencies: + "@babel/highlight" "^7.24.7" + picocolors "^1.0.0" + +"@babel/generator@^7.25.6": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.25.6.tgz#0df1ad8cb32fe4d2b01d8bf437f153d19342a87c" + integrity sha512-VPC82gr1seXOpkjAAKoLhP50vx4vGNlF4msF64dSFq1P8RfB+QAuJWGHPXXPc8QyfVWwwB/TNNU4+ayZmHNbZw== + dependencies: + "@babel/types" "^7.25.6" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^2.5.1" + +"@babel/helper-module-imports@^7.0.0": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" + integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + "@babel/helper-module-imports@^7.16.7": version "7.16.7" resolved "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.16.7.tgz" @@ -21,11 +47,21 @@ resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== +"@babel/helper-string-parser@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz#5b3329c9a58803d5df425e5785865881a81ca48d" + integrity sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ== + "@babel/helper-validator-identifier@^7.22.5": version "7.22.5" resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ== +"@babel/helper-validator-identifier@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" + integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== + "@babel/highlight@^7.22.5": version "7.22.5" resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.5.tgz#aa6c05c5407a67ebce408162b7ede789b4d22031" @@ -35,6 +71,23 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d" + integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.25.0", "@babel/parser@^7.25.6": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.25.6.tgz#85660c5ef388cbbf6e3d2a694ee97a38f18afe2f" + integrity sha512-trGdfBdbD0l1ZPmcJ83eNxB9rbEax4ALFTF7fN386TMYbeCQbyme5cOEXQhbGXKebwGaB/J52w1mrklMcbgy6Q== + dependencies: + "@babel/types" "^7.25.6" + "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.9.2": version "7.17.7" resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.7.tgz" @@ -49,6 +102,35 @@ dependencies: regenerator-runtime "^0.13.11" +"@babel/runtime@^7.7.2": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.25.6.tgz#9afc3289f7184d8d7f98b099884c26317b9264d2" + integrity sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ== + dependencies: + regenerator-runtime "^0.14.0" + +"@babel/template@^7.25.0": + version "7.25.0" + resolved "https://registry.npmjs.org/@babel/template/-/template-7.25.0.tgz#e733dc3134b4fede528c15bc95e89cb98c52592a" + integrity sha512-aOOgh1/5XzKvg1jvVz7AVrx2piJ2XBi227DHmbY6y+bM9H2FlN+IfecYu4Xl0cNiiVejlsCri89LUsbj8vJD9Q== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/parser" "^7.25.0" + "@babel/types" "^7.25.0" + +"@babel/traverse@^7.24.7": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.6.tgz#04fad980e444f182ecf1520504941940a90fea41" + integrity sha512-9Vrcx5ZW6UwK5tvqsj0nGpp/XzqthkT0dqIc9g1AdtygFToNtTF67XzYS//dm+SAK9cp3B9R4ZO/46p63SCjlQ== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.25.6" + "@babel/parser" "^7.25.6" + "@babel/template" "^7.25.0" + "@babel/types" "^7.25.6" + debug "^4.3.1" + globals "^11.1.0" + "@babel/types@^7.16.7": version "7.22.5" resolved "https://registry.npmjs.org/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe" @@ -58,6 +140,15 @@ "@babel/helper-validator-identifier" "^7.22.5" to-fast-properties "^2.0.0" +"@babel/types@^7.24.7", "@babel/types@^7.25.0", "@babel/types@^7.25.6": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/types/-/types-7.25.6.tgz#893942ddb858f32ae7a004ec9d3a76b3463ef8e6" + integrity sha512-/l42B1qxpG6RdfYf343Uw1vmDjeNhneUXtzhojE7pDgfpEypmRhI6j1kr17XCVv4Cgl9HdAiQY2x0GwKm7rWCw== + dependencies: + "@babel/helper-string-parser" "^7.24.8" + "@babel/helper-validator-identifier" "^7.24.7" + to-fast-properties "^2.0.0" + "@emotion/babel-plugin@^11.11.0": version "11.11.0" resolved "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz#c2d872b6a7767a9d176d007f5b31f7d504bb5d6c" @@ -75,6 +166,16 @@ source-map "^0.5.7" stylis "4.2.0" +"@emotion/cache@^10.0.27": + version "10.0.29" + resolved "https://registry.npmjs.org/@emotion/cache/-/cache-10.0.29.tgz#87e7e64f412c060102d589fe7c6dc042e6f9d1e0" + integrity sha512-fU2VtSVlHiF27empSbxi1O2JFdNWZO+2NFHfwO0pxgTep6Xa3uGb+3pVKfLww2l/IBGLNEZl5Xf/++A4wAYDYQ== + dependencies: + "@emotion/sheet" "0.9.4" + "@emotion/stylis" "0.8.5" + "@emotion/utils" "0.11.3" + "@emotion/weak-memoize" "0.2.5" + "@emotion/cache@^11.11.0": version "11.11.0" resolved "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz#809b33ee6b1cb1a625fef7a45bc568ccd9b8f3ff" @@ -86,6 +187,11 @@ "@emotion/weak-memoize" "^0.3.1" stylis "4.2.0" +"@emotion/hash@0.8.0": + version "0.8.0" + resolved "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" + integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== + "@emotion/hash@^0.9.1": version "0.9.1" resolved "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz#4ffb0055f7ef676ebc3a5a91fb621393294e2f43" @@ -98,6 +204,11 @@ dependencies: "@emotion/memoize" "^0.8.1" +"@emotion/memoize@0.7.4": + version "0.7.4" + resolved "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.7.4.tgz#19bf0f5af19149111c40d98bb0cf82119f5d9eeb" + integrity sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw== + "@emotion/memoize@^0.8.1": version "0.8.1" resolved "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz#c1ddb040429c6d21d38cc945fe75c818cfb68e17" @@ -117,6 +228,17 @@ "@emotion/weak-memoize" "^0.3.1" hoist-non-react-statics "^3.3.1" +"@emotion/serialize@^0.11.15", "@emotion/serialize@^0.11.16": + version "0.11.16" + resolved "https://registry.npmjs.org/@emotion/serialize/-/serialize-0.11.16.tgz#dee05f9e96ad2fb25a5206b6d759b2d1ed3379ad" + integrity sha512-G3J4o8by0VRrO+PFeSc3js2myYNOXVJ3Ya+RGVxnshRYgsvErfAOglKAiy1Eo1vhzxqtUvjCyS5gtewzkmvSSg== + dependencies: + "@emotion/hash" "0.8.0" + "@emotion/memoize" "0.7.4" + "@emotion/unitless" "0.7.5" + "@emotion/utils" "0.11.3" + csstype "^2.5.7" + "@emotion/serialize@^1.1.2": version "1.1.2" resolved "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.2.tgz#017a6e4c9b8a803bd576ff3d52a0ea6fa5a62b51" @@ -128,6 +250,11 @@ "@emotion/utils" "^1.2.1" csstype "^3.0.2" +"@emotion/sheet@0.9.4": + version "0.9.4" + resolved "https://registry.npmjs.org/@emotion/sheet/-/sheet-0.9.4.tgz#894374bea39ec30f489bbfc3438192b9774d32e5" + integrity sha512-zM9PFmgVSqBw4zL101Q0HrBVTGmpAxFZH/pYx/cjJT5advXguvcgjHFTCaIO3enL/xr89vK2bh0Mfyj9aa0ANA== + "@emotion/sheet@^1.2.2": version "1.2.2" resolved "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz#d58e788ee27267a14342303e1abb3d508b6d0fec" @@ -145,6 +272,16 @@ "@emotion/use-insertion-effect-with-fallbacks" "^1.0.1" "@emotion/utils" "^1.2.1" +"@emotion/stylis@0.8.5": + version "0.8.5" + resolved "https://registry.npmjs.org/@emotion/stylis/-/stylis-0.8.5.tgz#deacb389bd6ee77d1e7fcaccce9e16c5c7e78e04" + integrity sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ== + +"@emotion/unitless@0.7.5": + version "0.7.5" + resolved "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.7.5.tgz#77211291c1900a700b8a78cfafda3160d76949ed" + integrity sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg== + "@emotion/unitless@^0.8.1": version "0.8.1" resolved "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz#182b5a4704ef8ad91bde93f7a860a88fd92c79a3" @@ -155,11 +292,21 @@ resolved "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz#08de79f54eb3406f9daaf77c76e35313da963963" integrity sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw== +"@emotion/utils@0.11.3": + version "0.11.3" + resolved "https://registry.npmjs.org/@emotion/utils/-/utils-0.11.3.tgz#a759863867befa7e583400d322652a3f44820924" + integrity sha512-0o4l6pZC+hI88+bzuaX/6BgOvQVhbt2PfmxauVaYOGgbsAw14wdKyvMCZXnsnsHys94iadcF+RG/wZyx6+ZZBw== + "@emotion/utils@^1.2.1": version "1.2.1" resolved "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz#bbab58465738d31ae4cb3dbb6fc00a5991f755e4" integrity sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg== +"@emotion/weak-memoize@0.2.5": + version "0.2.5" + resolved "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz#8eed982e2ee6f7f4e44c253e12962980791efd46" + integrity sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA== + "@emotion/weak-memoize@^0.3.1": version "0.3.1" resolved "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" @@ -285,6 +432,38 @@ resolved "https://registry.npmjs.org/@fontsource/roboto/-/roboto-4.5.5.tgz" integrity sha512-Pe1p+gAO6K0aLxBXlLoJRHVx352tVc/v/7DOnvM3t+FYXb+KUga9aCD1NpnDfd0kKnWXqrZyAXguyyFWDDuphw== +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.5.0" + resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a" + integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== + +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@mui/base@5.0.0-beta.7": version "5.0.0-beta.7" resolved "https://registry.npmjs.org/@mui/base/-/base-5.0.0-beta.7.tgz#01cb99ac098af0ba989c7abc1474e3291c29414f" @@ -962,6 +1141,31 @@ array-flat-polyfill@^1.0.1: resolved "https://registry.npmjs.org/array-flat-polyfill/-/array-flat-polyfill-1.0.1.tgz" integrity sha512-hfJmKupmQN0lwi0xG6FQ5U8Rd97RnIERplymOv/qpq8AoNKPPAnxJadjFA23FNWm88wykh9HmpLJUUwUtNU/iw== +babel-plugin-emotion@^10.0.27: + version "10.2.2" + resolved "https://registry.npmjs.org/babel-plugin-emotion/-/babel-plugin-emotion-10.2.2.tgz#a1fe3503cff80abfd0bdda14abd2e8e57a79d17d" + integrity sha512-SMSkGoqTbTyUTDeuVuPIWifPdUGkTk1Kf9BWRiXIOIcuyMfsdp2EjeiiFvOzX8NOBvEh/ypKYvUh2rkgAJMCLA== + dependencies: + "@babel/helper-module-imports" "^7.0.0" + "@emotion/hash" "0.8.0" + "@emotion/memoize" "0.7.4" + "@emotion/serialize" "^0.11.16" + babel-plugin-macros "^2.0.0" + babel-plugin-syntax-jsx "^6.18.0" + convert-source-map "^1.5.0" + escape-string-regexp "^1.0.5" + find-root "^1.1.0" + source-map "^0.5.7" + +babel-plugin-macros@^2.0.0: + version "2.8.0" + resolved "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-2.8.0.tgz#0f958a7cc6556b1e65344465d99111a1e5e10138" + integrity sha512-SEP5kJpfGYqYKpBrj5XU3ahw5p5GOHJ0U5ssOSQ/WBVdwkD2Dzlce95exQTs3jOVWPPKLBN2rlEWkCK7dSmLvg== + dependencies: + "@babel/runtime" "^7.7.2" + cosmiconfig "^6.0.0" + resolve "^1.12.0" + babel-plugin-macros@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz" @@ -971,6 +1175,11 @@ babel-plugin-macros@^3.1.0: cosmiconfig "^7.0.0" resolve "^1.19.0" +babel-plugin-syntax-jsx@^6.18.0: + version "6.18.0" + resolved "https://registry.npmjs.org/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz#0af32a9a6e13ca7a3fd5069e62d7b0f58d0d8946" + integrity sha512-qrPaCSo9c8RHNRHIotaufGbuOBN8rtdC4QrrFFc43vyWCCz7Kl7GL1PGaXtMGQZUXrkCjNEgxDfmAuAabr/rlw== + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" @@ -988,7 +1197,7 @@ callsites@^3.0.0: resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -chalk@^2.0.0: +chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -1012,6 +1221,11 @@ chalk@^2.0.0: optionalDependencies: fsevents "~2.3.2" +classnames@^2.2.6: + version "2.5.1" + resolved "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz#ba774c614be0f016da105c858e7159eae8e7687b" + integrity sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow== + classnames@^2.3.1: version "2.3.2" resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.2.tgz#351d813bf0137fcc6a76a16b88208d2560a0d924" @@ -1080,6 +1294,17 @@ convert-source-map@^1.5.0: resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== +cosmiconfig@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982" + integrity sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg== + dependencies: + "@types/parse-json" "^4.0.0" + import-fresh "^3.1.0" + parse-json "^5.0.0" + path-type "^4.0.0" + yaml "^1.7.2" + cosmiconfig@^7.0.0: version "7.0.1" resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.1.tgz" @@ -1091,6 +1316,21 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +create-emotion@^10.0.14, create-emotion@^10.0.27: + version "10.0.27" + resolved "https://registry.npmjs.org/create-emotion/-/create-emotion-10.0.27.tgz#cb4fa2db750f6ca6f9a001a33fbf1f6c46789503" + integrity sha512-fIK73w82HPPn/RsAij7+Zt8eCE8SptcJ3WoRMfxMtjteYxud8GDTKKld7MYwAX2TVhrw29uR1N/bVGxeStHILg== + dependencies: + "@emotion/cache" "^10.0.27" + "@emotion/serialize" "^0.11.15" + "@emotion/sheet" "0.9.4" + "@emotion/utils" "0.11.3" + +csstype@^2.5.7: + version "2.6.21" + resolved "https://registry.npmjs.org/csstype/-/csstype-2.6.21.tgz#2efb85b7cc55c80017c66a5ad7cbd931fda3a90e" + integrity sha512-Z1PhmomIfypOpoMjRQB70jfvy/wxT50qW08YXO5lMIJkrdq4yOTR+AW7FqutScmB9NkLwxo+jU+kZLbofZZq/w== + csstype@^3.0.2, csstype@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b" @@ -1395,6 +1635,13 @@ d3@^7.3.0: d3-transition "3" d3-zoom "3" +debug@^4.3.1: + version "4.3.7" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + delaunator@5: version "5.0.0" resolved "https://registry.npmjs.org/delaunator/-/delaunator-5.0.0.tgz" @@ -1402,6 +1649,11 @@ delaunator@5: dependencies: robust-predicates "^3.0.0" +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + dnd-core@^16.0.1: version "16.0.1" resolved "https://registry.npmjs.org/dnd-core/-/dnd-core-16.0.1.tgz" @@ -1419,11 +1671,24 @@ dom-helpers@^5.0.1: "@babel/runtime" "^7.8.7" csstype "^3.0.2" +dompurify@^3.1.7: + version "3.1.7" + resolved "https://registry.npmjs.org/dompurify/-/dompurify-3.1.7.tgz#711a8c96479fb6ced93453732c160c3c72418a6a" + integrity sha512-VaTstWtsneJY8xzy7DekmYWEOZcmzIe3Qb3zPd4STve1OBTa+e+WmS1ITQec1fZYXI3HCsOZZiSMpG6oxoWMWQ== + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +emotion@^10.0.14: + version "10.0.27" + resolved "https://registry.npmjs.org/emotion/-/emotion-10.0.27.tgz#f9ca5df98630980a23c819a56262560562e5d75e" + integrity sha512-2xdDzdWWzue8R8lu4G76uWX5WhyQuzATon9LmNeCy/2BHVC6dsEpfhN1a0qhELgtDVdjyEA6J8Y/VlI5ZnaH0g== + dependencies: + babel-plugin-emotion "^10.0.27" + create-emotion "^10.0.27" + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -1512,6 +1777,11 @@ function-bind@^1.1.1: resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" @@ -1524,6 +1794,11 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + has-flag@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -1536,6 +1811,13 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2: version "3.3.2" resolved "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" @@ -1565,7 +1847,7 @@ immutable@^4.0.0: resolved "https://registry.yarnpkg.com/immutable/-/immutable-4.3.6.tgz#6a05f7858213238e587fb83586ffa3b4b27f0447" integrity sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ== -import-fresh@^3.2.1: +import-fresh@^3.1.0, import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -1590,6 +1872,13 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" +is-core-module@^2.13.0: + version "2.15.1" + resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" + integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== + dependencies: + hasown "^2.0.2" + is-core-module@^2.8.1: version "2.12.1" resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz#0c0b6885b6f80011c71541ce15c8d66cf5a4f9fd" @@ -1624,6 +1913,11 @@ is-number@^7.0.0: resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" @@ -1670,6 +1964,16 @@ markdown-to-jsx@^7.1.8: resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-7.1.8.tgz#49c3bb3c122aa714324034142c8829b93c889338" integrity sha512-rRSa1aFmFnpDRFAhv5vIkWM4nPaoB9vnzIjuIKa1wGupfn2hdCNeaQHKpu4/muoc8n8J7yowjTP2oncA4/Rbgg== +memoize-one@^5.0.4: + version "5.2.1" + resolved "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" + integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q== + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + mui-markdown@^1.1.13: version "1.1.13" resolved "https://registry.npmjs.org/mui-markdown/-/mui-markdown-1.1.13.tgz#749b3c77379924985c016c907dca1e5423bf9590" @@ -1726,6 +2030,11 @@ path-type@^4.0.0: resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +picocolors@^1.0.0: + version "1.1.0" + resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz#5358b76a78cde483ba5cef6a9dc9671440b27d59" + integrity sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw== + picocolors@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" @@ -1789,6 +2098,18 @@ react-animate-on-change@^2.2.0: resolved "https://registry.npmjs.org/react-animate-on-change/-/react-animate-on-change-2.2.0.tgz" integrity sha512-cM0YHbsxIh8fshX/U24+pk4nDG7Ike9NsEy21reqJPqVt6xRA+6oYkaQHEggINKjYEMbztwK40Ro0/EHZ5naVQ== +react-diff-viewer@^3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/react-diff-viewer/-/react-diff-viewer-3.1.1.tgz#21ac9c891193d05a3734bfd6bd54b107ee6d46cc" + integrity sha512-rmvwNdcClp6ZWdS11m1m01UnBA4OwYaLG/li0dB781e/bQEzsGyj+qewVd6W5ztBwseQ72pO7nwaCcq5jnlzcw== + dependencies: + classnames "^2.2.6" + create-emotion "^10.0.14" + diff "^4.0.1" + emotion "^10.0.14" + memoize-one "^5.0.4" + prop-types "^15.6.2" + react-dnd-html5-backend@^16.0.1: version "16.0.1" resolved "https://registry.npmjs.org/react-dnd-html5-backend/-/react-dnd-html5-backend-16.0.1.tgz" @@ -1944,6 +2265,11 @@ regenerator-runtime@^0.13.11, regenerator-runtime@^0.13.4: resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" @@ -1959,6 +2285,15 @@ resolve-from@^4.0.0: resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== +resolve@^1.12.0: + version "1.22.8" + resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^1.19.0: version "1.22.0" resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz" @@ -2126,6 +2461,11 @@ use-sync-external-store@^1.0.0: resolved "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz" integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== +validator@^13.12.0: + version "13.12.0" + resolved "https://registry.npmjs.org/validator/-/validator-13.12.0.tgz#7d78e76ba85504da3fee4fd1922b385914d4b35f" + integrity sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg== + vega-canvas@^1.2.6: version "1.2.6" resolved "https://registry.npmjs.org/vega-canvas/-/vega-canvas-1.2.6.tgz" @@ -2630,7 +2970,7 @@ y18n@^5.0.5: resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yaml@^1.10.0: +yaml@^1.10.0, yaml@^1.7.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==