From 4f44f84334e5b1f5c325873e8959d7d38abad784 Mon Sep 17 00:00:00 2001 From: rchiodo Date: Thu, 18 Apr 2019 10:08:26 -0700 Subject: [PATCH 01/60] Testing codemirror's autocomplete --- package-lock.json | 12 ++++---- package.json | 1 + src/datascience-ui/history-react/code.tsx | 34 +++++++++++++++++++++-- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8355c440c6f1..3a7449027a1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1310,9 +1310,9 @@ "dev": true }, "@types/codemirror": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@types/codemirror/-/codemirror-0.0.71.tgz", - "integrity": "sha512-b2oEEnno1LIGKMR7uBEsr40al1UijF1HEpRn0+Yf1xOLl24iQgB7DBpZVMM7y54G5wCNoclDrRO65E6KHPNO2w==", + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@types/codemirror/-/codemirror-0.0.74.tgz", + "integrity": "sha512-pjc14HE6KCCk3SYoC49k/gJJHkredJy4GdSK+lGRMIBfG+Uq6vT8oZiKToFH2J7pZ5pUd69UGD6CgmFG1V17wA==", "dev": true, "requires": { "@types/tern": "*" @@ -1677,9 +1677,9 @@ } }, "@types/tern": { - "version": "0.22.1", - "resolved": "https://registry.npmjs.org/@types/tern/-/tern-0.22.1.tgz", - "integrity": "sha512-CRzPRkg8hYLwunsj61r+rqPJQbiCIEQqlMMY/0k7krgIsoSaFgGg1ZH2f9qaR1YpenaMl6PnlTtUkCbNH/uo+A==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/@types/tern/-/tern-0.23.2.tgz", + "integrity": "sha512-OtEnzdjKJqQ5URz4DKvVkHDiPcjXKXc+NbqXsFb3rughaN4R7JVtoSRAvuoiDIA4Ev4mxX3u56WvPBpEac5ULA==", "dev": true, "requires": { "@types/estree": "*" diff --git a/package.json b/package.json index 945f2e7c0f74..b17e89327299 100644 --- a/package.json +++ b/package.json @@ -2188,6 +2188,7 @@ "@types/chai": "^4.1.2", "@types/chai-arrays": "^1.0.2", "@types/chai-as-promised": "^7.1.0", + "@types/codemirror": "0.0.74", "@types/copy-webpack-plugin": "^4.4.2", "@types/del": "^3.0.0", "@types/diff-match-patch": "^1.0.32", diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index 010ba46fcba8..d5d75ef9fe74 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -9,6 +9,10 @@ import * as CodeMirror from 'codemirror'; import * as React from 'react'; import * as RCM from 'react-codemirror'; +// tslint:disable-next-line: no-require-imports no-var-requires +require('codemirror/addon/hint/show-hint'); +import 'codemirror/addon/hint/show-hint.css'; + import './code.css'; import { getLocString } from '../react-common/locReactSide'; @@ -38,6 +42,13 @@ interface ICodeState { allowWatermark: boolean; } +interface ICompletionResult { + list: string[]; + from: CodeMirror.Position; + to: CodeMirror.Position; + selectedHint?: number; +} + export class Code extends React.Component { private codeMirror: CodeMirror.Editor | undefined; @@ -96,13 +107,16 @@ export class Code extends React.Component { Down: this.arrowDown, Enter: this.enter, 'Shift-Enter': this.shiftEnter, - Up: this.arrowUp + Up: this.arrowUp, + 'Ctrl-Space': 'autocomplete' }, theme: `${this.props.codeTheme} default`, mode: 'python', cursorBlinkRate: -1, readOnly: readOnly ? true : false, - lineWrapping: true + lineWrapping: true, + hintOptions: { hint: this.provideHint } + // onKeyEvent could be used to provide autocomplete on say '.' for key up } } ref={this.updateCodeMirror} @@ -302,4 +316,20 @@ export class Code extends React.Component { private onChange = (_newValue: string, _change: CodeMirror.EditorChange) => { this.setState({allowWatermark: false}); } + + // tslint:disable-next-line:no-any + private provideHint = async (instance: CodeMirror.Editor, _option: any) : Promise => { + const doc = instance.getDoc(); + const cursor = doc ? doc.getCursor() : undefined; + if (cursor) { + return { + list: ['foo', 'bar', 'baz'], + from: cursor, + to: cursor + }; + } + + return null; + } + } From 20f721d80e24f97afbc211799ba35f83c9b8e6d7 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 19 Apr 2019 10:53:40 -0700 Subject: [PATCH 02/60] Remember monacoEditor pieces --- package-lock.json | 43 +++++-- package.json | 1 + src/datascience-ui/history-react/code.css | 26 ++++ .../react-common/inlineMonacoEditor.tsx | 112 ++++++++++++++++++ 4 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 src/datascience-ui/react-common/inlineMonacoEditor.tsx diff --git a/package-lock.json b/package-lock.json index 3a7449027a1f..c1004e75c750 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1571,14 +1571,12 @@ "@types/prop-types": { "version": "15.5.6", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.5.6.tgz", - "integrity": "sha512-ZBFR7TROLVzCkswA3Fmqq+IIJt62/T7aY/Dmz+QkU7CaW2QFqAitCE8Ups7IzmGhcN1YWMBT4Qcoc07jU9hOJQ==", - "dev": true + "integrity": "sha512-ZBFR7TROLVzCkswA3Fmqq+IIJt62/T7aY/Dmz+QkU7CaW2QFqAitCE8Ups7IzmGhcN1YWMBT4Qcoc07jU9hOJQ==" }, "@types/react": { "version": "16.4.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-16.4.14.tgz", "integrity": "sha512-Gh8irag2dbZ2K6vPn+S8+LNrULuG3zlCgJjVUrvuiUK7waw9d9CFk2A/tZFyGhcMDUyO7tznbx1ZasqlAGjHxA==", - "dev": true, "requires": { "@types/prop-types": "*", "csstype": "^2.2.0" @@ -4399,8 +4397,7 @@ "csstype": { "version": "2.5.7", "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.5.7.tgz", - "integrity": "sha512-Nt5VDyOTIIV4/nRFswoCKps1R5CD1hkiyjBE9/thNaNZILLEviVw9yWQw15+O+CpNjQKB/uvdcxFFOrSflY3Yw==", - "dev": true + "integrity": "sha512-Nt5VDyOTIIV4/nRFswoCKps1R5CD1hkiyjBE9/thNaNZILLEviVw9yWQw15+O+CpNjQKB/uvdcxFFOrSflY3Yw==" }, "currently-unhandled": { "version": "0.4.1", @@ -9932,7 +9929,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", - "dev": true, "requires": { "js-tokens": "^3.0.0 || ^4.0.0" } @@ -10598,6 +10594,11 @@ "resolved": "http://registry.npmjs.org/moment/-/moment-2.21.0.tgz", "integrity": "sha512-TCZ36BjURTeFTM/CwRcViQlfkMvL1/vFISuNLO5GkcVm1+QHfbSiNqZuWeMFjj1/3+uAjXswgRk30j1kkLYJBQ==" }, + "monaco-editor": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.16.2.tgz", + "integrity": "sha512-NtGrFzf54jADe7qsWh3lazhS7Kj0XHkJUGBq9fA/Jbwc+sgVcyfsYF6z2AQ7hPqDC+JmdOt/OwFjBnRwqXtx6w==" + }, "moo": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/moo/-/moo-0.4.3.tgz", @@ -12223,8 +12224,7 @@ "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, "object-copy": { "version": "0.1.0", @@ -13603,6 +13603,33 @@ "xtend": "^4.0.1" } }, + "react-monaco-editor": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/react-monaco-editor/-/react-monaco-editor-0.25.1.tgz", + "integrity": "sha512-VLksQjt8fmeIZoET9fQgmR0sdCCBFaiss92MwW5JvgO3JfrnhHtrLBqrSQRjtbUmef1EmEKi4gZp5Mh0oidvVg==", + "requires": { + "@types/react": "*", + "monaco-editor": "^0.16.0", + "prop-types": "^15.7.2" + }, + "dependencies": { + "prop-types": { + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + } + }, + "react-is": { + "version": "16.8.6", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.8.6.tgz", + "integrity": "sha512-aUk3bHfZ2bRSVFFbbeVS4i+lNPZr3/WM5jT2J5omUVV1zzcs1nAaf3l51ctA5FFvCRbhrH0bdAsRRQddFJZPtA==" + } + } + }, "react-table": { "version": "6.8.6", "resolved": "https://registry.npmjs.org/react-table/-/react-table-6.8.6.tgz", diff --git a/package.json b/package.json index b17e89327299..ddfb3714cdc6 100644 --- a/package.json +++ b/package.json @@ -2149,6 +2149,7 @@ "named-js-regexp": "^1.3.3", "node-stream-zip": "^1.6.0", "pidusage": "^1.2.0", + "react-monaco-editor": "^0.25.1", "reflect-metadata": "^0.1.12", "request": "^2.87.0", "request-progress": "^3.0.0", diff --git a/src/datascience-ui/history-react/code.css b/src/datascience-ui/history-react/code.css index a01a3b4bffa8..d9336d38cba2 100644 --- a/src/datascience-ui/history-react/code.css +++ b/src/datascience-ui/history-react/code.css @@ -38,6 +38,32 @@ margin-bottom: 10px; } +.code-area .monaco-editor +{ + overflow: hidden; +} + +.code-area .monaco-editor .slider +{ + margin: 0; +} + +.code-area .monaco-editor .parameter-hints-widget { + border: 0px; + } + .code-area .monaco-editor .parameter-hints-widget .signature { + padding: 0px; + } + .code-area .monaco-editor .suggest-widget { + border: 0px; + } + .code-area .monaco-editor.vs-dark .suggest-widget { + border: 0px; + } + .code-area .monaco-editor .rename-box { + top: 0; + } + .code-watermark { position: absolute; top: 0; diff --git a/src/datascience-ui/react-common/inlineMonacoEditor.tsx b/src/datascience-ui/react-common/inlineMonacoEditor.tsx new file mode 100644 index 000000000000..343159014b5a --- /dev/null +++ b/src/datascience-ui/react-common/inlineMonacoEditor.tsx @@ -0,0 +1,112 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + +// Got this from here: https://gist.github.com/abersnaze/efac6927e17187550d4f5d795334ccae +// Used to solve editor issues with monaco editor +// Doesn't work for editing though as it keeps resizing +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; +import * as React from 'react'; +// tslint:disable-next-line: import-name +import MonacoEditor, { EditorDidMount, MonacoEditorProps } from 'react-monaco-editor'; + +const LINE_HEIGHT = 18; +const DEFAULT_STATE = { + editor: undefined as unknown as monacoEditor.editor.ICodeEditor, + prevLineCount: -1 +}; + +export class InlineMonacoEditor extends React.Component { + constructor(props: MonacoEditorProps) { + super(props); + this.state = DEFAULT_STATE; + } + + public componentWillUnmount() { + if (window) { + window.removeEventListener('resize', this.setEditorHeight); + } + } + + public render() { + const { options = {}, editorDidMount } = this.props; + + // override a word wrapping, disable and hide the scroll bars + const optionsOverride: monacoEditor.editor.IEditorConstructionOptions = { + ...options, + wordWrap: 'on', + scrollBeyondLastLine: false, + scrollbar: { + vertical: 'hidden', + horizontal: 'hidden' + } + }; + + return ( + + ); + } + + private editorDidMount(prevEditorDidMount: EditorDidMount | undefined): EditorDidMount { + return (editor, monaco) => { + // chain an pre-existing editorDidMount handler + if (prevEditorDidMount) { + prevEditorDidMount(editor, monaco); + } + + // put the edit in the state for the handler. + this.setState({ editor }); + + // do the initial set of the height (wait a bit) + setTimeout(this.setEditorHeight, 0); + + // adjust height when the window resizes + if (window) { + window.addEventListener('resize', this.setEditorHeight); + } + + // on each edit recompute height (wait a bit) + editor.onDidChangeModelDecorations(() => { + setTimeout(this.setEditorHeight, 0); + }); + }; + } + + private setEditorHeight = () => { + const { editor, prevLineCount } = this.state; + if (!editor) { return; } + const editorDomNode = editor.getDomNode(); + if (!editorDomNode) { return; } + const container = editorDomNode.getElementsByClassName('view-lines')[0] as HTMLElement; + const containerHeight = container.offsetHeight; + const lineHeight = container.firstChild + ? (container.firstChild as HTMLElement).offsetHeight + : LINE_HEIGHT; + + if (!containerHeight) { + // dom hasn't finished settling down. wait a bit more. + setTimeout(this.setEditorHeight, 0); + } else { + const currLineCount = container.childElementCount; + if (currLineCount !== prevLineCount) { + // Only resize when we get a new line size + const nextHeight = (prevLineCount > currLineCount) + // if line count is shrinking monaco tends to leave the extra + // space at the end, compute the height from the line count + ? currLineCount * lineHeight + // otherwise use the height of the container div as the height + // of the editor node + : containerHeight; + + // set the height and redo layout + editorDomNode.style.height = `${nextHeight}px`; + editor.layout(); + this.setState({ prevLineCount: currLineCount }); + } + } + } +} From 88ca7e17d72dd64752d47ff6702bb2fe7ed6752b Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Mon, 22 Apr 2019 10:53:11 -0700 Subject: [PATCH 03/60] New idea for using the language client --- .../languageServer/languageServer.ts | 33 +++++++++------- src/client/activation/types.ts | 4 ++ .../datascience/history/completionProvider.ts | 39 +++++++++++++++++++ src/client/datascience/types.ts | 11 +++++- src/datascience-ui/history-react/code.tsx | 9 +++-- 5 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 src/client/datascience/history/completionProvider.ts diff --git a/src/client/activation/languageServer/languageServer.ts b/src/client/activation/languageServer/languageServer.ts index e7621c7df920..d936b48ed675 100644 --- a/src/client/activation/languageServer/languageServer.ts +++ b/src/client/activation/languageServer/languageServer.ts @@ -20,10 +20,9 @@ import { ProgressReporting } from './progress'; @injectable() export class LanguageServer implements ILanguageServer { + public languageClient: LanguageClient | undefined; private readonly startupCompleted: Deferred; private readonly disposables: Disposable[] = []; - - private languageClient?: LanguageClient; private extensionLoadedArgs = new Set<{}>(); constructor( @@ -54,21 +53,25 @@ export class LanguageServer implements ILanguageServer { @traceDecorators.error('Failed to start language server') @captureTelemetry(EventName.PYTHON_LANGUAGE_SERVER_ENABLED, undefined, true) public async start(resource: Resource, options: LanguageClientOptions): Promise { - this.languageClient = await this.factory.createLanguageClient(resource, options); - this.disposables.push(this.languageClient!.start()); - await this.serverReady(); - const progressReporting = new ProgressReporting(this.languageClient!); - this.disposables.push(progressReporting); + if (!this.languageClient) { + this.languageClient = await this.factory.createLanguageClient(resource, options); + this.disposables.push(this.languageClient!.start()); + await this.serverReady(); + const progressReporting = new ProgressReporting(this.languageClient!); + this.disposables.push(progressReporting); - const settings = this.configurationService.getSettings(resource); - if (settings.downloadLanguageServer) { - this.languageClient.onTelemetry(telemetryEvent => { - const eventName = telemetryEvent.EventName || EventName.PYTHON_LANGUAGE_SERVER_TELEMETRY; - sendTelemetryEvent(eventName, telemetryEvent.Measurements, telemetryEvent.Properties); - }); - } + const settings = this.configurationService.getSettings(resource); + if (settings.downloadLanguageServer) { + this.languageClient.onTelemetry(telemetryEvent => { + const eventName = telemetryEvent.EventName || EventName.PYTHON_LANGUAGE_SERVER_TELEMETRY; + sendTelemetryEvent(eventName, telemetryEvent.Measurements, telemetryEvent.Properties); + }); + } - await this.registerTestServices(); + await this.registerTestServices(); + } else { + await this.startupCompleted.promise; + } } @traceDecorators.error('Failed to load Language Server extension') public loadExtension(args?: {}) { diff --git a/src/client/activation/types.ts b/src/client/activation/types.ts index ecb227dca265..b948e236361b 100644 --- a/src/client/activation/types.ts +++ b/src/client/activation/types.ts @@ -103,6 +103,10 @@ export interface ILanguageServerExtension extends IDisposable { } export const ILanguageServer = Symbol('ILanguageServer'); export interface ILanguageServer extends IDisposable { + /** + * LanguageClient in use + */ + languageClient: LanguageClient | undefined; start(resource: Resource, options: LanguageClientOptions): Promise; /** * Sends a request to LS so as to load other extensions. diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts new file mode 100644 index 000000000000..a019a2db0831 --- /dev/null +++ b/src/client/datascience/history/completionProvider.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import '../../common/extensions'; + +import { inject, injectable } from 'inversify'; + +import { ILanguageServer } from '../../activation/types'; +import { IHistoryCompletionProvider } from '../types'; +import { noop } from '../../common/utils/misc'; +import { Uri } from 'vscode'; + +@injectable() +export class CompletionProvider implements IHistoryCompletionProvider { + + constructor( + @inject(ILanguageServer) private languageServer: ILanguageServer + ) { + // Should have one of us per history + } + + public dispose() { + noop(); + } + + public async startup(resource?: Uri) : Promise { + + } + + public async provideCompletionItems() : Promise { + + } + public async addCell(code: string): Promise { + + } + public async editCell(newCode: string, oldCode: string): void; + + +} diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 125579f22786..9a78b21db38e 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -5,7 +5,7 @@ import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services/lib/kernel'; import { JSONObject } from '@phosphor/coreutils'; import { Observable } from 'rxjs/Observable'; -import { CancellationToken, CodeLens, CodeLensProvider, Disposable, Event, Range, TextDocument, TextEditor } from 'vscode'; +import { CancellationToken, CodeLens, CodeLensProvider, Disposable, Event, Range, TextDocument, TextEditor, CompletionItem, Uri } from 'vscode'; import { ICommandManager } from '../common/application/types'; import { ExecutionResult, ObservableExecutionResult, SpawnOptions } from '../common/process/types'; @@ -148,6 +148,15 @@ export interface IHistory extends Disposable { exportCells(): void; } +export const IHistoryCompletionProvider = Symbol('IHistoryCompletionProvider'); + +export interface IHistoryCompletionProvider extends IDisposable { + startup(resource?: Uri) : Promise; + provideCompletionItems() : Promise; + addCell(code: string): Promise; + editCell(newCode: string, oldCode: string): Promise; +} + // Wraps the vscode API in order to send messages back and forth from a webview export const IPostOffice = Symbol('IPostOffice'); export interface IPostOffice { diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index d5d75ef9fe74..dfdb282fb5e0 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -108,7 +108,7 @@ export class Code extends React.Component { Enter: this.enter, 'Shift-Enter': this.shiftEnter, Up: this.arrowUp, - 'Ctrl-Space': 'autocomplete' + 'Ctrl-Space': 'autocomplete', }, theme: `${this.props.codeTheme} default`, mode: 'python', @@ -116,7 +116,6 @@ export class Code extends React.Component { readOnly: readOnly ? true : false, lineWrapping: true, hintOptions: { hint: this.provideHint } - // onKeyEvent could be used to provide autocomplete on say '.' for key up } } ref={this.updateCodeMirror} @@ -313,8 +312,11 @@ export class Code extends React.Component { return CodeMirror.Pass; } - private onChange = (_newValue: string, _change: CodeMirror.EditorChange) => { + private onChange = (_newValue: string, change: CodeMirror.EditorChange) => { this.setState({allowWatermark: false}); + if (change.text.length === 1 && change.text[0] === '.' && this.codeMirror) { + this.codeMirror.execCommand('autocomplete'); + } } // tslint:disable-next-line:no-any @@ -331,5 +333,4 @@ export class Code extends React.Component { return null; } - } From 192db0d324232c80d899330ef752ab1f8c784009 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Mon, 22 Apr 2019 16:28:46 -0700 Subject: [PATCH 04/60] Farther along the completion provider idea. --- .../datascience/history/completionProvider.ts | 243 +++++++++++++++++- src/client/datascience/types.ts | 4 +- 2 files changed, 236 insertions(+), 11 deletions(-) diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index a019a2db0831..9cc5389382c3 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -4,36 +4,261 @@ import '../../common/extensions'; import { inject, injectable } from 'inversify'; +import * as path from 'path'; +import * as uuid from 'uuid/v4'; +import { + CancellationToken, + CompletionContext, + CompletionItem, + CompletionList, + CompletionTriggerKind, + EndOfLine, + Position, + Range, + TextDocument, + TextLine, + Uri, + TextDocumentContentChangeEvent +} from 'vscode'; +import { CompletionRequest, LanguageClient, DidOpenTextDocumentNotification, TextDocumentItem, DidChangeTextDocumentNotification, VersionedTextDocumentIdentifier } from 'vscode-languageclient'; -import { ILanguageServer } from '../../activation/types'; +import { ILanguageServer, ILanguageServerAnalysisOptions } from '../../activation/types'; +import { IWorkspaceService } from '../../common/application/types'; +import { PYTHON_LANGUAGE } from '../../common/constants'; +import { IFileSystem, TemporaryFile } from '../../common/platform/types'; import { IHistoryCompletionProvider } from '../types'; -import { noop } from '../../common/utils/misc'; -import { Uri } from 'vscode'; + +class HistoryLine implements TextLine { + + private _range : Range; + private _rangeWithLineBreak: Range; + private _firstNonWhitespaceIndex : number | undefined; + private _isEmpty : boolean | undefined; + + constructor(private _contents: string, private _line: number) { + this._range = new Range(new Position(_line, 0), new Position(_line, _contents.length)); + this._rangeWithLineBreak = new Range(this.range.start, new Position(_line, _contents.length + 1)); + } + public get lineNumber(): number { + return this._line; + } + public get text(): string { + return this._contents; + } + public get range(): Range { + return this._range; + } + public get rangeIncludingLineBreak(): Range { + return this._rangeWithLineBreak; + } + public get firstNonWhitespaceCharacterIndex(): number { + if (this._firstNonWhitespaceIndex === undefined) { + this._firstNonWhitespaceIndex = this._contents.trimLeft().length - this._contents.length; + } + return this._firstNonWhitespaceIndex; + } + public get isEmptyOrWhitespace(): boolean { + if (this._isEmpty === undefined) { + this._isEmpty = this._contents.length === 0 || this._contents.trim().length === 0; + } + return this._isEmpty; + } + +} + +class HistoryDocument implements TextDocument { + + private _uri : Uri; + private _version : number = 0; + private _lines: TextLine[] = []; + private _cells: string = ''; + private _editOffset: number = 0; + + constructor(fileName: string) { + // The file passed in is the base Uri for where we're basing this + // document. + // + // What about liveshare? + this._uri = Uri.file(fileName); + } + + public get uri(): Uri { + return this._uri; + } + public get fileName(): string { + return this._uri.fsPath; + } + + public get isUntitled(): boolean { + return true; + } + public get languageId(): string { + return PYTHON_LANGUAGE; + } + public get version(): number { + return this._version; + } + public get isDirty(): boolean { + return true; + } + public get isClosed(): boolean { + return false; + } + public save(): Thenable { + return Promise.resolve(true); + } + public get eol(): EndOfLine { + return EndOfLine.LF; + } + public get lineCount(): number { + return this._lines.length; + } + public lineAt(position: Position | number): TextLine { + if (typeof position === 'number') { + return this._lines[position as number]; + } else { + return this._lines[position.line]; + } + } + public offsetAt(_position: Position): number { + throw new Error('Method not implemented.'); + } + public positionAt(_offset: number): Position { + throw new Error('Method not implemented.'); + } + public getText(_range?: Range | undefined): string { + if (!_range) { + return this.getText(new Range(new Position(0, 0), new Position(this._lines.length, this._lines[this._lines.length-1]))) + } + } + public getWordRangeAtPosition(_position: Position, _regex?: RegExp | undefined): Range | undefined { + throw new Error('Method not implemented.'); + } + public validateRange(range: Range): Range { + return range; + } + public validatePosition(position: Position): Position { + return position; + } + + public get textDocumentItem() : TextDocumentItem { + return { + uri : this.fileName, + languageId: this.languageId, + version: this.version, + text: this.getText() + }; + } + + public get textDocumentId() : VersionedTextDocumentIdentifier { + return { + uri: this.fileName, + version: this.version + }; + } + public addLines(code: string): TextDocumentContentChangeEvent[] { + this._lines.splice(this._editOffset); + const lastIndex = this._lines.length; + this._lines.concat(code.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + lastIndex))); + this._editOffset = this._lines.length; + } + + public editLines(from: Position, to: Position, newCode: string, removedCode?: string): TextDocumentContentChangeEvent[] { + const replacedRange = new Range(new Position(this._editOffset, 0)) + this._lines.splice(this._editOffset); + this._lines.concat(newCode.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + this._editOffset))); + + } + + private createTextLine(line: string, index: number) : TextLine { + return new HistoryLine(line, index); + } +} + @injectable() export class CompletionProvider implements IHistoryCompletionProvider { + private languageClient : LanguageClient | undefined; + private document: HistoryDocument | undefined; + private temporaryFile: TemporaryFile | undefined; + private sentOpenDocument : boolean = false; + constructor( - @inject(ILanguageServer) private languageServer: ILanguageServer + @inject(ILanguageServer) private languageServer: ILanguageServer, + @inject(ILanguageServerAnalysisOptions) private readonly analysisOptions: ILanguageServerAnalysisOptions, + @inject(IWorkspaceService) private workspaceService: IWorkspaceService, + @inject(IFileSystem) private fileSystem: IFileSystem ) { - // Should have one of us per history } public dispose() { - noop(); + this.languageServer.dispose(); } public async startup(resource?: Uri) : Promise { + // Save our language client. We'll use this to talk to the language server + const options = await this.analysisOptions!.getAnalysisOptions(); + await this.languageServer.start(resource, options); + this.languageClient = this.languageServer.languageClient; + // Create our dummy document. Compute a file path for it. + let dummyFilePath = ''; + if (this.workspaceService.rootPath || resource) { + const dir = resource ? path.dirname(resource.fsPath) : this.workspaceService.rootPath!; + dummyFilePath = path.join(dir, `History_${uuid().replace('-', '')}`); + } else { + this.temporaryFile = await this.fileSystem.createTemporaryFile('.py'); + dummyFilePath = this.temporaryFile.filePath; + } + this.document = new HistoryDocument(dummyFilePath); } - public async provideCompletionItems() : Promise { + public async provideCompletionItems(line: number, ch: number, cancellationToken: CancellationToken) : Promise { + if (this.languageClient && this.document) { + const position = new Position(line, ch); // Need to add on last line here + const context: CompletionContext = { + triggerKind: CompletionTriggerKind.TriggerCharacter + }; + const result = await this.languageClient.sendRequest( + CompletionRequest.type, + this.languageClient.code2ProtocolConverter.asCompletionParams(this.document, position, context), + cancellationToken) as CompletionList; + return result ? result.items : []; + } + return []; } public async addCell(code: string): Promise { + let changes: TextDocumentContentChangeEvent[] = []; + if (this.document) { + changes = this.document.addLines(code); + } + // Broadcast an update to the language server + if (this.languageClient && this.document) { + if (!this.sentOpenDocument) { + this.sentOpenDocument = true; + return this.languageClient.sendNotification(DidOpenTextDocumentNotification.type, { textDocument: this.document.textDocumentItem }); + } else { + return this.languageClient.sendNotification(DidChangeTextDocumentNotification.type, { textDocument: this.document.textDocumentId, contentChanges: changes }); + } + } } - public async editCell(newCode: string, oldCode: string): void; - + public async editCell(from: Position, to: Position, newCode: string, removedCode?: string): Promise { + let changes: TextDocumentContentChangeEvent[] = []; + if (this.document) { + changes = this.document.editLines(from, to, newCode, removedCode); + } + // Broadcast an update to the language server + if (this.languageClient && this.document) { + if (!this.sentOpenDocument) { + this.sentOpenDocument = true; + return this.languageClient.sendNotification(DidOpenTextDocumentNotification.type, { textDocument: this.document.textDocumentItem }); + } else { + return this.languageClient.sendNotification(DidChangeTextDocumentNotification.type, { textDocument: this.document.textDocumentId, contentChanges: changes }); + } + } + } } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 9a78b21db38e..12cbec39979c 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -152,9 +152,9 @@ export const IHistoryCompletionProvider = Symbol('IHistoryCompletionProvider'); export interface IHistoryCompletionProvider extends IDisposable { startup(resource?: Uri) : Promise; - provideCompletionItems() : Promise; + provideCompletionItems(line: number, ch: number, token: CancellationToken) : Promise; addCell(code: string): Promise; - editCell(newCode: string, oldCode: string): Promise; + editCell(from: Position, to: Position, newCode: string, removedCode?: string): Promise; } // Wraps the vscode API in order to send messages back and forth from a webview From 00cb5057d3d6e25c8e9fc008d99bd1a2f0f41dc6 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Tue, 23 Apr 2019 13:21:22 -0700 Subject: [PATCH 05/60] New idea mapped together --- .../languageServer/languageServer.ts | 3 +- src/client/activation/serviceRegistry.ts | 2 +- src/client/common/extensions.ts | 13 +++ .../datascience/history/completionProvider.ts | 85 +++++++++++++++---- src/client/datascience/history/history.ts | 51 ++++++++++- .../datascience/history/historyTypes.ts | 33 +++++++ src/client/datascience/serviceRegistry.ts | 3 + src/client/datascience/types.ts | 16 +++- .../history-react/MainPanel.tsx | 60 ++++++++++++- src/datascience-ui/history-react/cell.tsx | 6 ++ src/datascience-ui/history-react/code.tsx | 32 +++++-- .../history-react/contentPanel.tsx | 8 +- typings/extensions.d.ts | 4 + 13 files changed, 281 insertions(+), 35 deletions(-) diff --git a/src/client/activation/languageServer/languageServer.ts b/src/client/activation/languageServer/languageServer.ts index d936b48ed675..7e3486b88e15 100644 --- a/src/client/activation/languageServer/languageServer.ts +++ b/src/client/activation/languageServer/languageServer.ts @@ -21,7 +21,7 @@ import { ProgressReporting } from './progress'; @injectable() export class LanguageServer implements ILanguageServer { public languageClient: LanguageClient | undefined; - private readonly startupCompleted: Deferred; + private startupCompleted: Deferred; private readonly disposables: Disposable[] = []; private extensionLoadedArgs = new Set<{}>(); @@ -47,6 +47,7 @@ export class LanguageServer implements ILanguageServer { } if (this.startupCompleted.completed) { this.startupCompleted.reject(new Error('Disposed Language Server')); + this.startupCompleted = createDeferred(); } } diff --git a/src/client/activation/serviceRegistry.ts b/src/client/activation/serviceRegistry.ts index e15f7ab82554..ad19bd0541d5 100644 --- a/src/client/activation/serviceRegistry.ts +++ b/src/client/activation/serviceRegistry.ts @@ -53,6 +53,6 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(ILanguageServerDownloader, LanguageServerDownloader); serviceManager.addSingleton(IPlatformData, PlatformData); serviceManager.add(ILanguageServerAnalysisOptions, LanguageServerAnalysisOptions); - serviceManager.add(ILanguageServer, LanguageServer); + serviceManager.addSingleton(ILanguageServer, LanguageServer); serviceManager.add(ILanguageServerManager, LanguageServerManager); } diff --git a/src/client/common/extensions.ts b/src/client/common/extensions.ts index 72a09e5a49db..7546f7232955 100644 --- a/src/client/common/extensions.ts +++ b/src/client/common/extensions.ts @@ -95,6 +95,10 @@ declare interface Promise { * Catches task error and ignores them. */ ignoreErrors(): void; + /** + * Catches task error and logs them. + */ + logErrors(): void; } /** @@ -105,6 +109,15 @@ Promise.prototype.ignoreErrors = function (this: Promise) { this.catch(() => { }); }; +/** + * Explicitly tells that promise should be run asynchonously. + */ +Promise.prototype.logErrors = function (this: Promise) { + // tslint:disable-next-line: no-require-imports + const logger = require('./logger'); + this.catch((e) => { logger.traceError(e); }); +}; + if (!String.prototype.format) { String.prototype.format = function (this: string) { const args = arguments; diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index 9cc5389382c3..d9e1c4ac925d 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -16,16 +16,24 @@ import { Position, Range, TextDocument, + TextDocumentContentChangeEvent, TextLine, - Uri, - TextDocumentContentChangeEvent + Uri } from 'vscode'; -import { CompletionRequest, LanguageClient, DidOpenTextDocumentNotification, TextDocumentItem, DidChangeTextDocumentNotification, VersionedTextDocumentIdentifier } from 'vscode-languageclient'; +import { + CompletionRequest, + DidChangeTextDocumentNotification, + DidOpenTextDocumentNotification, + LanguageClient, + TextDocumentItem, + VersionedTextDocumentIdentifier +} from 'vscode-languageclient'; import { ILanguageServer, ILanguageServerAnalysisOptions } from '../../activation/types'; import { IWorkspaceService } from '../../common/application/types'; import { PYTHON_LANGUAGE } from '../../common/constants'; import { IFileSystem, TemporaryFile } from '../../common/platform/types'; +import { Identifiers } from '../constants'; import { IHistoryCompletionProvider } from '../types'; class HistoryLine implements TextLine { @@ -71,7 +79,7 @@ class HistoryDocument implements TextDocument { private _uri : Uri; private _version : number = 0; private _lines: TextLine[] = []; - private _cells: string = ''; + private _contents: string = ''; private _editOffset: number = 0; constructor(fileName: string) { @@ -123,12 +131,20 @@ class HistoryDocument implements TextDocument { public offsetAt(_position: Position): number { throw new Error('Method not implemented.'); } - public positionAt(_offset: number): Position { - throw new Error('Method not implemented.'); + public positionAt(offset: number): Position { + const before = this._contents.slice(0, offset); + const newLines = before.match(/\n/g); + const line = newLines ? newLines.length : 0; + const preCharacters = before.match(/(\n|^).*$/g); + return new Position(line, preCharacters ? preCharacters[0].length : 0); } - public getText(_range?: Range | undefined): string { - if (!_range) { - return this.getText(new Range(new Position(0, 0), new Position(this._lines.length, this._lines[this._lines.length-1]))) + public getText(range?: Range | undefined): string { + if (!range) { + return this._contents; + } else { + const startOffset = this.convertToOffset(range.start); + const endOffset = this.convertToOffset(range.end); + return this._contents.substr(startOffset, endOffset - startOffset); } } public getWordRangeAtPosition(_position: Position, _regex?: RegExp | undefined): Range | undefined { @@ -159,22 +175,56 @@ class HistoryDocument implements TextDocument { public addLines(code: string): TextDocumentContentChangeEvent[] { this._lines.splice(this._editOffset); const lastIndex = this._lines.length; + const oldEnd = this._editOffset; this._lines.concat(code.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + lastIndex))); this._editOffset = this._lines.length; + this._contents += this._contents.length ? `\n${code}` : code; + return [ + { + range: new Range(new Position(lastIndex, 0), new Position(this._lines.length, 0)), + rangeOffset: oldEnd, + rangeLength: code.length, + text: code + } + ]; } - public editLines(from: Position, to: Position, newCode: string, removedCode?: string): TextDocumentContentChangeEvent[] { - const replacedRange = new Range(new Position(this._editOffset, 0)) - this._lines.splice(this._editOffset); - this._lines.concat(newCode.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + this._editOffset))); + public editLines(from: Position, to: Position, newCode: string, _removedCode?: string): TextDocumentContentChangeEvent[] { + // From and to are the offset from the beginning of a cell, not the offset of our document + const fromLine = from.line + this._editOffset; + const toLine = to.line + this._editOffset; + + // Recreate our contents, and then recompute all of our lines + const fromOffset = this.convertToOffset(new Position(fromLine, from.character)); + const toOffset = this.convertToOffset(new Position(toLine, to.character)); + const before = this._contents.substr(fromOffset); + const after = this._contents.substr(toOffset); + this._contents = `${before}${newCode}${after}`; + this._lines = this._contents.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i)); + + return [ + { + range: new Range(new Position(fromLine, from.character), new Position(toLine, to.character)), + rangeOffset: fromOffset, + rangeLength: newCode.length, + text: newCode + } + ]; } private createTextLine(line: string, index: number) : TextLine { return new HistoryLine(line, index); } -} + private convertToOffset(pos: Position) : number { + // Combine the text length up to this position + const lenUpToPos = this._lines.filter(l => l.range.start.line <= pos.line).map(l => l.text.length + 1).reduce((p, c) => p + c); + + // Add on the character + return lenUpToPos + pos.character; + } +} @injectable() export class CompletionProvider implements IHistoryCompletionProvider { @@ -206,7 +256,7 @@ export class CompletionProvider implements IHistoryCompletionProvider { let dummyFilePath = ''; if (this.workspaceService.rootPath || resource) { const dir = resource ? path.dirname(resource.fsPath) : this.workspaceService.rootPath!; - dummyFilePath = path.join(dir, `History_${uuid().replace('-', '')}`); + dummyFilePath = path.join(dir, `History_${uuid().replace(/-/g, '')}.py`); } else { this.temporaryFile = await this.fileSystem.createTemporaryFile('.py'); dummyFilePath = this.temporaryFile.filePath; @@ -229,7 +279,10 @@ export class CompletionProvider implements IHistoryCompletionProvider { return []; } - public async addCell(code: string): Promise { + public async addCell(code: string, file: string): Promise { + if (!this.languageClient) { + await this.startup(file === Identifiers.EmptyFileName ? undefined : Uri.file(file)); + } let changes: TextDocumentContentChangeEvent[] = []; if (this.document) { changes = this.document.addLines(code); diff --git a/src/client/datascience/history/history.ts b/src/client/datascience/history/history.ts index 8c284755e2bf..b0283114d390 100644 --- a/src/client/datascience/history/history.ts +++ b/src/client/datascience/history/history.ts @@ -8,7 +8,7 @@ import { inject, injectable } from 'inversify'; import * as path from 'path'; import * as uuid from 'uuid/v4'; import { Event, EventEmitter, Position, Range, Selection, TextEditor, Uri, ViewColumn } from 'vscode'; -import { Disposable } from 'vscode-jsonrpc'; +import { CancellationTokenSource, Disposable } from 'vscode-jsonrpc'; import * as vsls from 'vsls/vscode'; import { @@ -39,6 +39,7 @@ import { IConnection, IDataViewerProvider, IHistory, + IHistoryCompletionProvider, IHistoryInfo, IHistoryProvider, IJupyterExecution, @@ -53,7 +54,16 @@ import { } from '../types'; import { WebViewHost } from '../webViewHost'; import { HistoryMessageListener } from './historyMessageListener'; -import { HistoryMessages, IAddedSysInfo, IGotoCode, IHistoryMapping, IRemoteAddCode, ISubmitNewCell } from './historyTypes'; +import { + HistoryMessages, + IAddedSysInfo, + IEditCell, + IGotoCode, + IHistoryMapping, + IProvideCompletionItemsRequest, + IRemoteAddCode, + ISubmitNewCell +} from './historyTypes'; export enum SysInfoReason { Start, @@ -76,6 +86,7 @@ export class History extends WebViewHost implements IHistory { private jupyterServer: INotebookServer | undefined; private id : string; private executeEvent: EventEmitter = new EventEmitter(); + private autoCompleteCancelSource: CancellationTokenSource | undefined; constructor( @inject(ILiveShareApi) private liveShare : ILiveShareApi, @@ -96,7 +107,8 @@ export class History extends WebViewHost implements IHistory { @inject(IWorkspaceService) workspaceService: IWorkspaceService, @inject(IHistoryProvider) private historyProvider: IHistoryProvider, @inject(IDataViewerProvider) private dataExplorerProvider: IDataViewerProvider, - @inject(IJupyterVariables) private jupyterVariables: IJupyterVariables + @inject(IJupyterVariables) private jupyterVariables: IJupyterVariables, + @inject(IHistoryCompletionProvider) private completionProvider: IHistoryCompletionProvider ) { super( configuration, @@ -161,7 +173,7 @@ export class History extends WebViewHost implements IHistory { return this.submitCode(code, file, line, undefined, editor); } - // tslint:disable-next-line: no-any no-empty cyclomatic-complexity + // tslint:disable-next-line: no-any no-empty cyclomatic-complexity max-func-body-length public onMessage(message: string, payload: any) { switch (message) { case HistoryMessages.GotoCodeCell: @@ -240,6 +252,14 @@ export class History extends WebViewHost implements IHistory { this.dispatchMessage(message, payload, this.requestVariableValue); break; + case HistoryMessages.ProvideCompletionItemsRequest: + this.dispatchMessage(message, payload, this.provideCompletionItems); + break; + + case HistoryMessages.EditCell: + this.dispatchMessage(message, payload, this.editCell); + break; + default: break; } @@ -591,6 +611,9 @@ export class History extends WebViewHost implements IHistory { // Wait for the cell to finish await finishedAddingCode.promise; traceInfo(`Finished execution for ${id}`); + + // Add this to our completion provider so it can remember this code + await this.completionProvider.addCell(code, file); } } catch (err) { status.dispose(); @@ -989,4 +1012,24 @@ export class History extends WebViewHost implements IHistory { sendTelemetryEvent(Telemetry.VariableExplorerToggled, undefined, { open: openValue }); } } + + private provideCompletionItems(request: IProvideCompletionItemsRequest) { + if (this.autoCompleteCancelSource) { + this.autoCompleteCancelSource.cancel(); + } + this.autoCompleteCancelSource = new CancellationTokenSource(); + this.completionProvider.provideCompletionItems(request.line, request.ch, this.autoCompleteCancelSource.token).then(items => { + this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {items: items.map(i => i.insertText as string), line: request.line, ch: request.ch, id: request.id}).ignoreErrors(); + }).catch(_e => { + this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {items: [], line: request.line, ch: request.ch, id: request.id}).ignoreErrors(); + }); + } + + private editCell(request: IEditCell) { + this.completionProvider.editCell( + new Position(request.from.line, request.from.ch), + new Position(request.to.line, request.to.ch), + request.newCode, + request.removedCode).ignoreErrors(); + } } diff --git a/src/client/datascience/history/historyTypes.ts b/src/client/datascience/history/historyTypes.ts index 6dba830f7428..d1d93047ab19 100644 --- a/src/client/datascience/history/historyTypes.ts +++ b/src/client/datascience/history/historyTypes.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; + import { CssMessages, IGetCssRequest, IGetCssResponse, SharedMessages } from '../constants'; import { ICell, IHistoryInfo, IJupyterVariable, IJupyterVariablesResponse } from '../types'; @@ -35,6 +36,9 @@ export namespace HistoryMessages { export const GetVariableValueRequest = 'get_variable_value_request'; export const GetVariableValueResponse = 'get_variable_value_response'; export const VariableExplorerToggle = 'variable_explorer_toggle'; + export const ProvideCompletionItemsRequest = 'provide_completion_items_request'; + export const ProvideCompletionItemsResponse = 'provide_completion_items_response'; + export const EditCell = 'edit_cell'; } // These are the messages that will mirror'd to guest/hosts in @@ -70,6 +74,32 @@ export interface ISubmitNewCell { id: string; } +export interface IProvideCompletionItemsRequest { + line: number; + ch: number; + triggerKey: string; + id: string; +} + +export interface IProvideCompletionItemsResponse { + items: string[]; + line: number; + ch: number; + id: string; +} + +export interface IPosition { + line: number; + ch: number; +} + +export interface IEditCell { + from: IPosition; + to: IPosition; + newCode: string; + removedCode?: string; +} + // Map all messages to specific payloads export class IHistoryMapping { public [HistoryMessages.StartCell]: ICell; @@ -104,4 +134,7 @@ export class IHistoryMapping { public [HistoryMessages.VariableExplorerToggle]: boolean; public [CssMessages.GetCssRequest] : IGetCssRequest; public [CssMessages.GetCssResponse] : IGetCssResponse; + public [HistoryMessages.ProvideCompletionItemsRequest] : IProvideCompletionItemsRequest; + public [HistoryMessages.ProvideCompletionItemsResponse] : IProvideCompletionItemsResponse; + public [HistoryMessages.EditCell] : IEditCell; } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 3a98946c4472..1afbf6713ca9 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -10,6 +10,7 @@ import { DataScience } from './datascience'; import { DataScienceCodeLensProvider } from './editor-integration/codelensprovider'; import { CodeWatcher } from './editor-integration/codewatcher'; import { Decorator } from './editor-integration/decorator'; +import { CompletionProvider } from './history/completionProvider'; import { History } from './history/history'; import { HistoryCommandListener } from './history/historycommandlistener'; import { HistoryProvider } from './history/historyProvider'; @@ -31,6 +32,7 @@ import { IDataViewer, IDataViewerProvider, IHistory, + IHistoryCompletionProvider, IHistoryProvider, IJupyterCommandFactory, IJupyterExecution, @@ -63,4 +65,5 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IDataViewerProvider, DataViewerProvider); serviceManager.add(IDataViewer, DataViewer); serviceManager.addSingleton(IExtensionActivationService, Decorator); + serviceManager.addSingleton(IHistoryCompletionProvider, CompletionProvider); } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 12cbec39979c..b704b7c830e1 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -5,7 +5,18 @@ import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services/lib/kernel'; import { JSONObject } from '@phosphor/coreutils'; import { Observable } from 'rxjs/Observable'; -import { CancellationToken, CodeLens, CodeLensProvider, Disposable, Event, Range, TextDocument, TextEditor, CompletionItem, Uri } from 'vscode'; +import { + CancellationToken, + CodeLens, + CodeLensProvider, + CompletionItem, + Disposable, + Event, + Position, + Range, + TextDocument, + TextEditor +} from 'vscode'; import { ICommandManager } from '../common/application/types'; import { ExecutionResult, ObservableExecutionResult, SpawnOptions } from '../common/process/types'; @@ -151,9 +162,8 @@ export interface IHistory extends Disposable { export const IHistoryCompletionProvider = Symbol('IHistoryCompletionProvider'); export interface IHistoryCompletionProvider extends IDisposable { - startup(resource?: Uri) : Promise; provideCompletionItems(line: number, ch: number, token: CancellationToken) : Promise; - addCell(code: string): Promise; + addCell(code: string, file: string): Promise; editCell(from: Position, to: Position, newCode: string, removedCode?: string): Promise; } diff --git a/src/datascience-ui/history-react/MainPanel.tsx b/src/datascience-ui/history-react/MainPanel.tsx index 49aef9cd3e46..cbd10de0ce15 100644 --- a/src/datascience-ui/history-react/MainPanel.tsx +++ b/src/datascience-ui/history-react/MainPanel.tsx @@ -6,9 +6,14 @@ import './mainPanel.css'; import { min } from 'lodash'; import * as React from 'react'; +import { createDeferred, Deferred } from '../../client/common/utils/async'; import { CellMatcher } from '../../client/datascience/cellMatcher'; import { generateMarkdownFromCodeLines } from '../../client/datascience/common'; -import { HistoryMessages, IHistoryMapping } from '../../client/datascience/history/historyTypes'; +import { + HistoryMessages, + IHistoryMapping, + IProvideCompletionItemsResponse +} from '../../client/datascience/history/historyTypes'; import { CellState, ICell, IHistoryInfo, IJupyterVariable, IJupyterVariablesResponse } from '../../client/datascience/types'; import { IMessageHandler, PostOffice } from '../react-common/postOffice'; import { getSettings, updateSettings } from '../react-common/settingsReactSide'; @@ -37,6 +42,8 @@ export class MainPanel extends React.Component private styleInjectorRef: React.RefObject; private currentExecutionCount: number = 0; private postOffice: PostOffice = new PostOffice(); + private currentCompletionItemsRequest: Deferred | undefined; + private currentCompletionItemsRequestId: string | undefined; // tslint:disable-next-line:max-func-body-length constructor(props: IMainPanelProps, _state: IMainPanelState) { @@ -173,6 +180,10 @@ export class MainPanel extends React.Component this.getVariableValueResponse(payload); break; + case HistoryMessages.ProvideCompletionItemsResponse: + this.handleCompletionResponse(payload); + break; + default: break; } @@ -246,7 +257,9 @@ export class MainPanel extends React.Component gotoCellCode: this.gotoCellCode, deleteCell: this.deleteCell, submitInput: this.submitInput, - skipNextScroll: this.state.skipNextScroll ? true : false + skipNextScroll: this.state.skipNextScroll ? true : false, + onCodeChange: this.codeChange, + requestCompletionItems: this.requestCompletionItems }; } private getHeaderProps = (baseTheme: string): IHeaderPanelProps => { @@ -809,4 +822,47 @@ export class MainPanel extends React.Component } } } + + private codeChange = (fromLine: number, fromCh: number, toLine: number, toCh: number, text: string, removed?: string) => { + // Pass this onto the completion provider running in the extension + this.sendMessage(HistoryMessages.EditCell, + { + from: { + line: fromLine, + ch: fromCh + }, + to: { + line: toLine, + ch: toCh + }, + newCode: text, + removedCode: removed + } + ); + } + + private requestCompletionItems = (line: number, ch: number, id: string) : Promise => { + if (this.currentCompletionItemsRequest && !this.currentCompletionItemsRequest.resolved && this.currentCompletionItemsRequestId) { + this.currentCompletionItemsRequest.resolve({ items: [], line, ch, id: this.currentCompletionItemsRequestId}); + } + this.currentCompletionItemsRequest = createDeferred(); + this.currentCompletionItemsRequestId = id; + this.sendMessage(HistoryMessages.ProvideCompletionItemsRequest, { line, ch, id, triggerKey: '' }); + return this.currentCompletionItemsRequest.promise; + } + + // Handle completion response + // tslint:disable-next-line:no-any + private handleCompletionResponse = (payload?: any) => { + if (payload) { + const response = payload as IProvideCompletionItemsResponse; + + // Resolve our waiting promise if we have one + if (this.currentCompletionItemsRequest && !this.currentCompletionItemsRequest.resolved && response.id === this.currentCompletionItemsRequestId) { + this.currentCompletionItemsRequestId = undefined; + this.currentCompletionItemsRequest.resolve({ items: response.items, line: response.line, ch: response.ch, id: response.id }); + } + } + } + } diff --git a/src/datascience-ui/history-react/cell.tsx b/src/datascience-ui/history-react/cell.tsx index 091a34e36b6a..6675a1c124c0 100644 --- a/src/datascience-ui/history-react/cell.tsx +++ b/src/datascience-ui/history-react/cell.tsx @@ -13,6 +13,7 @@ import JSONTree from 'react-json-tree'; import '../../client/common/extensions'; import { concatMultilineString, formatStreamText } from '../../client/datascience/common'; import { Identifiers } from '../../client/datascience/constants'; +import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; import { CellState, ICell } from '../../client/datascience/types'; import { noop } from '../../test/core'; import { getLocString } from '../react-common/locReactSide'; @@ -42,6 +43,9 @@ interface ICellProps { gotoCode(): void; delete(): void; submitNewCode(code: string): void; + onCodeChange(fromLine: number, fromCh: number, toLine: number, toCh: number, text: string, removed?: string): void; + requestCompletionItems(line: number, ch: number, id: string) : Promise; + } export interface ICellViewModel { @@ -219,6 +223,8 @@ export class Cell extends React.Component { onSubmit={this.props.submitNewCode} onChangeLineCount={this.onChangeLineCount} ref={this.updateCodeRef} + onChange={this.props.onCodeChange} + requestCompletionItems={this.props.requestCompletionItems} /> ); diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index dfdb282fb5e0..94bd695d64ae 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -8,6 +8,7 @@ import 'codemirror/mode/python/python'; import * as CodeMirror from 'codemirror'; import * as React from 'react'; import * as RCM from 'react-codemirror'; +import * as uuid from 'uuid/v4'; // tslint:disable-next-line: no-require-imports no-var-requires require('codemirror/addon/hint/show-hint'); @@ -15,6 +16,7 @@ import 'codemirror/addon/hint/show-hint.css'; import './code.css'; +import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; import { getLocString } from '../react-common/locReactSide'; import { Cursor } from './cursor'; import { InputHistory } from './inputHistory'; @@ -30,6 +32,8 @@ export interface ICodeProps { showWatermark: boolean; onSubmit(code: string): void; onChangeLineCount(lineCount: number) : void; + onChange(fromLine: number, fromCh: number, toLine: number, toCh: number, text: string, removed?: string): void; + requestCompletionItems(line: number, ch: number, id: string) : Promise; } @@ -108,7 +112,7 @@ export class Code extends React.Component { Enter: this.enter, 'Shift-Enter': this.shiftEnter, Up: this.arrowUp, - 'Ctrl-Space': 'autocomplete', + 'Ctrl-Space': 'autocomplete' }, theme: `${this.props.codeTheme} default`, mode: 'python', @@ -314,9 +318,19 @@ export class Code extends React.Component { private onChange = (_newValue: string, change: CodeMirror.EditorChange) => { this.setState({allowWatermark: false}); - if (change.text.length === 1 && change.text[0] === '.' && this.codeMirror) { + if (change.text.length === 1 && change.text[0] === '.' && this.codeMirror && !this.props.readOnly) { this.codeMirror.execCommand('autocomplete'); } + // Pass this change onto any listeners + if (!this.props.readOnly) { + this.props.onChange( + change.from.line, + change.from.ch, + change.to.line, + change.to.ch, + change.text.join('\n'), + change.removed ? change.removed.join('\n') : undefined); + } } // tslint:disable-next-line:no-any @@ -324,11 +338,15 @@ export class Code extends React.Component { const doc = instance.getDoc(); const cursor = doc ? doc.getCursor() : undefined; if (cursor) { - return { - list: ['foo', 'bar', 'baz'], - from: cursor, - to: cursor - }; + // Ask for a set of completion items + const completionItems = await this.props.requestCompletionItems(cursor.line, cursor.ch, uuid()); + if (completionItems && completionItems.items && completionItems.line === cursor.line && completionItems.ch === cursor.ch) { + return { + list: completionItems.items, + from: cursor, + to: cursor + }; + } } return null; diff --git a/src/datascience-ui/history-react/contentPanel.tsx b/src/datascience-ui/history-react/contentPanel.tsx index a1895e0eee10..a1f65acad36a 100644 --- a/src/datascience-ui/history-react/contentPanel.tsx +++ b/src/datascience-ui/history-react/contentPanel.tsx @@ -4,6 +4,7 @@ import './contentPanel.css'; import * as React from 'react'; +import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; import { noop } from '../../test/core'; import { ErrorBoundary } from '../react-common/errorBoundary'; import { getSettings } from '../react-common/settingsReactSide'; @@ -23,6 +24,8 @@ export interface IContentPanelProps { gotoCellCode(index: number): void; deleteCell(index: number): void; submitInput(code: string): void; + onCodeChange(fromLine: number, fromCh: number, toLine: number, toCh: number, text: string, removed?: string): void; + requestCompletionItems(line: number, ch: number, id: string) : Promise; } export class ContentPanel extends React.Component { @@ -79,7 +82,10 @@ export class ContentPanel extends React.Component { errorBackgroundColor={actualErrorBackgroundColor} ref={(r) => cellVM.editable ? this.props.saveEditCellRef(r) : noop()} gotoCode={() => this.props.gotoCellCode(index)} - delete={() => this.props.deleteCell(index)}/> + delete={() => this.props.deleteCell(index)} + onCodeChange={this.props.onCodeChange} + requestCompletionItems={this.props.requestCompletionItems} + /> ); } diff --git a/typings/extensions.d.ts b/typings/extensions.d.ts index 4a423f329d57..56a0a696aeee 100644 --- a/typings/extensions.d.ts +++ b/typings/extensions.d.ts @@ -44,4 +44,8 @@ declare interface Promise { * Catches task errors and ignores them. */ ignoreErrors(): void; + /** + * Catches task errors and logs them + */ + logErrors(): void; } From f12e3f8a34045d5329cf405e27ad75619b6a32ee Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Apr 2019 08:48:28 -0700 Subject: [PATCH 06/60] Closer. no more errors --- .../datascience/history/completionProvider.ts | 57 +++++++++++++------ .../datascience/jupyter/jupyterServer.ts | 2 +- .../datascience/jupyter/jupyterSession.ts | 2 +- src/datascience-ui/history-react/code.tsx | 6 +- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index d9e1c4ac925d..ec3fb74aca97 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -159,7 +159,7 @@ class HistoryDocument implements TextDocument { public get textDocumentItem() : TextDocumentItem { return { - uri : this.fileName, + uri : this._uri.toString(), languageId: this.languageId, version: this.version, text: this.getText() @@ -168,24 +168,25 @@ class HistoryDocument implements TextDocument { public get textDocumentId() : VersionedTextDocumentIdentifier { return { - uri: this.fileName, + uri: this._uri.toString(), version: this.version }; } public addLines(code: string): TextDocumentContentChangeEvent[] { + const normalized = code.replace(/\r/g, ''); this._lines.splice(this._editOffset); const lastIndex = this._lines.length; - const oldEnd = this._editOffset; - this._lines.concat(code.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + lastIndex))); + this._lines = this._lines.concat(normalized.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + lastIndex))); this._editOffset = this._lines.length; - this._contents += this._contents.length ? `\n${code}` : code; + this._contents += this._contents.length ? `\n${normalized}` : normalized; return [ + // tslint:disable-next-line: no-object-literal-type-assertion { - range: new Range(new Position(lastIndex, 0), new Position(this._lines.length, 0)), - rangeOffset: oldEnd, - rangeLength: code.length, - text: code - } + range: this.createSerializableRange(new Position(lastIndex, 0), new Position(this._lines.length, 0)), + // Range offset not passed by the editor so don't use it. + rangeLength: normalized.length, + text: normalized + } as TextDocumentContentChangeEvent ]; } @@ -197,33 +198,52 @@ class HistoryDocument implements TextDocument { // Recreate our contents, and then recompute all of our lines const fromOffset = this.convertToOffset(new Position(fromLine, from.character)); const toOffset = this.convertToOffset(new Position(toLine, to.character)); - const before = this._contents.substr(fromOffset); + const before = this._contents.substr(0, fromOffset); const after = this._contents.substr(toOffset); this._contents = `${before}${newCode}${after}`; this._lines = this._contents.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i)); return [ + // tslint:disable-next-line: no-object-literal-type-assertion { - range: new Range(new Position(fromLine, from.character), new Position(toLine, to.character)), - rangeOffset: fromOffset, - rangeLength: newCode.length, + range: this.createSerializableRange(new Position(fromLine, from.character), new Position(toLine, to.character)), + // Range offset not passed by the editor so don't use it. + rangeLength: toOffset - fromOffset, text: newCode - } + } as TextDocumentContentChangeEvent ]; } + public convertToDocumentPosition(line: number, ch: number) : Position { + return new Position(line + this._editOffset, ch); + } + private createTextLine(line: string, index: number) : TextLine { return new HistoryLine(line, index); } private convertToOffset(pos: Position) : number { // Combine the text length up to this position - const lenUpToPos = this._lines.filter(l => l.range.start.line <= pos.line).map(l => l.text.length + 1).reduce((p, c) => p + c); + const lenUpToPos = this._lines.filter(l => l.range.start.line < pos.line).map(l => l.range.end.character).reduce((p, c) => p + c); // Add on the character return lenUpToPos + pos.character; } + + private createSerializableRange(start: Position, end: Position) : Range { + const result = { + start: { + line: start.line, + character: start.character + }, + end: { + line: end.line, + character: end.character + } + }; + return result as Range; + } } @injectable() @@ -266,9 +286,10 @@ export class CompletionProvider implements IHistoryCompletionProvider { public async provideCompletionItems(line: number, ch: number, cancellationToken: CancellationToken) : Promise { if (this.languageClient && this.document) { - const position = new Position(line, ch); // Need to add on last line here + const position = this.document.convertToDocumentPosition(line, ch); const context: CompletionContext = { - triggerKind: CompletionTriggerKind.TriggerCharacter + triggerKind: CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '.' }; const result = await this.languageClient.sendRequest( CompletionRequest.type, diff --git a/src/client/datascience/jupyter/jupyterServer.ts b/src/client/datascience/jupyter/jupyterServer.ts index 34d863e202e0..93e77609d66c 100644 --- a/src/client/datascience/jupyter/jupyterServer.ts +++ b/src/client/datascience/jupyter/jupyterServer.ts @@ -173,7 +173,7 @@ export class JupyterServerBase implements INotebookServer { // Wait for it to be ready traceInfo(`Waiting for idle ${this.id}`); - await this.session.waitForIdle(Number.MAX_SAFE_INTEGER); + await this.session.waitForIdle(this.configService.getSettings().datascience.jupyterLaunchTimeout); traceInfo(`Performing initial setup ${this.id}`); // Run our initial setup and plot magics diff --git a/src/client/datascience/jupyter/jupyterSession.ts b/src/client/datascience/jupyter/jupyterSession.ts index 1aa2851da598..da630605cfc2 100644 --- a/src/client/datascience/jupyter/jupyterSession.ts +++ b/src/client/datascience/jupyter/jupyterSession.ts @@ -84,7 +84,7 @@ export class JupyterSession implements IJupyterSession { this.session.kernel.status !== 'idle' && (Date.now() - startTime < timeout)) { traceInfo(`Waiting for idle: ${this.session.kernel.status}`); - await sleep(10); + await sleep(100); } // If we didn't make it out in ten seconds, indicate an error diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index 94bd695d64ae..4dbf6928d3a1 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -318,9 +318,6 @@ export class Code extends React.Component { private onChange = (_newValue: string, change: CodeMirror.EditorChange) => { this.setState({allowWatermark: false}); - if (change.text.length === 1 && change.text[0] === '.' && this.codeMirror && !this.props.readOnly) { - this.codeMirror.execCommand('autocomplete'); - } // Pass this change onto any listeners if (!this.props.readOnly) { this.props.onChange( @@ -331,6 +328,9 @@ export class Code extends React.Component { change.text.join('\n'), change.removed ? change.removed.join('\n') : undefined); } + if (change.text.length === 1 && change.text[0] === '.' && this.codeMirror && !this.props.readOnly) { + this.codeMirror.execCommand('autocomplete'); + } } // tslint:disable-next-line:no-any From 16df587d6fa33f8200906f1f2bd8e5b82baa672f Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Apr 2019 10:04:43 -0700 Subject: [PATCH 07/60] Working completion --- .../datascience/history/completionProvider.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index ec3fb74aca97..fef36c195509 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -173,12 +173,13 @@ class HistoryDocument implements TextDocument { }; } public addLines(code: string): TextDocumentContentChangeEvent[] { + this._version += 1; const normalized = code.replace(/\r/g, ''); this._lines.splice(this._editOffset); const lastIndex = this._lines.length; - this._lines = this._lines.concat(normalized.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i + lastIndex))); - this._editOffset = this._lines.length; this._contents += this._contents.length ? `\n${normalized}` : normalized; + this._lines = this.createLines(this._contents); + this._editOffset = this._lines.length; return [ // tslint:disable-next-line: no-object-literal-type-assertion { @@ -191,6 +192,7 @@ class HistoryDocument implements TextDocument { } public editLines(from: Position, to: Position, newCode: string, _removedCode?: string): TextDocumentContentChangeEvent[] { + this._version += 1; // From and to are the offset from the beginning of a cell, not the offset of our document const fromLine = from.line + this._editOffset; const toLine = to.line + this._editOffset; @@ -201,7 +203,7 @@ class HistoryDocument implements TextDocument { const before = this._contents.substr(0, fromOffset); const after = this._contents.substr(toOffset); this._contents = `${before}${newCode}${after}`; - this._lines = this._contents.splitLines({trim: false, removeEmptyEntries: false}).map((c, i) => this.createTextLine(c, i)); + this._lines = this.createLines(this._contents); return [ // tslint:disable-next-line: no-object-literal-type-assertion @@ -219,13 +221,22 @@ class HistoryDocument implements TextDocument { return new Position(line + this._editOffset, ch); } + private createLines(contents: string) : TextLine[] { + let split = contents.splitLines({trim: false, removeEmptyEntries: false}); + // Skip an empty last line if there is one. + if (split && split.length > 0 && split[split.length - 1].length === 0) { + split = split.slice(0, split.length - 1); + } + return split.map((s, i) => this.createTextLine(s, i)); + } + private createTextLine(line: string, index: number) : TextLine { return new HistoryLine(line, index); } private convertToOffset(pos: Position) : number { // Combine the text length up to this position - const lenUpToPos = this._lines.filter(l => l.range.start.line < pos.line).map(l => l.range.end.character).reduce((p, c) => p + c); + const lenUpToPos = this._lines.filter(l => l.range.start.line < pos.line).map(l => l.rangeIncludingLineBreak.end.character).reduce((p, c) => p + c); // Add on the character return lenUpToPos + pos.character; From fa93010f3b6e38adeadf93e5a2af95a362506b79 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Apr 2019 10:24:38 -0700 Subject: [PATCH 08/60] Sorting --- src/client/datascience/history/history.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/client/datascience/history/history.ts b/src/client/datascience/history/history.ts index b0283114d390..d236266ab97f 100644 --- a/src/client/datascience/history/history.ts +++ b/src/client/datascience/history/history.ts @@ -1019,7 +1019,17 @@ export class History extends WebViewHost implements IHistory { } this.autoCompleteCancelSource = new CancellationTokenSource(); this.completionProvider.provideCompletionItems(request.line, request.ch, this.autoCompleteCancelSource.token).then(items => { - this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {items: items.map(i => i.insertText as string), line: request.line, ch: request.ch, id: request.id}).ignoreErrors(); + // Sort the items by kind and insertText + const sorted = items.sort((a, b) => { + if (a.kind === b.kind || !a.kind || !b.kind) { + const aStr = a.insertText ? a.insertText.toString() : ''; + const bStr = b.insertText ? b.insertText.toString() : ''; + return aStr.localeCompare(bStr); + } else { + return a.kind - b.kind; + } + }).map(i => i.insertText as string); + this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {items: sorted, line: request.line, ch: request.ch, id: request.id}).ignoreErrors(); }).catch(_e => { this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {items: [], line: request.line, ch: request.ch, id: request.id}).ignoreErrors(); }); From 0f738aea971d2f37e21173ed8d2730b306d3325e Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Apr 2019 14:05:26 -0700 Subject: [PATCH 09/60] Reverse sort kind --- src/client/datascience/history/history.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/history/history.ts b/src/client/datascience/history/history.ts index d236266ab97f..d2128a70e778 100644 --- a/src/client/datascience/history/history.ts +++ b/src/client/datascience/history/history.ts @@ -1026,7 +1026,7 @@ export class History extends WebViewHost implements IHistory { const bStr = b.insertText ? b.insertText.toString() : ''; return aStr.localeCompare(bStr); } else { - return a.kind - b.kind; + return b.kind - a.kind; // Reverse sort kind } }).map(i => i.insertText as string); this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {items: sorted, line: request.line, ch: request.ch, id: request.id}).ignoreErrors(); From d8bacb02ddfa3c7163930ed954a9520067deb06b Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 25 Apr 2019 10:17:00 -0700 Subject: [PATCH 10/60] Getting monaco to show context menu and completion --- package-lock.json | 9 + package.json | 3 +- src/client/common/extensions.ts | 13 - src/datascience-ui/history-react/code.css | 31 +- src/datascience-ui/history-react/code.tsx | 405 ++++++------------ .../history-react/mainPanelState.ts | 2 +- typings/extensions.d.ts | 4 - webpack.datascience-ui.config.js | 6 +- 8 files changed, 154 insertions(+), 319 deletions(-) diff --git a/package-lock.json b/package-lock.json index d29b115d7063..de092d8b09e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10611,6 +10611,15 @@ "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.16.2.tgz", "integrity": "sha512-NtGrFzf54jADe7qsWh3lazhS7Kj0XHkJUGBq9fA/Jbwc+sgVcyfsYF6z2AQ7hPqDC+JmdOt/OwFjBnRwqXtx6w==" }, + "monaco-editor-webpack-plugin": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-1.7.0.tgz", + "integrity": "sha512-oItymcnlL14Sjd7EF7q+CMhucfwR/2BxsqrXIBrWL6LQplFfAfV+grLEQRmVHeGSBZ/Gk9ptzfueXnWcoEcFuA==", + "dev": true, + "requires": { + "@types/webpack": "^4.4.19" + } + }, "moo": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/moo/-/moo-0.4.3.tgz", diff --git a/package.json b/package.json index 293c45c83453..0750aa6576e8 100644 --- a/package.json +++ b/package.json @@ -2181,11 +2181,11 @@ "@babel/core": "^7.1.0", "@babel/preset-env": "^7.1.0", "@babel/preset-react": "^7.0.0", + "@nteract/plotly": "^1.47.1", "@nteract/transform-dataresource": "^4.3.5", "@nteract/transform-geojson": "^3.2.3", "@nteract/transform-model-debug": "^3.2.3", "@nteract/transform-plotly": "^5.0.0", - "@nteract/plotly": "^1.47.1", "@nteract/transforms": "^4.4.4", "@types/chai": "^4.1.2", "@types/chai-arrays": "^1.0.2", @@ -2271,6 +2271,7 @@ "mocha": "^6.0.2", "mocha-junit-reporter": "^1.17.0", "mocha-multi-reporters": "^1.1.7", + "monaco-editor-webpack-plugin": "^1.7.0", "node-has-native-dependencies": "^1.0.2", "node-html-parser": "^1.1.13", "node-sass": "^4.11.0", diff --git a/src/client/common/extensions.ts b/src/client/common/extensions.ts index 7546f7232955..72a09e5a49db 100644 --- a/src/client/common/extensions.ts +++ b/src/client/common/extensions.ts @@ -95,10 +95,6 @@ declare interface Promise { * Catches task error and ignores them. */ ignoreErrors(): void; - /** - * Catches task error and logs them. - */ - logErrors(): void; } /** @@ -109,15 +105,6 @@ Promise.prototype.ignoreErrors = function (this: Promise) { this.catch(() => { }); }; -/** - * Explicitly tells that promise should be run asynchonously. - */ -Promise.prototype.logErrors = function (this: Promise) { - // tslint:disable-next-line: no-require-imports - const logger = require('./logger'); - this.catch((e) => { logger.traceError(e); }); -}; - if (!String.prototype.format) { String.prototype.format = function (this: string) { const args = arguments; diff --git a/src/datascience-ui/history-react/code.css b/src/datascience-ui/history-react/code.css index d9336d38cba2..15aad5ab2b53 100644 --- a/src/datascience-ui/history-react/code.css +++ b/src/datascience-ui/history-react/code.css @@ -38,32 +38,6 @@ margin-bottom: 10px; } -.code-area .monaco-editor -{ - overflow: hidden; -} - -.code-area .monaco-editor .slider -{ - margin: 0; -} - -.code-area .monaco-editor .parameter-hints-widget { - border: 0px; - } - .code-area .monaco-editor .parameter-hints-widget .signature { - padding: 0px; - } - .code-area .monaco-editor .suggest-widget { - border: 0px; - } - .code-area .monaco-editor.vs-dark .suggest-widget { - border: 0px; - } - .code-area .monaco-editor .rename-box { - top: 0; - } - .code-watermark { position: absolute; top: 0; @@ -72,3 +46,8 @@ font-style: italic; color: var(--override-watermark-color, var(--vscode-pickerGroup-border)); } + +.measure-width-div { + width: 100%; + visibility: none; +} diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index 4dbf6928d3a1..8b02ca67061d 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -1,25 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; - -import 'codemirror/lib/codemirror.css'; -import 'codemirror/mode/python/python'; - -import * as CodeMirror from 'codemirror'; +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; -import * as RCM from 'react-codemirror'; -import * as uuid from 'uuid/v4'; +// tslint:disable-next-line: import-name +import MonacoEditor from 'react-monaco-editor'; -// tslint:disable-next-line: no-require-imports no-var-requires -require('codemirror/addon/hint/show-hint'); -import 'codemirror/addon/hint/show-hint.css'; +import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; +import { InputHistory } from './inputHistory'; import './code.css'; -import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; -import { getLocString } from '../react-common/locReactSide'; -import { Cursor } from './cursor'; -import { InputHistory } from './inputHistory'; +const LINE_HEIGHT = 18; export interface ICodeProps { autoFocus: boolean; @@ -44,311 +36,178 @@ interface ICodeState { cursorBottom: number; charUnderCursor: string; allowWatermark: boolean; -} - -interface ICompletionResult { - list: string[]; - from: CodeMirror.Position; - to: CodeMirror.Position; - selectedHint?: number; + editor: monacoEditor.editor.IStandaloneCodeEditor | undefined; + model: monacoEditor.editor.ITextModel | null; } export class Code extends React.Component { - - private codeMirror: CodeMirror.Editor | undefined; - private codeMirrorOwner: HTMLDivElement | undefined; - private baseIndentation : number | undefined; + private containerRef: React.RefObject; + private measureWidthRef: React.RefObject; + private resizeTimer?: number; + private subscriptions: monacoEditor.IDisposable[] = []; constructor(prop: ICodeProps) { super(prop); - this.state = {focused: false, cursorLeft: 0, cursorTop: 0, cursorBottom: 0, charUnderCursor: '', allowWatermark: true}; + this.state = {focused: false, cursorLeft: 0, cursorTop: 0, cursorBottom: 0, charUnderCursor: '', allowWatermark: true, editor: undefined, model: null}; + this.containerRef = React.createRef(); + this.measureWidthRef = React.createRef(); } - public componentDidUpdate(prevProps: Readonly, _prevState: Readonly, _snapshot?: {}) { - // Force our new value. the RCM control doesn't do this correctly - if (this.codeMirror && this.props.readOnly && this.codeMirror.getValue() !== this.props.code) { - this.codeMirror.setValue(this.props.code); - } - // If we are suddenly changing a readonly to not, somebody is reusing a different control. Update - // to be empty - if (this.codeMirror && !this.props.readOnly && prevProps.readOnly) { - this.codeMirror.setOption('readOnly', false); - this.codeMirror.setValue(''); + public componentDidMount = () => { + if (window) { + window.addEventListener('resize', this.windowResized); } + this.updateEditorSize(); } - public componentWillUnmount() { - if (this.codeMirrorOwner) { - const activeElement = document.activeElement as HTMLElement; - if (activeElement && this.codeMirrorOwner.contains(activeElement)) { - activeElement.blur(); - } + public componentWillUnmount = () => { + if (this.resizeTimer) { + window.clearTimeout(this.resizeTimer); } + + if (window) { + window.removeEventListener('resize', this.windowResized); + } + + this.subscriptions.forEach(d => d.dispose()); + } + + public componentDidUpdate = () => { + this.updateEditorSize(); } public render() { const readOnly = this.props.readOnly; const classes = readOnly ? 'code-area' : 'code-area code-area-editable'; - const waterMarkClass = this.props.showWatermark && this.state.allowWatermark && !readOnly ? 'code-watermark' : 'hide'; + const options: monacoEditor.editor.IEditorConstructionOptions = { + minimap: { + enabled: false + }, + glyphMargin: false, + wordWrap: 'on', + scrollBeyondLastLine: false, + scrollbar: { + vertical: 'hidden', + horizontal: 'hidden' + }, + lineNumbers: 'off', + renderLineHighlight: 'none', + highlightActiveIndentGuide: false, + renderIndentGuides: false, + overviewRulerBorder: false, + overviewRulerLanes: 0, + hideCursorInOverviewRuler: true, + folding: false, + readOnly: readOnly + }; + return ( -
-
); diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index b0e2d36f686d..ac7287c52897 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -6,7 +6,6 @@ import * as React from 'react'; // tslint:disable-next-line: import-name import MonacoEditor from 'react-monaco-editor'; -import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; import { InputHistory } from './inputHistory'; // This next line is necessary to get webpack to load the python language settings. Otherwise @@ -27,9 +26,7 @@ export interface ICodeProps { showWatermark: boolean; onSubmit(code: string): void; onChangeLineCount(lineCount: number) : void; - onChange(fromLine: number, fromCh: number, toLine: number, toCh: number, text: string, removed?: string): void; - requestCompletionItems(line: number, ch: number, id: string) : Promise; - + onChange(changes: monacoEditor.editor.IModelContentChange[]): void; } interface ICodeState { @@ -176,6 +173,7 @@ export class Code extends React.Component { if (e.changes.length) { this.windowResized(); } + this.props.onChange(e.changes); } private compositionStart = () => { diff --git a/src/datascience-ui/history-react/completionProvider.ts b/src/datascience-ui/history-react/completionProvider.ts index 13a5cdab3e3c..4a58ce65bb3a 100644 --- a/src/datascience-ui/history-react/completionProvider.ts +++ b/src/datascience-ui/history-react/completionProvider.ts @@ -3,13 +3,78 @@ 'use strict'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; -export class CompletionProvider implements monacoEditor.languages.CompletionItemProvider { - triggerCharacters?: string[] | undefined; - provideCompletionItems(model: monacoEditor.editor.ITextModel, position: monacoEditor.Position, context: monacoEditor.languages.CompletionContext, token: monacoEditor.CancellationToken): monacoEditor.languages.ProviderResult { - throw new Error('Method not implemented.'); - } - constructor() { +import { IDisposable } from '../../client/common/types'; +import { createDeferred, Deferred } from '../../client/common/utils/async'; +import { + HistoryMessages, + IHistoryMapping, + IProvideCompletionItemsResponse +} from '../../client/datascience/history/historyTypes'; +import { IMessageHandler, PostOffice } from '../react-common/postOffice'; + +export class CompletionProvider implements monacoEditor.languages.CompletionItemProvider, IDisposable, IMessageHandler { + public triggerCharacters?: string[] | undefined; + private currentCompletionItemsRequest: Deferred | undefined; + private currentCompletionDisposable: monacoEditor.IDisposable | undefined; + private registerDisposable: monacoEditor.IDisposable; + constructor(private postOffice: PostOffice) { // Register a completion provider - monacoEditor.languages.registerCompletionItemProvider('python', this) + this.registerDisposable = monacoEditor.languages.registerCompletionItemProvider('python', this); + this.postOffice.addHandler(this); + } + + public provideCompletionItems( + _model: monacoEditor.editor.ITextModel, + position: monacoEditor.Position, + context: monacoEditor.languages.CompletionContext, + token: monacoEditor.CancellationToken): monacoEditor.languages.ProviderResult { + + // Emit a new request + const request = createDeferred(); + this.registerDisposable = token.onCancellationRequested(() => { + request.resolve(); + }); + this.currentCompletionItemsRequest = request; + this.sendMessage(HistoryMessages.ProvideCompletionItemsRequest, { position, context }); + + return request.promise; + } + + public dispose() { + this.registerDisposable.dispose(); + if (this.currentCompletionDisposable) { + this.currentCompletionDisposable.dispose(); + } + this.postOffice.removeHandler(this); + } + + // tslint:disable-next-line: no-any + public handleMessage(type: string, payload?: any): boolean { + switch (type) { + case HistoryMessages.ProvideCompletionItemsResponse: + this.handleCompletionResponse(payload); + return true; + + default: + break; + } + + return false; + } + + // Handle completion response + // tslint:disable-next-line:no-any + private handleCompletionResponse = (payload?: any) => { + if (payload) { + const response = payload as IProvideCompletionItemsResponse; + + // Resolve our waiting promise if we have one + if (this.currentCompletionItemsRequest && !this.currentCompletionItemsRequest.resolved) { + this.currentCompletionItemsRequest.resolve(response.list); + } + } + } + private sendMessage(type: T, payload?: M[T]) { + this.postOffice.sendMessage(type, payload); } -} \ No newline at end of file +} diff --git a/src/datascience-ui/history-react/contentPanel.tsx b/src/datascience-ui/history-react/contentPanel.tsx index a1f65acad36a..108396b7cc38 100644 --- a/src/datascience-ui/history-react/contentPanel.tsx +++ b/src/datascience-ui/history-react/contentPanel.tsx @@ -3,8 +3,8 @@ 'use strict'; import './contentPanel.css'; +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; -import { IProvideCompletionItemsResponse } from '../../client/datascience/history/historyTypes'; import { noop } from '../../test/core'; import { ErrorBoundary } from '../react-common/errorBoundary'; import { getSettings } from '../react-common/settingsReactSide'; @@ -24,8 +24,7 @@ export interface IContentPanelProps { gotoCellCode(index: number): void; deleteCell(index: number): void; submitInput(code: string): void; - onCodeChange(fromLine: number, fromCh: number, toLine: number, toCh: number, text: string, removed?: string): void; - requestCompletionItems(line: number, ch: number, id: string) : Promise; + onCodeChange(changes: monacoEditor.editor.IModelContentChange[]): void; } export class ContentPanel extends React.Component { @@ -84,7 +83,6 @@ export class ContentPanel extends React.Component { gotoCode={() => this.props.gotoCellCode(index)} delete={() => this.props.deleteCell(index)} onCodeChange={this.props.onCodeChange} - requestCompletionItems={this.props.requestCompletionItems} /> ); From f2f6d9bfe1e173f71f3314ed927c27bdce8cde34 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 26 Apr 2019 10:39:35 -0700 Subject: [PATCH 15/60] New IHistoryListener idea --- .../datascience/history/completionProvider.ts | 143 ++++++++++++++---- src/client/datascience/history/history.ts | 58 ++----- .../datascience/history/historyTypes.ts | 8 + src/client/datascience/serviceRegistry.ts | 4 +- src/client/datascience/types.ts | 28 ++-- src/datascience-ui/history-react/code.tsx | 4 +- .../history-react/completionProvider.ts | 8 +- 7 files changed, 156 insertions(+), 97 deletions(-) diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index 77521117df71..425e2907171f 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -9,7 +9,10 @@ import * as path from 'path'; import * as uuid from 'uuid/v4'; import { CancellationToken, + CancellationTokenSource, EndOfLine, + Event, + EventEmitter, Position, Range, TextDocument, @@ -18,14 +21,14 @@ import { Uri } from 'vscode'; import { + CompletionItem, + CompletionList, CompletionRequest, DidChangeTextDocumentNotification, DidOpenTextDocumentNotification, LanguageClient, TextDocumentItem, - VersionedTextDocumentIdentifier, - CompletionItem, - CompletionList + VersionedTextDocumentIdentifier } from 'vscode-languageclient'; import { ILanguageServer, ILanguageServerAnalysisOptions } from '../../activation/types'; @@ -33,7 +36,15 @@ import { IWorkspaceService } from '../../common/application/types'; import { PYTHON_LANGUAGE } from '../../common/constants'; import { IFileSystem, TemporaryFile } from '../../common/platform/types'; import { Identifiers } from '../constants'; -import { IHistoryCompletionProvider } from '../types'; +import { IHistoryListener } from '../types'; +import { + HistoryMessages, + ICancelCompletionItemsRequest, + IEditCell, + IHistoryMapping, + IProvideCompletionItemsRequest, + IRemoteAddCode +} from './historyTypes'; class HistoryLine implements TextLine { @@ -190,30 +201,36 @@ class HistoryDocument implements TextDocument { ]; } - public editLines(_editorChanges: monacoEditor.editor.IModelContentChange[]): TextDocumentContentChangeEvent[] { + public editLines(editorChanges: monacoEditor.editor.IModelContentChange[]): TextDocumentContentChangeEvent[] { this._version += 1; - // From and to are the offset from the beginning of a cell, not the offset of our document - // const fromLine = from.line + this._editOffset; - // const toLine = to.line + this._editOffset; - - // // Recreate our contents, and then recompute all of our lines - // const fromOffset = this.convertToOffset(new Position(fromLine, from.character)); - // const toOffset = this.convertToOffset(new Position(toLine, to.character)); - // const before = this._contents.substr(0, fromOffset); - // const after = this._contents.substr(toOffset); - // this._contents = `${before}${newCode}${after}`; - // this._lines = this.createLines(this._contents); - return [ - // tslint:disable-next-line: no-object-literal-type-assertion - // { - // range: this.createSerializableRange(new Position(fromLine, from.character), new Position(toLine, to.character)), - // // Range offset not passed by the editor so don't use it. - // rangeLength: toOffset - fromOffset, - // text: newCode - // } as TextDocumentContentChangeEvent - ]; + // Convert the range to local (and remove 1 based) + if (editorChanges && editorChanges.length) { + const fromLine = editorChanges[0].range.startLineNumber - 1 + this._editOffset; + const toLine = editorChanges[0].range.endLineNumber - 1 + this._editOffset; + const fromChar = editorChanges[0].range.startColumn - 1; + const toChar = editorChanges[0].range.endColumn - 1; + + // Recreate our contents, and then recompute all of our lines + const fromOffset = this.convertToOffset(new Position(fromLine, fromChar)); + const toOffset = this.convertToOffset(new Position(toLine, toChar)); + const before = this._contents.substr(0, fromOffset); + const after = this._contents.substr(toOffset); + this._contents = `${before}${editorChanges[0].text}${after}`; + this._lines = this.createLines(this._contents); + + return [ + // tslint:disable-next-line: no-object-literal-type-assertion + { + range: this.createSerializableRange(new Position(fromLine, fromChar), new Position(toLine, toChar)), + // Range offset not passed by the editor so don't use it. + rangeLength: toOffset - fromOffset, + text: editorChanges[0].text + } as TextDocumentContentChangeEvent + ]; + } + return []; } public convertToDocumentPosition(line: number, ch: number) : Position { @@ -256,13 +273,16 @@ class HistoryDocument implements TextDocument { } } +// tslint:disable:no-any @injectable() -export class CompletionProvider implements IHistoryCompletionProvider { +export class CompletionProvider implements IHistoryListener { private languageClient : LanguageClient | undefined; private document: HistoryDocument | undefined; private temporaryFile: TemporaryFile | undefined; private sentOpenDocument : boolean = false; + private postEmitter: EventEmitter<{message: string; payload: any}> = new EventEmitter<{message: string; payload: any}>(); + private cancellationSources : { [key: string] : CancellationTokenSource } = {}; constructor( @inject(ILanguageServer) private languageServer: ILanguageServer, @@ -276,7 +296,61 @@ export class CompletionProvider implements IHistoryCompletionProvider { this.languageServer.dispose(); } - public async startup(resource?: Uri) : Promise { + public get postMessage(): Event<{message: string; payload: any}> { + return this.postEmitter.event; + } + + public onMessage(message: string, payload?: any) { + switch (message) { + case HistoryMessages.CancelCompletionItemsRequest: + this.dispatchMessage(message, payload, this.handleCompletionItemsCancel); + break; + + case HistoryMessages.ProvideCompletionItemsRequest: + this.dispatchMessage(message, payload, this.handleCompletionItemsRequest); + break; + + case HistoryMessages.EditCell: + this.dispatchMessage(message, payload, this.editCell); + break; + + case HistoryMessages.RemoteAddCode: // Might want to rethink this. Seems weird. + this.dispatchMessage(message, payload, this.addCell); + break; + + default: + break; + } + } + + private dispatchMessage(_message: T, payload: any, handler: (args : M[T]) => void) { + const args = payload as M[T]; + handler.bind(this)(args); + } + + private postResponse(type: T, payload?: M[T]) : void { + this.postEmitter.fire({message: type.toString(), payload}); + } + + private handleCompletionItemsCancel(request: ICancelCompletionItemsRequest) { + const cancelSource = this.cancellationSources[request.id]; + if (cancelSource) { + cancelSource.cancel(); + cancelSource.dispose(); + } + } + + private handleCompletionItemsRequest(request: IProvideCompletionItemsRequest) { + const cancelSource = new CancellationTokenSource(); + this.cancellationSources[request.id] = cancelSource; + this.provideCompletionItems(request.position, request.context, cancelSource.token).then(list => { + this.postResponse(HistoryMessages.ProvideCompletionItemsResponse, {list, id: request.id}); + }).catch(_e => { + this.postResponse(HistoryMessages.ProvideCompletionItemsResponse, {list: { suggestions: [], incomplete: true }, id: request.id}); + }); + } + + private async startup(resource?: Uri) : Promise { // Save our language client. We'll use this to talk to the language server const options = await this.analysisOptions!.getAnalysisOptions(); await this.languageServer.start(resource, options); @@ -294,7 +368,7 @@ export class CompletionProvider implements IHistoryCompletionProvider { this.document = new HistoryDocument(dummyFilePath); } - public async provideCompletionItems(position: monacoEditor.Position, context: monacoEditor.languages.CompletionContext, token: CancellationToken) : Promise { + private async provideCompletionItems(position: monacoEditor.Position, context: monacoEditor.languages.CompletionContext, token: CancellationToken) : Promise { if (this.languageClient && this.document) { const docPos = this.document.convertToDocumentPosition(position.lineNumber, position.column); const result = await this.languageClient.sendRequest( @@ -309,13 +383,13 @@ export class CompletionProvider implements IHistoryCompletionProvider { incomplete: true }; } - public async addCell(code: string, file: string): Promise { + private async addCell(request: IRemoteAddCode): Promise { if (!this.languageClient) { - await this.startup(file === Identifiers.EmptyFileName ? undefined : Uri.file(file)); + await this.startup(request.file === Identifiers.EmptyFileName ? undefined : Uri.file(request.file)); } let changes: TextDocumentContentChangeEvent[] = []; if (this.document) { - changes = this.document.addLines(code); + changes = this.document.addLines(request.code); } // Broadcast an update to the language server @@ -328,10 +402,13 @@ export class CompletionProvider implements IHistoryCompletionProvider { } } } - public async editCell(editorChanges: monacoEditor.editor.IModelContentChange[]): Promise { + private async editCell(request: IEditCell): Promise { + if (!this.languageClient) { + await this.startup(undefined); + } let changes: TextDocumentContentChangeEvent[] = []; if (this.document) { - changes = this.document.editLines(editorChanges); + changes = this.document.editLines(request.changes); } // Broadcast an update to the language server diff --git a/src/client/datascience/history/history.ts b/src/client/datascience/history/history.ts index 446c4160a434..b58d3a91abf0 100644 --- a/src/client/datascience/history/history.ts +++ b/src/client/datascience/history/history.ts @@ -4,11 +4,11 @@ import '../../common/extensions'; import * as fs from 'fs-extra'; -import { inject, injectable } from 'inversify'; +import { inject, injectable, multiInject } from 'inversify'; import * as path from 'path'; import * as uuid from 'uuid/v4'; import { Event, EventEmitter, Position, Range, Selection, TextEditor, Uri, ViewColumn } from 'vscode'; -import { CancellationTokenSource, Disposable } from 'vscode-jsonrpc'; +import { Disposable } from 'vscode-jsonrpc'; import * as vsls from 'vsls/vscode'; import { @@ -39,8 +39,8 @@ import { IConnection, IDataViewerProvider, IHistory, - IHistoryCompletionProvider, IHistoryInfo, + IHistoryListener, IHistoryProvider, IJupyterExecution, IJupyterVariable, @@ -54,16 +54,7 @@ import { } from '../types'; import { WebViewHost } from '../webViewHost'; import { HistoryMessageListener } from './historyMessageListener'; -import { - HistoryMessages, - IAddedSysInfo, - IEditCell, - IGotoCode, - IHistoryMapping, - IProvideCompletionItemsRequest, - IRemoteAddCode, - ISubmitNewCell -} from './historyTypes'; +import { HistoryMessages, IAddedSysInfo, IGotoCode, IHistoryMapping, IRemoteAddCode, ISubmitNewCell } from './historyTypes'; export enum SysInfoReason { Start, @@ -86,9 +77,9 @@ export class History extends WebViewHost implements IHistory { private jupyterServer: INotebookServer | undefined; private id : string; private executeEvent: EventEmitter = new EventEmitter(); - private autoCompleteCancelSource: CancellationTokenSource | undefined; constructor( + @multiInject(IHistoryListener) private readonly listeners: IHistoryListener[], @inject(ILiveShareApi) private liveShare : ILiveShareApi, @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(IDocumentManager) private documentManager: IDocumentManager, @@ -107,8 +98,7 @@ export class History extends WebViewHost implements IHistory { @inject(IWorkspaceService) workspaceService: IWorkspaceService, @inject(IHistoryProvider) private historyProvider: IHistoryProvider, @inject(IDataViewerProvider) private dataExplorerProvider: IDataViewerProvider, - @inject(IJupyterVariables) private jupyterVariables: IJupyterVariables, - @inject(IHistoryCompletionProvider) private completionProvider: IHistoryCompletionProvider + @inject(IJupyterVariables) private jupyterVariables: IJupyterVariables ) { super( configuration, @@ -140,6 +130,9 @@ export class History extends WebViewHost implements IHistory { // Load on a background thread. this.loadPromise = this.load(); + + // For each listener sign up for their post events + this.listeners.forEach(l => l.postMessage((e) => this.postMessageInternal(e.message, e.payload))); } public get ready() : Promise { @@ -252,18 +245,15 @@ export class History extends WebViewHost implements IHistory { this.dispatchMessage(message, payload, this.requestVariableValue); break; - case HistoryMessages.ProvideCompletionItemsRequest: - this.dispatchMessage(message, payload, this.provideCompletionItems); - break; - - case HistoryMessages.EditCell: - this.dispatchMessage(message, payload, this.editCell); - break; - default: break; } + // Let our listeners handle the message too + if (this.listeners) { + this.listeners.forEach(l => l.onMessage(message, payload)); + } + // Pass onto our base class. super.onMessage(message, payload); @@ -286,6 +276,7 @@ export class History extends WebViewHost implements IHistory { super.dispose(); if (!this.disposed) { this.disposed = true; + this.listeners.forEach(l => l.dispose()); if (this.interpreterChangedDisposable) { this.interpreterChangedDisposable.dispose(); } @@ -611,9 +602,6 @@ export class History extends WebViewHost implements IHistory { // Wait for the cell to finish await finishedAddingCode.promise; traceInfo(`Finished execution for ${id}`); - - // Add this to our completion provider so it can remember this code - await this.completionProvider.addCell(code, file); } } catch (err) { status.dispose(); @@ -1012,20 +1000,4 @@ export class History extends WebViewHost implements IHistory { sendTelemetryEvent(Telemetry.VariableExplorerToggled, undefined, { open: openValue }); } } - - private provideCompletionItems(request: IProvideCompletionItemsRequest) { - if (this.autoCompleteCancelSource) { - this.autoCompleteCancelSource.cancel(); - } - this.autoCompleteCancelSource = new CancellationTokenSource(); - this.completionProvider.provideCompletionItems(request.position, request.context, this.autoCompleteCancelSource.token).then(list => { - this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {list}).ignoreErrors(); - }).catch(_e => { - this.postMessage(HistoryMessages.ProvideCompletionItemsResponse, {list: { suggestions: [], incomplete: true }}).ignoreErrors(); - }); - } - - private editCell(request: IEditCell) { - this.completionProvider.editCell(request.changes).ignoreErrors(); - } } diff --git a/src/client/datascience/history/historyTypes.ts b/src/client/datascience/history/historyTypes.ts index e19ce8fa773e..494bbf82c671 100644 --- a/src/client/datascience/history/historyTypes.ts +++ b/src/client/datascience/history/historyTypes.ts @@ -38,6 +38,7 @@ export namespace HistoryMessages { export const GetVariableValueResponse = 'get_variable_value_response'; export const VariableExplorerToggle = 'variable_explorer_toggle'; export const ProvideCompletionItemsRequest = 'provide_completion_items_request'; + export const CancelCompletionItemsRequest = 'cancel_completion_items_request'; export const ProvideCompletionItemsResponse = 'provide_completion_items_response'; export const EditCell = 'edit_cell'; } @@ -78,10 +79,16 @@ export interface ISubmitNewCell { export interface IProvideCompletionItemsRequest { position: monacoEditor.Position; context: monacoEditor.languages.CompletionContext; + id: string; +} + +export interface ICancelCompletionItemsRequest { + id: string; } export interface IProvideCompletionItemsResponse { list: monacoEditor.languages.CompletionList; + id: string; } export interface IPosition { @@ -128,6 +135,7 @@ export class IHistoryMapping { public [CssMessages.GetCssRequest] : IGetCssRequest; public [CssMessages.GetCssResponse] : IGetCssResponse; public [HistoryMessages.ProvideCompletionItemsRequest] : IProvideCompletionItemsRequest; + public [HistoryMessages.CancelCompletionItemsRequest] : ICancelCompletionItemsRequest; public [HistoryMessages.ProvideCompletionItemsResponse] : IProvideCompletionItemsResponse; public [HistoryMessages.EditCell] : IEditCell; } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 1afbf6713ca9..81ce881df11c 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -32,7 +32,7 @@ import { IDataViewer, IDataViewerProvider, IHistory, - IHistoryCompletionProvider, + IHistoryListener, IHistoryProvider, IJupyterCommandFactory, IJupyterExecution, @@ -65,5 +65,5 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IDataViewerProvider, DataViewerProvider); serviceManager.add(IDataViewer, DataViewer); serviceManager.addSingleton(IExtensionActivationService, Decorator); - serviceManager.addSingleton(IHistoryCompletionProvider, CompletionProvider); + serviceManager.addSingleton(IHistoryListener, CompletionProvider); } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 29df64ba6bbb..d8865ef9ec4f 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -4,7 +4,6 @@ import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services/lib/kernel'; import { JSONObject } from '@phosphor/coreutils'; -import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import { Observable } from 'rxjs/Observable'; import { CancellationToken, @@ -158,25 +157,24 @@ export interface IHistory extends Disposable { exportCells(): void; } -export const IHistoryCompletionProvider = Symbol('IHistoryCompletionProvider'); +export const IHistoryListener = Symbol('IHistoryListener'); /** - * Provides completion for the editor in the History window + * Listens to history messages to provide extra functionality */ -export interface IHistoryCompletionProvider extends IDisposable { +export interface IHistoryListener extends IDisposable { /** - * Gets a list of completion items for the editor - * @param position - * @param context - * @param token + * Fires this event when posting a response message */ - provideCompletionItems( - position: monacoEditor.Position, - context: monacoEditor.languages.CompletionContext, - token: CancellationToken - ) : Promise; - addCell(code: string, file: string): Promise; - editCell(changes: monacoEditor.editor.IModelContentChange[]): Promise; + // tslint:disable-next-line: no-any + postMessage: Event<{message: string; payload: any}>; + /** + * Handles messages that the history window receives + * @param message message type + * @param payload message payload + */ + // tslint:disable-next-line: no-any + onMessage(message: string, payload?: any): void; } // Wraps the vscode API in order to send messages back and forth from a webview diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index ac7287c52897..e4fb2a0d5a0f 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -173,7 +173,9 @@ export class Code extends React.Component { if (e.changes.length) { this.windowResized(); } - this.props.onChange(e.changes); + if (!this.props.readOnly) { + this.props.onChange(e.changes); + } } private compositionStart = () => { diff --git a/src/datascience-ui/history-react/completionProvider.ts b/src/datascience-ui/history-react/completionProvider.ts index 4a58ce65bb3a..d054873a5027 100644 --- a/src/datascience-ui/history-react/completionProvider.ts +++ b/src/datascience-ui/history-react/completionProvider.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. 'use strict'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; - +import * as uuid from 'uuid/v4'; import { IDisposable } from '../../client/common/types'; import { createDeferred, Deferred } from '../../client/common/utils/async'; import { @@ -13,7 +13,7 @@ import { import { IMessageHandler, PostOffice } from '../react-common/postOffice'; export class CompletionProvider implements monacoEditor.languages.CompletionItemProvider, IDisposable, IMessageHandler { - public triggerCharacters?: string[] | undefined; + public triggerCharacters?: string[] | undefined = ['.']; private currentCompletionItemsRequest: Deferred | undefined; private currentCompletionDisposable: monacoEditor.IDisposable | undefined; private registerDisposable: monacoEditor.IDisposable; @@ -30,12 +30,14 @@ export class CompletionProvider implements monacoEditor.languages.CompletionItem token: monacoEditor.CancellationToken): monacoEditor.languages.ProviderResult { // Emit a new request + const id = uuid(); const request = createDeferred(); this.registerDisposable = token.onCancellationRequested(() => { request.resolve(); + this.sendMessage(HistoryMessages.CancelCompletionItemsRequest, { id }); }); this.currentCompletionItemsRequest = request; - this.sendMessage(HistoryMessages.ProvideCompletionItemsRequest, { position, context }); + this.sendMessage(HistoryMessages.ProvideCompletionItemsRequest, { position, context, id }); return request.promise; } From 939dd5e6fa365eb0e31f8a5b7196b6da9103c07c Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 26 Apr 2019 10:56:10 -0700 Subject: [PATCH 16/60] Fix document position --- src/client/datascience/history/completionProvider.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index 425e2907171f..7904de905846 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -234,7 +234,8 @@ class HistoryDocument implements TextDocument { } public convertToDocumentPosition(line: number, ch: number) : Position { - return new Position(line + this._editOffset, ch); + // Monaco is 1 based, and we need to add in our cell offset. + return new Position(line - 1 + this._editOffset, ch - 1); } private createLines(contents: string) : TextLine[] { From 54b9f6b457e81c9816cc706a88097dddb534868d Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 26 Apr 2019 13:34:52 -0700 Subject: [PATCH 17/60] Theme support --- src/client/datascience/codeCssGenerator.ts | 225 +++++++++++++----- src/client/datascience/constants.ts | 12 +- .../datascience/history/completionProvider.ts | 3 +- src/client/datascience/types.ts | 1 + src/client/datascience/webViewHost.ts | 18 +- .../data-explorer/mainPanel.tsx | 4 +- .../history-react/MainPanel.tsx | 22 +- src/datascience-ui/history-react/cell.tsx | 2 + src/datascience-ui/history-react/code.tsx | 3 +- .../history-react/contentPanel.tsx | 2 + .../history-react/mainPanelState.ts | 1 + .../react-common/styleInjector.tsx | 29 ++- 12 files changed, 246 insertions(+), 76 deletions(-) diff --git a/src/client/datascience/codeCssGenerator.ts b/src/client/datascience/codeCssGenerator.ts index c4ec3bddb591..dc5233545ca7 100644 --- a/src/client/datascience/codeCssGenerator.ts +++ b/src/client/datascience/codeCssGenerator.ts @@ -1,15 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { JSONArray, JSONObject, JSONValue } from '@phosphor/coreutils'; +import { JSONArray, JSONObject } from '@phosphor/coreutils'; import * as fs from 'fs-extra'; import { inject, injectable } from 'inversify'; +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as path from 'path'; import * as stripJsonComments from 'strip-json-comments'; import { IWorkspaceService } from '../common/application/types'; import { IConfigurationService, ILogger } from '../common/types'; -import { DefaultTheme, Identifiers } from './constants'; +import { DefaultTheme } from './constants'; import { ICodeCssGenerator, IThemeFinder } from './types'; // tslint:disable:no-any @@ -67,6 +68,14 @@ const DefaultColors: { [key: string] : string } = { 'dark.punctuation' : '#1e1e1e' }; +interface IApplyThemeArgs { + tokenColors?: JSONArray | null; + baseColors?: JSONObject | null; + fontFamily: string; + fontSize: number; + defaultStyle: string | undefined; +} + // This class generates css using the current theme in order to colorize code. // // NOTE: This is all a big hack. It's relying on the theme json files to have a certain format @@ -82,31 +91,38 @@ export class CodeCssGenerator implements ICodeCssGenerator { @inject(ILogger) private logger: ILogger) { } - public async generateThemeCss(isDark: boolean, theme: string): Promise { - let css : string = ''; + public generateThemeCss(isDark: boolean, theme: string): Promise { + return this.applyThemeData(isDark, theme, '', this.generateCss.bind(this)); + } + + public generateMonacoTheme(isDark: boolean, theme: string) : Promise { + return this.applyThemeData(isDark, theme, {}, this.generateMonacoThemeObject.bind(this)); + } + + private async applyThemeData(isDark: boolean, theme: string, defaultT: T, applier: (args: IApplyThemeArgs) => T) : Promise { + let result = defaultT; try { // First compute our current theme. - const workbench = this.workspaceService.getConfiguration('workbench'); const ignoreTheme = this.configService.getSettings().datascience.ignoreVscodeTheme ? true : false; theme = ignoreTheme ? DefaultTheme : theme; - const terminalCursor = workbench ? workbench.get('terminal.integrated.cursorStyle', 'block') : 'block'; const editor = this.workspaceService.getConfiguration('editor', undefined); - const font = editor ? editor.get('fontFamily', 'Consolas, \'Courier New\', monospace') : 'Consolas, \'Courier New\', monospace'; + const fontFamily = editor ? editor.get('fontFamily', 'Consolas, \'Courier New\', monospace') : 'Consolas, \'Courier New\', monospace'; const fontSize = editor ? editor.get('fontSize', 14) : 14; // Then we have to find where the theme resources are loaded from if (theme) { this.logger.logInformation('Searching for token colors ...'); const tokenColors = await this.findTokenColors(theme); + const baseColors = await this.findBaseColors(theme); // The tokens object then contains the necessary data to generate our css - if (tokenColors && font && fontSize) { + if (tokenColors && fontFamily && fontSize) { this.logger.logInformation('Using colors to generate CSS ...'); - css = this.generateCss(theme, tokenColors, font, fontSize, terminalCursor, ignoreTheme ? LightTheme : undefined); - } else if (tokenColors === null && font && fontSize) { + result = applier({ tokenColors, baseColors, fontFamily, fontSize, defaultStyle: ignoreTheme ? LightTheme : undefined }); + } else if (tokenColors === null && fontFamily && fontSize) { // No colors found. See if we can figure out what type of theme we have const style = isDark ? DarkTheme : LightTheme ; - css = this.generateCss(theme, null, font, fontSize, terminalCursor, style); + result = applier({ fontFamily, fontSize, defaultStyle: style}); } } } catch (err) { @@ -114,26 +130,27 @@ export class CodeCssGenerator implements ICodeCssGenerator { this.logger.logError(err); } - return css; + return result; + } + + private getScopes(entry: any) : JSONArray { + if (entry && entry.scope) { + return Array.isArray(entry.scope) ? entry.scope as JSONArray : entry.scope.toString().split(','); + } + return []; } private matchTokenColor(tokenColors: JSONArray, scope: string) : number { return tokenColors.findIndex((entry: any) => { - if (entry) { - const scopes = entry.scope as JSONValue; - if (scopes) { - const scopeArray = Array.isArray(scope) ? scopes as JSONArray : scopes.toString().split(','); - if (scopeArray.find(v => v !== null && v !== undefined && v.toString().trim() === scope)) { - return true; - } - } + const scopeArray = this.getScopes(entry); + if (scopeArray.find(v => v !== null && v !== undefined && v.toString().trim() === scope)) { + return true; } - return false; }); } - private getScopeStyle = (tokenColors: JSONArray | null, scope: string, secondary: string, defaultStyle: string | undefined): { color: string; fontStyle: string } => { + private getScopeStyle = (tokenColors: JSONArray | null | undefined, scope: string, secondary: string, defaultStyle: string | undefined): { color: string; fontStyle: string } => { // Search through the scopes on the json object if (tokenColors) { let match = this.matchTokenColor(tokenColors, scope); @@ -161,28 +178,14 @@ export class CodeCssGenerator implements ICodeCssGenerator { } // tslint:disable-next-line:max-func-body-length - private generateCss(theme: string, tokenColors: JSONArray | null, fontFamily: string, fontSize: number, cursorType: string, defaultStyle: string | undefined): string { - const escapedThemeName = Identifiers.GeneratedThemeName; + private generateCss(args: IApplyThemeArgs): string { // There's a set of values that need to be found - const commentStyle = this.getScopeStyle(tokenColors, 'comment', 'comment', defaultStyle); - const numericStyle = this.getScopeStyle(tokenColors, 'constant.numeric', 'constant', defaultStyle); - const stringStyle = this.getScopeStyle(tokenColors, 'string', 'string', defaultStyle); - const keywordStyle = this.getScopeStyle(tokenColors, 'keyword.control', 'keyword', defaultStyle); - const operatorStyle = this.getScopeStyle(tokenColors, 'keyword.operator', 'keyword', defaultStyle); - const variableStyle = this.getScopeStyle(tokenColors, 'variable', 'variable', defaultStyle); - const entityTypeStyle = this.getScopeStyle(tokenColors, 'entity.name.type', 'entity.name.type', defaultStyle); - // const atomic = this.getScopeColor(tokenColors, 'atomic'); - const builtinStyle = this.getScopeStyle(tokenColors, 'support.function', 'support.function', defaultStyle); - const punctuationStyle = this.getScopeStyle(tokenColors, 'punctuation', 'punctuation', defaultStyle); - - const def = 'var(--vscode-editor-foreground)'; - - // Define our cursor style based on the cursor type - const cursorStyle = cursorType === 'block' ? - `{ border: 1px solid ${def}; background: ${def}; width: 5px; z-index=100; }` : cursorType === 'underline' ? - `{ border-bottom: 1px solid ${def}; z-index=100; width: 5px; }` : - `{ border-left: 1px solid ${def}; border-right: none; z-index=100; }`; + const commentStyle = this.getScopeStyle(args.tokenColors, 'comment', 'comment', args.defaultStyle); + const numericStyle = this.getScopeStyle(args.tokenColors, 'constant.numeric', 'constant', args.defaultStyle); + const stringStyle = this.getScopeStyle(args.tokenColors, 'string', 'string', args.defaultStyle); + const variableStyle = this.getScopeStyle(args.tokenColors, 'variable', 'variable', args.defaultStyle); + const entityTypeStyle = this.getScopeStyle(args.tokenColors, 'entity.name.type', 'entity.name.type', args.defaultStyle); // Use these values to fill in our format string return ` @@ -192,40 +195,63 @@ export class CodeCssGenerator implements ICodeCssGenerator { --code-string-color: ${stringStyle.color}; --code-variable-color: ${variableStyle.color}; --code-type-color: ${entityTypeStyle.color}; - --code-font-family: ${fontFamily}; - --code-font-size: ${fontSize}px; + --code-font-family: ${args.fontFamily}; + --code-font-size: ${args.fontSize}px; } - ${defaultStyle ? DefaultCssVars[defaultStyle] : undefined } - - .cm-header, .cm-strong {font-weight: bold;} - .cm-em {font-style: italic;} - .cm-link {text-decoration: underline;} - .cm-strikethrough {text-decoration: line-through;} - - .cm-s-${escapedThemeName} span.cm-keyword {color: ${keywordStyle.color}; font-style: ${keywordStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-number {color: ${numericStyle.color}; font-style: ${numericStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-def {color: ${def}; } - .cm-s-${escapedThemeName} span.cm-variable {color: ${variableStyle.color}; font-style: ${variableStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-punctuation {color: ${punctuationStyle.color}; font-style: ${punctuationStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-property, - .cm-s-${escapedThemeName} span.cm-operator {color: ${operatorStyle.color}; font-style: ${operatorStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-variable-2 {color: ${variableStyle.color}; font-style: ${variableStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-variable-3, .cm-s-${theme} .cm-type {color: ${variableStyle.color}; font-style: ${variableStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-comment {color: ${commentStyle.color}; font-style: ${commentStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-string {color: ${stringStyle.color}; font-style: ${stringStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-string-2 {color: ${stringStyle.color}; font-style: ${stringStyle.fontStyle}; } - .cm-s-${escapedThemeName} span.cm-builtin {color: ${builtinStyle.color}; font-style: ${builtinStyle.fontStyle}; } - .cm-s-${escapedThemeName} div.CodeMirror-cursor ${cursorStyle} - .cm-s-${escapedThemeName} div.CodeMirror-selected {background: var(--vscode-editor-selectionBackground) !important;} + ${args.defaultStyle ? DefaultCssVars[args.defaultStyle] : undefined } `; + } + // Based on this data here: + // https://github.com/Microsoft/vscode/blob/master/src/vs/editor/standalone/common/themes.ts#L13 + private generateMonacoThemeObject(args: IApplyThemeArgs) : monacoEditor.editor.IStandaloneThemeData { + const result: monacoEditor.editor.IStandaloneThemeData = { + base: 'vs', + inherit: false, + rules: [], + colors: {} + }; + // If we have token colors enumerate them and add them into the rules + if (args.tokenColors && args.tokenColors.length) { + args.tokenColors.forEach((t: any) => { + const scopes = this.getScopes(t); + const settings = t && t.settings ? t.settings : undefined; + if (scopes && settings) { + scopes.forEach(s => { + result.rules.push({ + token: s ? s.toString() : '', // This shouldn't happen, but checking for linter's sake. + foreground: settings.foreground, + background: settings.background, + fontStyle: settings.fontStyle + }); + }); + } + }); + } else { + // Otherwise use our default values. + result.base = args.defaultStyle === DarkTheme ? 'vs-dark' : 'vs'; + result.inherit = true; + } + // If we have base colors enumerate them and add them to the colors + if (args.baseColors) { + const keys = Object.keys(args.baseColors); + keys.forEach(k => { + const color = args.baseColors && args.baseColors[k] ? args.baseColors[k] : '#000000'; + result.colors[k] = color ? color.toString() : '#000000'; + }); + } // The else case here should end up inheriting. + return result; } private mergeColors = (colors1: JSONArray, colors2: JSONArray): JSONArray => { return [...colors1, ...colors2]; } + private mergeBaseColors = (colors1: JSONObject, colors2: JSONObject) : JSONObject => { + return {...colors1, ...colors2}; + } + private readTokenColors = async (themeFile: string): Promise => { const tokenContent = await fs.readFile(themeFile, 'utf8'); const theme = JSON.parse(stripJsonComments(tokenContent)) as JSONObject; @@ -252,6 +278,23 @@ export class CodeCssGenerator implements ICodeCssGenerator { return []; } + private readBaseColors = async (themeFile: string): Promise => { + const tokenContent = await fs.readFile(themeFile, 'utf8'); + const theme = JSON.parse(stripJsonComments(tokenContent)) as JSONObject; + const colors = theme.colors as JSONObject; + + // This theme may include others. If so we need to combine the two together + const include = theme ? theme.include : undefined; + if (include && include !== null) { + const includePath = path.join(path.dirname(themeFile), include.toString()); + const includedColors = await this.readBaseColors(includePath); + return this.mergeBaseColors(colors, includedColors); + } + + // Theme is a root, don't need to include others + return colors; + } + private findTokenColors = async (theme: string): Promise => { try { @@ -305,4 +348,54 @@ export class CodeCssGenerator implements ICodeCssGenerator { // Force the colors to the defaults return null; } + + private findBaseColors = async (theme: string): Promise => { + try { + this.logger.logInformation('Attempting search for colors ...'); + const themeRoot = await this.themeFinder.findThemeRootJson(theme); + + // Use the first result if we have one + if (themeRoot) { + this.logger.logInformation(`Loading base colors from ${themeRoot} ...`); + + // This should be the path to the file. Load it as a json object + const contents = await fs.readFile(themeRoot, 'utf8'); + const json = JSON.parse(stripJsonComments(contents)) as JSONObject; + + // There should be a theme colors section + const contributes = json.contributes as JSONObject; + + // If no contributes section, see if we have a tokenColors section. This means + // this is a direct token colors file + if (!contributes) { + return await this.readBaseColors(themeRoot); + } + + // This should have a themes section + const themes = contributes.themes as JSONArray; + + // One of these (it's an array), should have our matching theme entry + const index = themes.findIndex((e: any) => { + return e !== null && (e.id === theme || e.name === theme); + }); + + const found = index >= 0 ? themes[index] as any : null; + if (found !== null) { + // Then the path entry should contain a relative path to the json file with + // the tokens in it + const themeFile = path.join(path.dirname(themeRoot), found.path); + this.logger.logInformation(`Reading base colors from ${themeFile}`); + return await this.readBaseColors(themeFile); + } + } else { + this.logger.logWarning(`Color theme ${theme} not found. Using default colors.`); + } + } catch (err) { + // Swallow any exceptions with searching or parsing + this.logger.logError(err); + } + + // Force the colors to the defaults + return null; + } } diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 9f4343513a69..1a1480607575 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - 'use strict'; +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import { IS_WINDOWS } from '../common/platform/constants'; @@ -175,6 +175,8 @@ export namespace LiveShareCommands { export namespace CssMessages { export const GetCssRequest = 'get_css_request'; export const GetCssResponse = 'get_css_response'; + export const GetMonacoThemeRequest = 'get_monaco_theme_request'; + export const GetMonacoThemeResponse = 'get_monaco_theme_response'; } export namespace SharedMessages { @@ -186,8 +188,16 @@ export interface IGetCssRequest { isDark: boolean; } +export interface IGetMonacoThemeRequest { + isDark: boolean; +} + export interface IGetCssResponse { css: string; theme: string; knownDark?: boolean; } + +export interface IGetMonacoThemeResponse { + theme: monacoEditor.editor.IStandaloneThemeData; +} diff --git a/src/client/datascience/history/completionProvider.ts b/src/client/datascience/history/completionProvider.ts index 7904de905846..9edad36e64f8 100644 --- a/src/client/datascience/history/completionProvider.ts +++ b/src/client/datascience/history/completionProvider.ts @@ -294,7 +294,8 @@ export class CompletionProvider implements IHistoryListener { } public dispose() { - this.languageServer.dispose(); + // Actually don't dispose here. The extension does this elsewhere. + // this.languageServer.dispose(); } public get postMessage(): Event<{message: string; payload: any}> { diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index d8865ef9ec4f..d0b0c3e756e2 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -247,6 +247,7 @@ export interface ISysInfo extends nbformat.IBaseCell { export const ICodeCssGenerator = Symbol('ICodeCssGenerator'); export interface ICodeCssGenerator { generateThemeCss(isDark: boolean, theme: string) : Promise; + generateMonacoTheme(isDark: boolean, theme: string) : Promise; } export const IThemeFinder = Symbol('IThemeFinder'); diff --git a/src/client/datascience/webViewHost.ts b/src/client/datascience/webViewHost.ts index ddd95e16de6c..651178e886fe 100644 --- a/src/client/datascience/webViewHost.ts +++ b/src/client/datascience/webViewHost.ts @@ -10,7 +10,7 @@ import { IWebPanel, IWebPanelMessageListener, IWebPanelProvider, IWorkspaceServi import { traceInfo } from '../common/logger'; import { IConfigurationService, IDisposable } from '../common/types'; import { createDeferred, Deferred } from '../common/utils/async'; -import { CssMessages, DefaultTheme, IGetCssRequest, SharedMessages } from './constants'; +import { CssMessages, DefaultTheme, IGetCssRequest, IGetMonacoThemeRequest, SharedMessages } from './constants'; import { ICodeCssGenerator, IDataScienceExtraSettings, IThemeFinder } from './types'; @injectable() // For some reason this is necessary to get the class hierarchy to work. @@ -104,6 +104,10 @@ export class WebViewHost implements IDisposable { this.handleCssRequest(payload as IGetCssRequest).ignoreErrors(); break; + case CssMessages.GetMonacoThemeRequest: + this.handleMonacoThemeRequest(payload as IGetMonacoThemeRequest).ignoreErrors(); + break; + default: break; } @@ -177,6 +181,18 @@ export class WebViewHost implements IDisposable { return this.postMessageInternal(CssMessages.GetCssResponse, { css, theme: settings.extraSettings.theme, knownDark: isDark }); } + private async handleMonacoThemeRequest(request: IGetMonacoThemeRequest) : Promise { + if (!this.themeIsDarkPromise.resolved) { + this.themeIsDarkPromise.resolve(request.isDark); + } else { + this.themeIsDarkPromise = createDeferred(); + this.themeIsDarkPromise.resolve(request.isDark); + } + const settings = this.generateDataScienceExtraSettings(); + const monacoTheme = await this.cssGenerator.generateMonacoTheme(request.isDark, settings.extraSettings.theme); + return this.postMessageInternal(CssMessages.GetMonacoThemeResponse, { theme: monacoTheme }); + } + // tslint:disable-next-line:no-any private webPanelRendered() { if (!this.webPanelInit.resolved) { diff --git a/src/datascience-ui/data-explorer/mainPanel.tsx b/src/datascience-ui/data-explorer/mainPanel.tsx index 1165c9ce78be..892005ad9382 100644 --- a/src/datascience-ui/data-explorer/mainPanel.tsx +++ b/src/datascience-ui/data-explorer/mainPanel.tsx @@ -138,7 +138,9 @@ export class MainPanel extends React.Component return (
- + {this.container && this.renderGrid()}
diff --git a/src/datascience-ui/history-react/MainPanel.tsx b/src/datascience-ui/history-react/MainPanel.tsx index 4cbd86922e93..a040396dd747 100644 --- a/src/datascience-ui/history-react/MainPanel.tsx +++ b/src/datascience-ui/history-react/MainPanel.tsx @@ -108,7 +108,12 @@ export class MainPanel extends React.Component return (
- +
@@ -227,6 +232,18 @@ export class MainPanel extends React.Component } } + private monacoThemeChanged = (theme: string) => { + // update our base theme if allowed. Don't do this + // during testing as it will mess up the expected render count. + if (!this.props.testMode) { + this.setState( + { + monacoTheme: theme + } + ); + } + } + private computeBaseTheme(): string { // If we're ignoring, always light if (getSettings && getSettings().ignoreVscodeTheme) { @@ -256,7 +273,8 @@ export class MainPanel extends React.Component deleteCell: this.deleteCell, submitInput: this.submitInput, skipNextScroll: this.state.skipNextScroll ? true : false, - onCodeChange: this.codeChange + onCodeChange: this.codeChange, + monacoTheme: this.state.monacoTheme }; } private getHeaderProps = (baseTheme: string): IHeaderPanelProps => { diff --git a/src/datascience-ui/history-react/cell.tsx b/src/datascience-ui/history-react/cell.tsx index 6027dfb69ffc..9f2d1b5a3121 100644 --- a/src/datascience-ui/history-react/cell.tsx +++ b/src/datascience-ui/history-react/cell.tsx @@ -39,6 +39,7 @@ interface ICellProps { history: InputHistory | undefined; showWatermark: boolean; errorBackgroundColor: string; + monacoTheme: string | undefined; gotoCode(): void; delete(): void; submitNewCode(code: string): void; @@ -221,6 +222,7 @@ export class Cell extends React.Component { onChangeLineCount={this.onChangeLineCount} ref={this.updateCodeRef} onChange={this.props.onCodeChange} + monacoTheme={this.props.monacoTheme} /> ); diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index e4fb2a0d5a0f..a2c5f2a8999f 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -24,6 +24,7 @@ export interface ICodeProps { history: InputHistory | undefined; cursorType: string; showWatermark: boolean; + monacoTheme: string | undefined; onSubmit(code: string): void; onChangeLineCount(lineCount: number) : void; onChange(changes: monacoEditor.editor.IModelContentChange[]): void; @@ -106,7 +107,7 @@ export class Code extends React.Component {
{ gotoCode={() => this.props.gotoCellCode(index)} delete={() => this.props.deleteCell(index)} onCodeChange={this.props.onCodeChange} + monacoTheme={this.props.monacoTheme} /> ); diff --git a/src/datascience-ui/history-react/mainPanelState.ts b/src/datascience-ui/history-react/mainPanelState.ts index 672f30021fcb..3f81c7088807 100644 --- a/src/datascience-ui/history-react/mainPanelState.ts +++ b/src/datascience-ui/history-react/mainPanelState.ts @@ -26,6 +26,7 @@ export interface IMainPanelState { rootStyle?: string; theme?: string; forceDark?: boolean; + monacoTheme?: string; } // tslint:disable-next-line: no-multiline-string diff --git a/src/datascience-ui/react-common/styleInjector.tsx b/src/datascience-ui/react-common/styleInjector.tsx index 13c9f72ddfed..d28b10213e8e 100644 --- a/src/datascience-ui/react-common/styleInjector.tsx +++ b/src/datascience-ui/react-common/styleInjector.tsx @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; -import { CssMessages, IGetCssResponse, SharedMessages } from '../../client/datascience/constants'; +import { CssMessages, IGetCssResponse, IGetMonacoThemeResponse, SharedMessages } from '../../client/datascience/constants'; import { IDataScienceExtraSettings } from '../../client/datascience/types'; import { IMessageHandler, PostOffice } from './postOffice'; import { detectBaseTheme } from './themeDetector'; @@ -12,6 +13,7 @@ export interface IStyleInjectorProps { expectingDark: boolean; postOffice: PostOffice; darkChanged?(newDark: boolean): void; + monacoThemeChanged?(theme: string): void; } interface IStyleInjectorState { @@ -42,6 +44,7 @@ export class StyleInjector extends React.Component { switch (msg) { case CssMessages.GetCssResponse: - this.handleResponse(payload); + this.handleCssResponse(payload); + break; + + case CssMessages.GetMonacoThemeResponse: + this.handleMonacoThemeResponse(payload); break; case SharedMessages.UpdateSettings: @@ -75,7 +82,7 @@ export class StyleInjector extends React.Component Date: Mon, 29 Apr 2019 18:27:19 -0700 Subject: [PATCH 18/60] Colorization mostly working --- package-lock.json | 64 +- package.json | 4 +- resources/MagicPython.tmLanguage.json | 5279 +++++++++++++++++ src/client/datascience/codeCssGenerator.ts | 33 +- src/client/datascience/history/history.ts | 30 +- .../datascience/history/historyTypes.ts | 8 + src/client/datascience/themeFinder.ts | 85 +- src/client/datascience/types.ts | 1 + src/client/datascience/webViewHost.ts | 2 +- .../history-react/MainPanel.tsx | 84 +- src/datascience-ui/history-react/code.css | 5 + src/datascience-ui/history-react/code.tsx | 26 +- .../history-react/mainPanelState.ts | 1 + src/datascience-ui/history-react/tokenizer.ts | 48 + webpack.datascience-ui.config.js | 2 +- 15 files changed, 5615 insertions(+), 57 deletions(-) create mode 100644 resources/MagicPython.tmLanguage.json create mode 100644 src/datascience-ui/history-react/tokenizer.ts diff --git a/package-lock.json b/package-lock.json index de092d8b09e4..e49cf1ee3bce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3813,12 +3813,6 @@ "urlgrey": "0.4.4" } }, - "codemirror": { - "version": "5.42.2", - "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.42.2.tgz", - "integrity": "sha512-Tkv6im39VuhduFMsDA3MlXcC/kKas3Z0PI1/8N88QvFQbtOeiiwnfFJE4juGyC8/a4sb1BSxQlzsil8XLQdxRw==", - "dev": true - }, "collapse-white-space": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-1.0.4.tgz", @@ -4192,17 +4186,6 @@ "sha.js": "^2.4.8" } }, - "create-react-class": { - "version": "15.6.3", - "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", - "dev": true, - "requires": { - "fbjs": "^0.8.9", - "loose-envify": "^1.3.1", - "object-assign": "^4.1.1" - } - }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -5961,6 +5944,11 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fast-plist": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/fast-plist/-/fast-plist-0.1.2.tgz", + "integrity": "sha1-pFr/NFGWAG1AbKbNzQX2kFHvNbg=" + }, "fastparse": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/fastparse/-/fastparse-1.1.2.tgz", @@ -9971,7 +9959,6 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz", "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==", - "dev": true, "requires": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -10611,6 +10598,11 @@ "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.16.2.tgz", "integrity": "sha512-NtGrFzf54jADe7qsWh3lazhS7Kj0XHkJUGBq9fA/Jbwc+sgVcyfsYF6z2AQ7hPqDC+JmdOt/OwFjBnRwqXtx6w==" }, + "monaco-editor-textmate": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/monaco-editor-textmate/-/monaco-editor-textmate-2.1.1.tgz", + "integrity": "sha512-7jbOpjHhjJn5BYNjBSTD/yVf+Pnd6gBqr69skvFw8n1gJaUvjlVBZBCc5nrF5E8Q/4s1nOKuvqH/OvE+loDebg==" + }, "monaco-editor-webpack-plugin": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-1.7.0.tgz", @@ -10620,6 +10612,14 @@ "@types/webpack": "^4.4.19" } }, + "monaco-textmate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/monaco-textmate/-/monaco-textmate-3.0.0.tgz", + "integrity": "sha512-llE/NasQkbAEDx/RPp0ili5ZEXH4e/UkYFMACvJrEY0aybq6FVW9qySt5C4kWwRXCJDL+4ewgoTt4XO3M+bfIg==", + "requires": { + "fast-plist": "^0.1.2" + } + }, "moo": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/moo/-/moo-0.4.3.tgz", @@ -12418,6 +12418,14 @@ "mimic-fn": "^1.0.0" } }, + "onigasm": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/onigasm/-/onigasm-2.2.2.tgz", + "integrity": "sha512-TQTMk+RmPYx4sGzNAgV0q7At7PABDNHVqZBlC4aRXHg8hpCdemLOF0qq0gUCjwUbc7mhJMBOo3XpTRYwyr45Gw==", + "requires": { + "lru-cache": "^4.1.1" + } + }, "opener": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.1.tgz", @@ -13211,8 +13219,7 @@ "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", - "dev": true + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "psl": { "version": "1.1.29", @@ -13449,20 +13456,6 @@ "pure-color": "^1.2.0" } }, - "react-codemirror": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/react-codemirror/-/react-codemirror-1.0.0.tgz", - "integrity": "sha1-kUZ7U7H12A2Rai/QtMetuFqQAbo=", - "dev": true, - "requires": { - "classnames": "^2.2.5", - "codemirror": "^5.18.2", - "create-react-class": "^15.5.1", - "lodash.debounce": "^4.0.8", - "lodash.isequal": "^4.5.0", - "prop-types": "^15.5.4" - } - }, "react-color": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/react-color/-/react-color-2.14.1.tgz", @@ -18229,8 +18222,7 @@ "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", - "dev": true + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" }, "yargs": { "version": "12.0.2", diff --git a/package.json b/package.json index 0750aa6576e8..f6d3d560a4c1 100644 --- a/package.json +++ b/package.json @@ -2146,8 +2146,11 @@ "lodash": "^4.17.11", "md5": "^2.2.1", "minimatch": "^3.0.4", + "monaco-editor-textmate": "^2.1.1", + "monaco-textmate": "^3.0.0", "named-js-regexp": "^1.3.3", "node-stream-zip": "^1.6.0", + "onigasm": "^2.2.2", "pidusage": "^1.2.0", "react-monaco-editor": "^0.25.1", "reflect-metadata": "^0.1.12", @@ -2278,7 +2281,6 @@ "nyc": "^13.3.0", "raw-loader": "^0.5.1", "react": "^16.5.2", - "react-codemirror": "^1.0.0", "react-data-grid": "^6.0.2-0", "react-data-grid-addons": "^6.0.2-0", "react-dev-utils": "^5.0.2", diff --git a/resources/MagicPython.tmLanguage.json b/resources/MagicPython.tmLanguage.json new file mode 100644 index 000000000000..e51fcce1a859 --- /dev/null +++ b/resources/MagicPython.tmLanguage.json @@ -0,0 +1,5279 @@ +{ + "information_for_contributors": [ + "This file has been converted from https://github.com/MagicStack/MagicPython/blob/master/grammars/MagicPython.tmLanguage", + "If you want to provide a fix or improvement, please create a pull request against the original repository.", + "Once accepted there, we are happy to receive an update request." + ], + "version": "https://github.com/MagicStack/MagicPython/commit/8ff35b3e5fcde471fae62a57ea1ae1c7cd34c9fc", + "name": "MagicPython", + "scopeName": "source.python", + "patterns": [ + { + "include": "#statement" + }, + { + "include": "#expression" + } + ], + "repository": { + "impossible": { + "comment": "This is a special rule that should be used where no match is desired. It is not a good idea to match something like '1{0}' because in some cases that can result in infinite loops in token generation. So the rule instead matches and impossible expression to allow a match to fail and move to the next token.", + "match": "$.^" + }, + "statement": { + "patterns": [ + { + "include": "#import" + }, + { + "include": "#class-declaration" + }, + { + "include": "#function-declaration" + }, + { + "include": "#statement-keyword" + }, + { + "include": "#assignment-operator" + }, + { + "include": "#decorator" + }, + { + "include": "#docstring-statement" + }, + { + "include": "#semicolon" + } + ] + }, + "semicolon": { + "patterns": [ + { + "name": "invalid.deprecated.semicolon.python", + "match": "\\;$" + } + ] + }, + "comments": { + "patterns": [ + { + "name": "comment.line.number-sign.python", + "contentName": "meta.typehint.comment.python", + "begin": "(?x)\n (?:\n \\# \\s* (type:)\n \\s*+ (?# we want `\\s*+` which is possessive quantifier since\n we do not actually want to backtrack when matching\n whitespace here)\n (?! $ | \\#)\n )\n", + "end": "(?:$|(?=\\#))", + "beginCaptures": { + "0": { + "name": "meta.typehint.comment.python" + }, + "1": { + "name": "comment.typehint.directive.notation.python" + } + }, + "patterns": [ + { + "name": "comment.typehint.ignore.notation.python", + "match": "(?x)\n \\G ignore\n (?= \\s* (?: $ | \\#))\n" + }, + { + "name": "comment.typehint.type.notation.python", + "match": "(?x)\n (?))" + }, + { + "name": "comment.typehint.variable.notation.python", + "match": "([[:alpha:]_]\\w*)" + } + ] + }, + { + "include": "#comments-base" + } + ] + }, + "docstring-statement": { + "begin": "^(?=\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))", + "comment": "the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring", + "end": "((?<=\\1)|^)(?!\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))", + "patterns": [ + { + "include": "#docstring" + } + ] + }, + "docstring": { + "patterns": [ + { + "name": "string.quoted.docstring.multi.python", + "begin": "(\\'\\'\\'|\\\"\\\"\\\")", + "end": "(\\1)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.string.begin.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python" + } + }, + "patterns": [ + { + "include": "#docstring-prompt" + }, + { + "include": "#codetags" + }, + { + "include": "#docstring-guts-unicode" + } + ] + }, + { + "name": "string.quoted.docstring.raw.multi.python", + "begin": "([rR])(\\'\\'\\'|\\\"\\\"\\\")", + "end": "(\\2)", + "beginCaptures": { + "1": { + "name": "storage.type.string.python" + }, + "2": { + "name": "punctuation.definition.string.begin.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python" + } + }, + "patterns": [ + { + "include": "#string-consume-escape" + }, + { + "include": "#docstring-prompt" + }, + { + "include": "#codetags" + } + ] + }, + { + "name": "string.quoted.docstring.single.python", + "begin": "(\\'|\\\")", + "end": "(\\1)|(\\n)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.string.begin.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#codetags" + }, + { + "include": "#docstring-guts-unicode" + } + ] + }, + { + "name": "string.quoted.docstring.raw.single.python", + "begin": "([rR])(\\'|\\\")", + "end": "(\\2)|(\\n)", + "beginCaptures": { + "1": { + "name": "storage.type.string.python" + }, + "2": { + "name": "punctuation.definition.string.begin.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#string-consume-escape" + }, + { + "include": "#codetags" + } + ] + } + ] + }, + "docstring-guts-unicode": { + "patterns": [ + { + "include": "#escape-sequence-unicode" + }, + { + "include": "#escape-sequence" + }, + { + "include": "#string-line-continuation" + } + ] + }, + "docstring-prompt": { + "match": "(?x)\n (?:\n (?:^|\\G) \\s* (?# '\\G' is necessary for ST)\n ((?:>>>|\\.\\.\\.) \\s) (?=\\s*\\S)\n )\n", + "captures": { + "1": { + "name": "keyword.control.flow.python" + } + } + }, + "statement-keyword": { + "patterns": [ + { + "name": "storage.type.function.python", + "match": "\\b((async\\s+)?\\s*def)\\b" + }, + { + "name": "keyword.control.flow.python", + "comment": "if `as` is eventually followed by `:` or line continuation\nit's probably control flow like:\n with foo as bar, \\\n Foo as Bar:\n try:\n do_stuff()\n except Exception as e:\n pass\n", + "match": "\\b(?>= | //= | \\*\\*=\n | \\+= | -= | /= | @=\n | \\*= | %= | ~= | \\^= | &= | \\|=\n | =(?!=)\n" + }, + "operator": { + "match": "(?x)\n \\b(?> | & | \\| | \\^ | ~) (?# 3)\n\n | (\\*\\* | \\* | \\+ | - | % | // | / | @) (?# 4)\n\n | (!= | == | >= | <= | < | >) (?# 5)\n", + "captures": { + "1": { + "name": "keyword.operator.logical.python" + }, + "2": { + "name": "keyword.control.flow.python" + }, + "3": { + "name": "keyword.operator.bitwise.python" + }, + "4": { + "name": "keyword.operator.arithmetic.python" + }, + "5": { + "name": "keyword.operator.comparison.python" + } + } + }, + "punctuation": { + "patterns": [ + { + "name": "punctuation.separator.colon.python", + "match": ":" + }, + { + "name": "punctuation.separator.element.python", + "match": "," + } + ] + }, + "literal": { + "patterns": [ + { + "name": "constant.language.python", + "match": "\\b(True|False|None|NotImplemented|Ellipsis)\\b" + }, + { + "include": "#number" + } + ] + }, + "number": { + "name": "constant.numeric.python", + "patterns": [ + { + "include": "#number-float" + }, + { + "include": "#number-dec" + }, + { + "include": "#number-hex" + }, + { + "include": "#number-oct" + }, + { + "include": "#number-bin" + }, + { + "include": "#number-long" + }, + { + "name": "invalid.illegal.name.python", + "match": "\\b[0-9]+\\w+" + } + ] + }, + "number-float": { + "name": "constant.numeric.float.python", + "match": "(?x)\n (?=^]? [-+ ]? \\#?\n \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?\n })\n )\n", + "captures": { + "1": { + "name": "constant.character.format.placeholder.other.python" + }, + "3": { + "name": "storage.type.format.python" + }, + "4": { + "name": "storage.type.format.python" + } + } + }, + { + "name": "meta.format.brace.python", + "match": "(?x)\n (\n {\n \\w* (\\.[[:alpha:]_]\\w* | \\[[^\\]'\"]+\\])*\n (![rsa])?\n (:)\n [^'\"{}\\n]* (?:\n \\{ [^'\"}\\n]*? \\} [^'\"{}\\n]*\n )*\n }\n )\n", + "captures": { + "1": { + "name": "constant.character.format.placeholder.other.python" + }, + "3": { + "name": "storage.type.format.python" + }, + "4": { + "name": "storage.type.format.python" + } + } + } + ] + }, + "fstring-formatting": { + "patterns": [ + { + "include": "#fstring-formatting-braces" + }, + { + "include": "#fstring-formatting-singe-brace" + } + ] + }, + "fstring-formatting-singe-brace": { + "name": "invalid.illegal.brace.python", + "match": "(}(?!}))" + }, + "import": { + "comment": "Import statements used to correctly mark `from`, `import`, and `as`\n", + "patterns": [ + { + "begin": "\\b(?)", + "end": "(?=:)", + "beginCaptures": { + "1": { + "name": "punctuation.separator.annotation.result.python" + } + }, + "patterns": [ + { + "include": "#expression" + } + ] + }, + "item-access": { + "patterns": [ + { + "name": "meta.item-access.python", + "begin": "(?x)\n \\b(?=\n [[:alpha:]_]\\w* \\s* \\[\n )\n", + "end": "(\\])", + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.python" + } + }, + "patterns": [ + { + "include": "#item-name" + }, + { + "include": "#item-index" + }, + { + "include": "#expression" + } + ] + } + ] + }, + "item-name": { + "patterns": [ + { + "include": "#special-variables" + }, + { + "include": "#builtin-functions" + }, + { + "include": "#special-names" + }, + { + "match": "(?x)\n \\b ([[:alpha:]_]\\w*) \\b\n" + } + ] + }, + "item-index": { + "begin": "(\\[)", + "end": "(?=\\])", + "beginCaptures": { + "1": { + "name": "punctuation.definition.arguments.begin.python" + } + }, + "contentName": "meta.item-access.arguments.python", + "patterns": [ + { + "name": "punctuation.separator.slice.python", + "match": ":" + }, + { + "include": "#expression" + } + ] + }, + "decorator": { + "name": "meta.function.decorator.python", + "begin": "(?x)\n ^\\s*\n ((@)) \\s* (?=[[:alpha:]_]\\w*)\n", + "end": "(?x)\n ( \\) )\n # trailing whitespace and comments are legal\n (?: (.*?) (?=\\s*(?:\\#|$)) )\n | (?=\\n|\\#)\n", + "beginCaptures": { + "1": { + "name": "entity.name.function.decorator.python" + }, + "2": { + "name": "punctuation.definition.decorator.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.python" + }, + "2": { + "name": "invalid.illegal.decorator.python" + } + }, + "patterns": [ + { + "include": "#decorator-name" + }, + { + "include": "#function-arguments" + } + ] + }, + "decorator-name": { + "patterns": [ + { + "include": "#builtin-callables" + }, + { + "include": "#illegal-object-name" + }, + { + "name": "entity.name.function.decorator.python", + "match": "(?x)\n ([[:alpha:]_]\\w*) | (\\.)\n", + "captures": { + "2": { + "name": "punctuation.separator.period.python" + } + } + }, + { + "include": "#line-continuation" + }, + { + "name": "invalid.illegal.decorator.python", + "match": "(?x)\n \\s* ([^([:alpha:]\\s_\\.#\\\\] .*?) (?=\\#|$)\n", + "captures": { + "1": { + "name": "invalid.illegal.decorator.python" + } + } + } + ] + }, + "call-wrapper-inheritance": { + "comment": "same as a function call, but in inheritance context", + "name": "meta.function-call.python", + "begin": "(?x)\n \\b(?=\n ([[:alpha:]_]\\w*) \\s* (\\()\n )\n", + "end": "(\\))", + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.python" + } + }, + "patterns": [ + { + "include": "#inheritance-name" + }, + { + "include": "#function-arguments" + } + ] + }, + "inheritance-name": { + "patterns": [ + { + "include": "#lambda-incomplete" + }, + { + "include": "#builtin-possible-callables" + }, + { + "include": "#inheritance-identifier" + } + ] + }, + "function-call": { + "name": "meta.function-call.python", + "begin": "(?x)\n \\b(?=\n ([[:alpha:]_]\\w*) \\s* (\\()\n )\n", + "end": "(\\))", + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.python" + } + }, + "patterns": [ + { + "include": "#special-variables" + }, + { + "include": "#function-name" + }, + { + "include": "#function-arguments" + } + ] + }, + "function-name": { + "patterns": [ + { + "include": "#builtin-possible-callables" + }, + { + "comment": "Some color schemas support meta.function-call.generic scope", + "name": "meta.function-call.generic.python", + "match": "(?x)\n \\b ([[:alpha:]_]\\w*) \\b\n" + } + ] + }, + "function-arguments": { + "begin": "(\\()", + "end": "(?=\\))(?!\\)\\s*\\()", + "beginCaptures": { + "1": { + "name": "punctuation.definition.arguments.begin.python" + } + }, + "contentName": "meta.function-call.arguments.python", + "patterns": [ + { + "name": "punctuation.separator.arguments.python", + "match": "(,)" + }, + { + "match": "(?x)\n (?:(?<=[,(])|^) \\s* (\\*{1,2})\n", + "captures": { + "1": { + "name": "keyword.operator.unpacking.arguments.python" + } + } + }, + { + "include": "#lambda-incomplete" + }, + { + "include": "#illegal-names" + }, + { + "match": "\\b([[:alpha:]_]\\w*)\\s*(=)(?!=)", + "captures": { + "1": { + "name": "variable.parameter.function-call.python" + }, + "2": { + "name": "keyword.operator.assignment.python" + } + } + }, + { + "name": "keyword.operator.assignment.python", + "match": "=(?!=)" + }, + { + "include": "#expression" + }, + { + "match": "\\s*(\\))\\s*(\\()", + "captures": { + "1": { + "name": "punctuation.definition.arguments.end.python" + }, + "2": { + "name": "punctuation.definition.arguments.begin.python" + } + } + } + ] + }, + "builtin-callables": { + "patterns": [ + { + "include": "#illegal-names" + }, + { + "include": "#illegal-object-name" + }, + { + "include": "#builtin-exceptions" + }, + { + "include": "#builtin-functions" + }, + { + "include": "#builtin-types" + } + ] + }, + "builtin-possible-callables": { + "patterns": [ + { + "include": "#builtin-callables" + }, + { + "include": "#magic-names" + } + ] + }, + "builtin-exceptions": { + "name": "support.type.exception.python", + "match": "(?x) (?" + }, + "regexp-base-expression": { + "patterns": [ + { + "include": "#regexp-quantifier" + }, + { + "include": "#regexp-base-common" + } + ] + }, + "fregexp-base-expression": { + "patterns": [ + { + "include": "#fregexp-quantifier" + }, + { + "include": "#fstring-formatting-braces" + }, + { + "match": "\\{.*?\\}" + }, + { + "include": "#regexp-base-common" + } + ] + }, + "fstring-formatting-braces": { + "patterns": [ + { + "comment": "empty braces are illegal", + "match": "({)(\\s*?)(})", + "captures": { + "1": { + "name": "constant.character.format.placeholder.other.python" + }, + "2": { + "name": "invalid.illegal.brace.python" + }, + "3": { + "name": "constant.character.format.placeholder.other.python" + } + } + }, + { + "name": "constant.character.escape.python", + "match": "({{|}})" + } + ] + }, + "regexp-base-common": { + "patterns": [ + { + "name": "support.other.match.any.regexp", + "match": "\\." + }, + { + "name": "support.other.match.begin.regexp", + "match": "\\^" + }, + { + "name": "support.other.match.end.regexp", + "match": "\\$" + }, + { + "name": "keyword.operator.quantifier.regexp", + "match": "[+*?]\\??" + }, + { + "name": "keyword.operator.disjunction.regexp", + "match": "\\|" + }, + { + "include": "#regexp-escape-sequence" + } + ] + }, + "regexp-quantifier": { + "name": "keyword.operator.quantifier.regexp", + "match": "(?x)\n \\{(\n \\d+ | \\d+,(\\d+)? | ,\\d+\n )\\}\n" + }, + "fregexp-quantifier": { + "name": "keyword.operator.quantifier.regexp", + "match": "(?x)\n \\{\\{(\n \\d+ | \\d+,(\\d+)? | ,\\d+\n )\\}\\}\n" + }, + "regexp-backreference-number": { + "name": "meta.backreference.regexp", + "match": "(\\\\[1-9]\\d?)", + "captures": { + "1": { + "name": "entity.name.tag.backreference.regexp" + } + } + }, + "regexp-backreference": { + "name": "meta.backreference.named.regexp", + "match": "(?x)\n (\\() (\\?P= \\w+(?:\\s+[[:alnum:]]+)?) (\\))\n", + "captures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.backreference.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.backreference.regexp" + }, + "3": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.backreference.named.end.regexp" + } + } + }, + "regexp-flags": { + "name": "storage.modifier.flag.regexp", + "match": "\\(\\?[aiLmsux]+\\)" + }, + "regexp-escape-special": { + "name": "support.other.escape.special.regexp", + "match": "\\\\([AbBdDsSwWZ])" + }, + "regexp-escape-character": { + "name": "constant.character.escape.regexp", + "match": "(?x)\n \\\\ (\n x[0-9A-Fa-f]{2}\n | 0[0-7]{1,2}\n | [0-7]{3}\n )\n" + }, + "regexp-escape-unicode": { + "name": "constant.character.unicode.regexp", + "match": "(?x)\n \\\\ (\n u[0-9A-Fa-f]{4}\n | U[0-9A-Fa-f]{8}\n )\n" + }, + "regexp-escape-catchall": { + "name": "constant.character.escape.regexp", + "match": "\\\\(.|\\n)" + }, + "regexp-escape-sequence": { + "patterns": [ + { + "include": "#regexp-escape-special" + }, + { + "include": "#regexp-escape-character" + }, + { + "include": "#regexp-escape-unicode" + }, + { + "include": "#regexp-backreference-number" + }, + { + "include": "#regexp-escape-catchall" + } + ] + }, + "regexp-charecter-set-escapes": { + "patterns": [ + { + "name": "constant.character.escape.regexp", + "match": "\\\\[abfnrtv\\\\]" + }, + { + "include": "#regexp-escape-special" + }, + { + "name": "constant.character.escape.regexp", + "match": "\\\\([0-7]{1,3})" + }, + { + "include": "#regexp-escape-character" + }, + { + "include": "#regexp-escape-unicode" + }, + { + "include": "#regexp-escape-catchall" + } + ] + }, + "codetags": { + "match": "(?:\\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\\b)", + "captures": { + "1": { + "name": "keyword.codetag.notation.python" + } + } + }, + "comments-base": { + "name": "comment.line.number-sign.python", + "begin": "(\\#)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.comment.python" + } + }, + "end": "($)", + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "comments-string-single-three": { + "name": "comment.line.number-sign.python", + "begin": "(\\#)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.comment.python" + } + }, + "end": "($|(?='''))", + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "comments-string-double-three": { + "name": "comment.line.number-sign.python", + "begin": "(\\#)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.comment.python" + } + }, + "end": "($|(?=\"\"\"))", + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "single-one-regexp-expression": { + "patterns": [ + { + "include": "#regexp-base-expression" + }, + { + "include": "#single-one-regexp-character-set" + }, + { + "include": "#single-one-regexp-comments" + }, + { + "include": "#regexp-flags" + }, + { + "include": "#single-one-regexp-named-group" + }, + { + "include": "#regexp-backreference" + }, + { + "include": "#single-one-regexp-lookahead" + }, + { + "include": "#single-one-regexp-lookahead-negative" + }, + { + "include": "#single-one-regexp-lookbehind" + }, + { + "include": "#single-one-regexp-lookbehind-negative" + }, + { + "include": "#single-one-regexp-conditional" + }, + { + "include": "#single-one-regexp-parentheses-non-capturing" + }, + { + "include": "#single-one-regexp-parentheses" + } + ] + }, + "single-one-regexp-character-set": { + "patterns": [ + { + "match": "(?x)\n \\[ \\^? \\] (?! .*?\\])\n" + }, + { + "name": "meta.character.set.regexp", + "begin": "(\\[)(\\^)?(\\])?", + "end": "(\\]|(?=\\'))|((?=(?)\n", + "end": "(\\)|(?=\\'))|((?=(?)\n", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-comments": { + "name": "comment.regexp", + "begin": "\\(\\?#", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "punctuation.comment.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.comment.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "single-three-regexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-lookbehind-negative": { + "begin": "(\\()\\?)\n", + "end": "(\\)|(?=\"))|((?=(?)\n", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-comments": { + "name": "comment.regexp", + "begin": "\\(\\?#", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "punctuation.comment.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.comment.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "double-three-regexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-lookbehind-negative": { + "begin": "(\\()\\?)\n", + "end": "(\\)|(?=\\'))|((?=(?)\n", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookbehind-negative": { + "begin": "(\\()\\?)\n", + "end": "(\\)|(?=\"))|((?=(?)\n", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookbehind-negative": { + "begin": "(\\()\\?=^]? [-+ ]? \\#?\n \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=})\n", + "captures": { + "1": { + "name": "storage.type.format.python" + }, + "2": { + "name": "storage.type.format.python" + } + } + }, + { + "include": "#fstring-terminator-single-tail" + } + ] + }, + "fstring-terminator-single-tail": { + "begin": "(![rsa])?(:)(?=.*?{)", + "end": "(?=})|(?=\\n)", + "beginCaptures": { + "1": { + "name": "storage.type.format.python" + }, + "2": { + "name": "storage.type.format.python" + } + }, + "patterns": [ + { + "include": "#fstring-illegal-single-brace" + }, + { + "include": "#fstring-single-brace" + }, + { + "name": "storage.type.format.python", + "match": "([bcdeEfFgGnosxX%])(?=})" + }, + { + "name": "storage.type.format.python", + "match": "(\\.\\d+)" + }, + { + "name": "storage.type.format.python", + "match": "(,)" + }, + { + "name": "storage.type.format.python", + "match": "(\\d+)" + }, + { + "name": "storage.type.format.python", + "match": "(\\#)" + }, + { + "name": "storage.type.format.python", + "match": "([-+ ])" + }, + { + "name": "storage.type.format.python", + "match": "([<>=^])" + }, + { + "name": "storage.type.format.python", + "match": "(\\w)" + } + ] + }, + "fstring-fnorm-quoted-multi-line": { + "name": "meta.fstring.python", + "begin": "(\\b[fF])([bBuU])?('''|\"\"\")", + "end": "(\\3)", + "beginCaptures": { + "1": { + "name": "string.interpolated.python string.quoted.multi.python storage.type.string.python" + }, + "2": { + "name": "invalid.illegal.prefix.python" + }, + "3": { + "name": "punctuation.definition.string.begin.python string.interpolated.python string.quoted.multi.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python string.interpolated.python string.quoted.multi.python" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#fstring-guts" + }, + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "include": "#fstring-multi-core" + } + ] + }, + "fstring-normf-quoted-multi-line": { + "name": "meta.fstring.python", + "begin": "(\\b[bBuU])([fF])('''|\"\"\")", + "end": "(\\3)", + "beginCaptures": { + "1": { + "name": "invalid.illegal.prefix.python" + }, + "2": { + "name": "string.interpolated.python string.quoted.multi.python storage.type.string.python" + }, + "3": { + "name": "punctuation.definition.string.begin.python string.quoted.multi.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python string.interpolated.python string.quoted.multi.python" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#fstring-guts" + }, + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "include": "#fstring-multi-core" + } + ] + }, + "fstring-raw-quoted-multi-line": { + "name": "meta.fstring.python", + "begin": "(\\b(?:[R][fF]|[fF][R]))('''|\"\"\")", + "end": "(\\2)", + "beginCaptures": { + "1": { + "name": "string.interpolated.python string.quoted.raw.multi.python storage.type.string.python" + }, + "2": { + "name": "punctuation.definition.string.begin.python string.quoted.raw.multi.python" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python string.interpolated.python string.quoted.raw.multi.python" + }, + "2": { + "name": "invalid.illegal.newline.python" + } + }, + "patterns": [ + { + "include": "#fstring-raw-guts" + }, + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "include": "#fstring-raw-multi-core" + } + ] + }, + "fstring-multi-core": { + "name": "string.interpolated.python string.quoted.multi.python", + "match": "(?x)\n (.+?)\n (\n (?# .* and .*? in multi-line match need special handling of\n newlines otherwise SublimeText and Atom will match slightly\n differently.\n\n The guard for newlines has to be separate from the\n lookahead because of special $ matching rule.)\n ($\\n?)\n |\n (?=[\\\\\\}\\{]|'''|\"\"\")\n )\n (?# due to how multiline regexps are matched we need a special case\n for matching a newline character)\n | \\n\n" + }, + "fstring-raw-multi-core": { + "name": "string.interpolated.python string.quoted.raw.multi.python", + "match": "(?x)\n (.+?)\n (\n (?# .* and .*? in multi-line match need special handling of\n newlines otherwise SublimeText and Atom will match slightly\n differently.\n\n The guard for newlines has to be separate from the\n lookahead because of special $ matching rule.)\n ($\\n?)\n |\n (?=[\\\\\\}\\{]|'''|\"\"\")\n )\n (?# due to how multiline regexps are matched we need a special case\n for matching a newline character)\n | \\n\n" + }, + "fstring-multi-brace": { + "comment": "value interpolation using { ... }", + "begin": "(\\{)", + "end": "(?x)\n (\\})\n", + "beginCaptures": { + "1": { + "name": "constant.character.format.placeholder.other.python" + } + }, + "endCaptures": { + "1": { + "name": "constant.character.format.placeholder.other.python" + } + }, + "patterns": [ + { + "include": "#fstring-terminator-multi" + }, + { + "include": "#f-expression" + } + ] + }, + "fstring-terminator-multi": { + "patterns": [ + { + "name": "storage.type.format.python", + "match": "(![rsa])(?=})" + }, + { + "match": "(?x)\n (![rsa])?\n ( : \\w? [<>=^]? [-+ ]? \\#?\n \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=})\n", + "captures": { + "1": { + "name": "storage.type.format.python" + }, + "2": { + "name": "storage.type.format.python" + } + } + }, + { + "include": "#fstring-terminator-multi-tail" + } + ] + }, + "fstring-terminator-multi-tail": { + "begin": "(![rsa])?(:)(?=.*?{)", + "end": "(?=})", + "beginCaptures": { + "1": { + "name": "storage.type.format.python" + }, + "2": { + "name": "storage.type.format.python" + } + }, + "patterns": [ + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "name": "storage.type.format.python", + "match": "([bcdeEfFgGnosxX%])(?=})" + }, + { + "name": "storage.type.format.python", + "match": "(\\.\\d+)" + }, + { + "name": "storage.type.format.python", + "match": "(,)" + }, + { + "name": "storage.type.format.python", + "match": "(\\d+)" + }, + { + "name": "storage.type.format.python", + "match": "(\\#)" + }, + { + "name": "storage.type.format.python", + "match": "([-+ ])" + }, + { + "name": "storage.type.format.python", + "match": "([<>=^])" + }, + { + "name": "storage.type.format.python", + "match": "(\\w)" + } + ] + } + } +} \ No newline at end of file diff --git a/src/client/datascience/codeCssGenerator.ts b/src/client/datascience/codeCssGenerator.ts index dc5233545ca7..51b4dcbddb63 100644 --- a/src/client/datascience/codeCssGenerator.ts +++ b/src/client/datascience/codeCssGenerator.ts @@ -214,20 +214,41 @@ export class CodeCssGenerator implements ICodeCssGenerator { }; // If we have token colors enumerate them and add them into the rules if (args.tokenColors && args.tokenColors.length) { + const tokenSet = new Set(); args.tokenColors.forEach((t: any) => { const scopes = this.getScopes(t); const settings = t && t.settings ? t.settings : undefined; if (scopes && settings) { scopes.forEach(s => { - result.rules.push({ - token: s ? s.toString() : '', // This shouldn't happen, but checking for linter's sake. - foreground: settings.foreground, - background: settings.background, - fontStyle: settings.fontStyle - }); + const token = s ? s.toString() : ''; + if (!tokenSet.has(token)) { + tokenSet.add(token); + result.rules.push({ + token, + foreground: settings.foreground, + background: settings.background, + fontStyle: settings.fontStyle + }); + + // Special case some items. punctuation.definition.comment doesn't seem to + // be listed anywhere. Add it manually when we find a 'comment' + // tslint:disable-next-line: possible-timing-attack + if (token === 'comment') { + result.rules.push({ + token: 'punctuation.definition.comment', + foreground: settings.foreground, + background: settings.background, + fontStyle: settings.fontStyle + }); + } + } }); } }); + + result.rules = result.rules.sort((a: monacoEditor.editor.ITokenThemeRule, b: monacoEditor.editor.ITokenThemeRule) => { + return a.token.localeCompare(b.token); + }); } else { // Otherwise use our default values. result.base = args.defaultStyle === DarkTheme ? 'vs-dark' : 'vs'; diff --git a/src/client/datascience/history/history.ts b/src/client/datascience/history/history.ts index b58d3a91abf0..656e722faf80 100644 --- a/src/client/datascience/history/history.ts +++ b/src/client/datascience/history/history.ts @@ -20,7 +20,7 @@ import { IWorkspaceService } from '../../common/application/types'; import { CancellationError } from '../../common/cancellation'; -import { EXTENSION_ROOT_DIR } from '../../common/constants'; +import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../../common/constants'; import { ContextKey } from '../../common/contextKey'; import { traceInfo } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; @@ -245,6 +245,14 @@ export class History extends WebViewHost implements IHistory { this.dispatchMessage(message, payload, this.requestVariableValue); break; + case HistoryMessages.LoadTmLanguageRequest: + this.dispatchMessage(message, payload, this.requestTmLanguage); + break; + + case HistoryMessages.LoadOnigasmAssemblyRequest: + this.dispatchMessage(message, payload, this.requestOnigasm); + break; + default: break; } @@ -1000,4 +1008,24 @@ export class History extends WebViewHost implements IHistory { sendTelemetryEvent(Telemetry.VariableExplorerToggled, undefined, { open: openValue }); } } + + private requestTmLanguage = () => { + // Get the contents of the appropriate tmLanguage file. + this.themeFinder.findTmLanguage(PYTHON_LANGUAGE).then(s => { + this.postMessage(HistoryMessages.LoadTmLanguageResponse, s).ignoreErrors(); + }).catch(_e => { + this.postMessage(HistoryMessages.LoadTmLanguageResponse, undefined).ignoreErrors(); + }); + } + + private requestOnigasm = async () : Promise => { + // This should just be in node_modules + const filePath = path.join(EXTENSION_ROOT_DIR, 'node_modules', 'onigasm', 'lib', 'onigasm.wasm'); + if (await this.fileSystem.fileExists(filePath)) { + const contents = await fs.readFile(filePath); + this.postMessage(HistoryMessages.LoadOnigasmAssemblyResponse, contents).ignoreErrors(); + } else { + this.postMessage(HistoryMessages.LoadOnigasmAssemblyResponse, undefined).ignoreErrors(); + } + } } diff --git a/src/client/datascience/history/historyTypes.ts b/src/client/datascience/history/historyTypes.ts index 494bbf82c671..7cb9d9a383ee 100644 --- a/src/client/datascience/history/historyTypes.ts +++ b/src/client/datascience/history/historyTypes.ts @@ -41,6 +41,10 @@ export namespace HistoryMessages { export const CancelCompletionItemsRequest = 'cancel_completion_items_request'; export const ProvideCompletionItemsResponse = 'provide_completion_items_response'; export const EditCell = 'edit_cell'; + export const LoadOnigasmAssemblyRequest = 'load_onigasm_assembly_request'; + export const LoadOnigasmAssemblyResponse = 'load_onigasm_assembly_response'; + export const LoadTmLanguageRequest = 'load_tmlanguage_request'; + export const LoadTmLanguageResponse = 'load_tmlanguage_response'; } // These are the messages that will mirror'd to guest/hosts in @@ -138,4 +142,8 @@ export class IHistoryMapping { public [HistoryMessages.CancelCompletionItemsRequest] : ICancelCompletionItemsRequest; public [HistoryMessages.ProvideCompletionItemsResponse] : IProvideCompletionItemsResponse; public [HistoryMessages.EditCell] : IEditCell; + public [HistoryMessages.LoadOnigasmAssemblyRequest]: never | undefined; + public [HistoryMessages.LoadOnigasmAssemblyResponse]: Buffer; + public [HistoryMessages.LoadTmLanguageRequest]: never | undefined; + public [HistoryMessages.LoadTmLanguageResponse]: string | undefined; } diff --git a/src/client/datascience/themeFinder.ts b/src/client/datascience/themeFinder.ts index a1b623ec9dd0..05155f9bf5a4 100644 --- a/src/client/datascience/themeFinder.ts +++ b/src/client/datascience/themeFinder.ts @@ -6,7 +6,9 @@ import * as glob from 'glob'; import { inject, injectable } from 'inversify'; import * as path from 'path'; +import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../common/constants'; import { ICurrentProcess, IExtensions, ILogger } from '../common/types'; +import { IThemeFinder } from './types'; // tslint:disable:no-any @@ -16,8 +18,9 @@ interface IThemeData { } @injectable() -export class ThemeFinder { +export class ThemeFinder implements IThemeFinder { private themeCache : { [key: string] : IThemeData | undefined } = {}; + private languageCache: { [key: string] : string | undefined } = {}; constructor( @inject(IExtensions) private extensions: IExtensions, @@ -35,6 +38,18 @@ export class ThemeFinder { } } + public async findTmLanguage(language: string) : Promise { + // See if already found it or not + if (!this.themeCache.hasOwnProperty(language)) { + try { + this.languageCache[language] = await this.findMatchingLanguage(language); + } catch (exc) { + this.logger.logError(exc); + } + } + return this.languageCache[language]; + } + public async isThemeDark(themeName: string) : Promise { // find our data const themeData = await this.findThemeData(themeName); @@ -57,6 +72,52 @@ export class ThemeFinder { return this.themeCache[themeName]; } + private async findMatchingLanguage(language: string) : Promise { + const currentExe = this.currentProcess.execPath; + let currentPath = path.dirname(currentExe); + + // Should be somewhere under currentPath/resources/app/extensions inside of a json file + let extensionsPath = path.join(currentPath, 'resources', 'app', 'extensions'); + if (!(await fs.pathExists(extensionsPath))) { + // Might be on mac or linux. try a different path + currentPath = path.resolve(currentPath, '../../../..'); + extensionsPath = path.join(currentPath, 'resources', 'app', 'extensions'); + } + + // Search through all of the files in this folder + let results = await this.findMatchingLanguages(language, extensionsPath); + + // If that didn't work, see if it's our MagicPython predefined tmLanguage + if (!results && language === PYTHON_LANGUAGE) { + results = await fs.readFile(path.join(EXTENSION_ROOT_DIR, 'resources', 'MagicPython.tmLanguage.json'), 'utf-8'); + } + + return results; + } + + private async findMatchingLanguages(language: string, rootPath: string) : Promise { + // Search through all package.json files in the directory and below, looking + // for the themeName in them. + const foundPackages = await new Promise((resolve, reject) => { + glob('**/package.json', { cwd: rootPath }, (err, matches) => { + if (err) { + reject(err); + } + resolve(matches); + }); + }); + if (foundPackages.length > 0) { + // For each one, open it up and look for the theme name. + for (const f of foundPackages) { + const fpath = path.join(rootPath, f); + const data = await this.findMatchingLanguageFromJson(fpath, language); + if (data) { + return data; + } + } + } + } + private async findMatchingTheme(themeName: string) : Promise { // Environment variable to mimic missing json problem if (process.env.VSC_PYTHON_MIMIC_REMOTE) { @@ -114,6 +175,28 @@ export class ThemeFinder { } } + private async findMatchingLanguageFromJson(packageJson: string, language: string) : Promise { + // Read the contents of the json file + const json = await fs.readJSON(packageJson, { encoding: 'utf-8'}); + + // Should have a name entry and a contributes entry + if (json.hasOwnProperty('name') && json.hasOwnProperty('contributes')) { + // See if contributes has a grammars + const contributes = json.contributes; + if (contributes.hasOwnProperty('grammars')) { + const grammars = contributes.grammars as any[]; + // Go through each theme, seeing if the label matches our theme name + for (const t of grammars) { + if (t.hasOwnProperty('language') && t.language === language) { + // Path is relative to the package.json file. + const rootFile = t.hasOwnProperty('path') ? path.join(path.dirname(packageJson), t.path.toString()) : ''; + return fs.readFile(rootFile, 'utf-8'); + } + } + } + } + } + private async findMatchingThemeFromJson(packageJson: string, themeName: string) : Promise { // Read the contents of the json file const json = await fs.readJSON(packageJson, { encoding: 'utf-8'}); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index d0b0c3e756e2..49fb7b70c521 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -253,6 +253,7 @@ export interface ICodeCssGenerator { export const IThemeFinder = Symbol('IThemeFinder'); export interface IThemeFinder { findThemeRootJson(themeName: string) : Promise; + findTmLanguage(language: string) : Promise; isThemeDark(themeName: string) : Promise; } diff --git a/src/client/datascience/webViewHost.ts b/src/client/datascience/webViewHost.ts index 651178e886fe..b103d40974a4 100644 --- a/src/client/datascience/webViewHost.ts +++ b/src/client/datascience/webViewHost.ts @@ -29,7 +29,7 @@ export class WebViewHost implements IDisposable { @unmanaged() private configService: IConfigurationService, @unmanaged() private provider: IWebPanelProvider, @unmanaged() private cssGenerator: ICodeCssGenerator, - @unmanaged() private themeFinder: IThemeFinder, + @unmanaged() protected themeFinder: IThemeFinder, @unmanaged() private workspaceService: IWorkspaceService, // tslint:disable-next-line:no-any @unmanaged() messageListenerCtor: (callback: (message: string, payload: any) => void, viewChanged: (panel: IWebPanel) => void, disposed: () => void) => IWebPanelMessageListener, diff --git a/src/datascience-ui/history-react/MainPanel.tsx b/src/datascience-ui/history-react/MainPanel.tsx index a040396dd747..048d0909d883 100644 --- a/src/datascience-ui/history-react/MainPanel.tsx +++ b/src/datascience-ui/history-react/MainPanel.tsx @@ -7,6 +7,7 @@ import { min } from 'lodash'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; +import { createDeferred, Deferred } from '../../client/common/utils/async'; import { CellMatcher } from '../../client/datascience/cellMatcher'; import { generateMarkdownFromCodeLines } from '../../client/datascience/common'; import { HistoryMessages, IHistoryMapping } from '../../client/datascience/history/historyTypes'; @@ -20,6 +21,7 @@ import { ContentPanel, IContentPanelProps } from './contentPanel'; import { HeaderPanel, IHeaderPanelProps } from './headerPanel'; import { InputHistory } from './inputHistory'; import { createCellVM, createEditableCellVM, extractInputText, generateTestState, IMainPanelState } from './mainPanelState'; +import { initializeTokenizer } from './tokenizer'; import { VariableExplorer } from './variableExplorer'; export interface IMainPanelProps { @@ -40,6 +42,8 @@ export class MainPanel extends React.Component private currentExecutionCount: number = 0; private postOffice: PostOffice = new PostOffice(); private completionProvider: CompletionProvider; + private onigasmPromise: Deferred | undefined; + private tmlangugePromise: Deferred | undefined; // tslint:disable-next-line:max-func-body-length constructor(props: IMainPanelProps, _state: IMainPanelState) { @@ -66,6 +70,9 @@ export class MainPanel extends React.Component // Setup the completion provider for monaco. We only need one this.completionProvider = new CompletionProvider(this.postOffice); + + // Setup the tokenizer for monaco + initializeTokenizer(this.loadOnigasm, this.loadTmlanguage, this.tokenizerLoaded).ignoreErrors(); } public componentWillMount() { @@ -100,12 +107,7 @@ export class MainPanel extends React.Component if (this.props.testMode) { this.renderCount = this.renderCount + 1; } - const baseTheme = this.computeBaseTheme(); - - const headerProps = this.getHeaderProps(baseTheme); - const contentProps = this.getContentProps(baseTheme); - return (
darkChanged={this.darkChanged} monacoThemeChanged={this.monacoThemeChanged} ref={this.styleInjectorRef} /> - - + {this.renderInnerContent(baseTheme)}
); } @@ -187,6 +188,14 @@ export class MainPanel extends React.Component this.getVariableValueResponse(payload); break; + case HistoryMessages.LoadOnigasmAssemblyResponse: + this.handleOnigasmResponse(payload); + break; + + case HistoryMessages.LoadTmLanguageResponse: + this.handleTmLanguageResponse(payload); + break; + default: break; } @@ -215,6 +224,25 @@ export class MainPanel extends React.Component // this.addCell(cell); // } + private renderInnerContent(baseTheme: string) { + // Skip if the tokenizer isn't finished yet. It needs + // to finish loading so our code editors work. + if (!this.state.tokenizerLoaded && !this.props.testMode) { + return null; + } + + // Otherwise render our cells and variable explorer. + const headerProps = this.getHeaderProps(baseTheme); + const contentProps = this.getContentProps(baseTheme); + + return ( +
+ + +
+ ); + } + // Called by the header control when size changes (such as expanding variables) private onHeaderHeightChange = (newHeight: number) => { this.setState({contentTop: newHeight}); @@ -842,4 +870,46 @@ export class MainPanel extends React.Component // Pass this onto the completion provider running in the extension this.sendMessage(HistoryMessages.EditCell, { changes }); } + + // tslint:disable-next-line: no-any + private tokenizerLoaded = (_e?: any) => { + this.setState({ tokenizerLoaded: true }); + } + + private loadOnigasm = () : Promise => { + if (!this.onigasmPromise) { + this.onigasmPromise = createDeferred(); + // Send our load onigasm request + this.sendMessage(HistoryMessages.LoadOnigasmAssemblyRequest); + } + return this.onigasmPromise.promise; + } + + private loadTmlanguage = () : Promise => { + if (!this.tmlangugePromise) { + this.tmlangugePromise = createDeferred(); + // Send our load onigasm request + this.sendMessage(HistoryMessages.LoadTmLanguageRequest); + } + return this.tmlangugePromise.promise; + } + + // tslint:disable-next-line: no-any + private handleOnigasmResponse(payload: any) { + if (payload && this.onigasmPromise) { + const typedArray = new Uint8Array(payload.data); + this.onigasmPromise.resolve(typedArray.buffer); + } else if (this.onigasmPromise) { + this.onigasmPromise.resolve(undefined); + } + } + + // tslint:disable-next-line: no-any + private handleTmLanguageResponse(payload: any) { + if (payload && this.tmlangugePromise) { + this.tmlangugePromise.resolve(payload.toString()); + } else if (this.tmlangugePromise) { + this.tmlangugePromise.resolve(undefined); + } + } } diff --git a/src/datascience-ui/history-react/code.css b/src/datascience-ui/history-react/code.css index 15aad5ab2b53..5aaae12184a0 100644 --- a/src/datascience-ui/history-react/code.css +++ b/src/datascience-ui/history-react/code.css @@ -51,3 +51,8 @@ width: 100%; visibility: none; } + +.code-area .mtk1 { + /* For some reason the monaco editor refuses to update this style no matter the theme. It's always black */ + color: var(--override-foreground, var(--vscode-editor-foreground)); +} diff --git a/src/datascience-ui/history-react/code.tsx b/src/datascience-ui/history-react/code.tsx index a2c5f2a8999f..e995ff2ba007 100644 --- a/src/datascience-ui/history-react/code.tsx +++ b/src/datascience-ui/history-react/code.tsx @@ -8,9 +8,6 @@ import MonacoEditor from 'react-monaco-editor'; import { InputHistory } from './inputHistory'; -// This next line is necessary to get webpack to load the python language settings. Otherwise -// it will fail to do so dynamically at run time. -import '../../../node_modules/monaco-editor/esm/vs/basic-languages/python/python.js'; import './code.css'; const LINE_HEIGHT = 18; @@ -149,6 +146,8 @@ export class Code extends React.Component { this.subscriptions.push(editor.onCompositionStart(this.compositionStart)); this.subscriptions.push(editor.onDidFocusEditorWidget(this.focusEditorWidget)); + this.subscriptions.push(editor.onKeyDown(this.onKeyDown)); + this.subscriptions.push(editor.onKeyUp(this.onKeyUp)); // Setup our context menu to show up outside. Autocomplete doesn't have this problem so it just works this.subscriptions.push(editor.onContextMenu((e) => { @@ -215,4 +214,25 @@ export class Code extends React.Component { } this.resizeTimer = window.setTimeout(this.updateEditorSize, 0); } + + private onKeyDown = (e: monacoEditor.IKeyboardEvent) => { + if (e.shiftKey && e.keyCode === monacoEditor.KeyCode.Enter && this.state.model && this.state.editor) { + // Shift enter was hit + e.stopPropagation(); + window.setTimeout(this.submitContent, 0); + } + } + + private onKeyUp = (e: monacoEditor.IKeyboardEvent) => { + if (e.shiftKey && e.keyCode === monacoEditor.KeyCode.Enter) { + // Shift enter was hit + e.stopPropagation(); + } + } + + private submitContent = () => { + if (this.state.model) { + this.props.onSubmit(this.state.model.getLinesContent().join('\n')); + } + } } diff --git a/src/datascience-ui/history-react/mainPanelState.ts b/src/datascience-ui/history-react/mainPanelState.ts index 3f81c7088807..4c160b632b77 100644 --- a/src/datascience-ui/history-react/mainPanelState.ts +++ b/src/datascience-ui/history-react/mainPanelState.ts @@ -27,6 +27,7 @@ export interface IMainPanelState { theme?: string; forceDark?: boolean; monacoTheme?: string; + tokenizerLoaded?: boolean; } // tslint:disable-next-line: no-multiline-string diff --git a/src/datascience-ui/history-react/tokenizer.ts b/src/datascience-ui/history-react/tokenizer.ts new file mode 100644 index 000000000000..267776243b61 --- /dev/null +++ b/src/datascience-ui/history-react/tokenizer.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { wireTmGrammars } from 'monaco-editor-textmate'; +import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; +import { Registry } from 'monaco-textmate'; +import { loadWASM } from 'onigasm'; +import { PYTHON_LANGUAGE } from '../../client/common/constants'; + +// tslint:disable: no-any +export async function initializeTokenizer( + getOnigasm: () => Promise, + getTmlanguageJSON: () => Promise, + loadingFinished: (e?: any) => void): Promise { + try { + // Tell monaco about our language + monacoEditor.languages.register({ + id: PYTHON_LANGUAGE, + extensions: ['.py'] + }); + + // Load the web assembly + const blob = await getOnigasm(); + await loadWASM(blob); + + // Setup our registry of different + const registry = new Registry({ + getGrammarDefinition: async (_scopeName) => { + return { + format: 'json', + content: await getTmlanguageJSON() + }; + } + }); + + // map of monaco "language id's" to TextMate scopeNames + const grammars = new Map(); + grammars.set('python', 'source.python'); + + // Wire everything together. + await wireTmGrammars(monacoEditor, registry, grammars); + + // Indicate to the callback that we're done. + loadingFinished(); + } catch (e) { + loadingFinished(e); + } +} diff --git a/webpack.datascience-ui.config.js b/webpack.datascience-ui.config.js index 6aa7d9548984..4d1d54e92895 100644 --- a/webpack.datascience-ui.config.js +++ b/webpack.datascience-ui.config.js @@ -35,7 +35,7 @@ module.exports = [ { from: './**/*theme*.json', to: '.' } ], { context: 'src' }), new MonacoWebpackPlugin({ - languages: ['python'] + languages: [] // force to empty so onigasm will be used }) ], resolve: { From c48c6fc24afb91656df50c83b1bd10a596ce65cc Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Tue, 30 Apr 2019 11:11:45 -0700 Subject: [PATCH 19/60] Move editor to standalone panel at the bottom --- src/client/datascience/codeCssGenerator.ts | 11 ++ .../history-react/MainPanel.tsx | 121 ++++++++++++------ src/datascience-ui/history-react/cell.css | 3 + src/datascience-ui/history-react/cell.tsx | 3 +- src/datascience-ui/history-react/code.tsx | 4 + .../history-react/contentPanel.css | 1 - .../history-react/contentPanel.tsx | 16 +-- .../history-react/headerPanel.css | 4 - .../history-react/mainPanel.css | 24 ++++ .../history-react/mainPanelState.ts | 5 +- src/datascience-ui/history-react/tokenizer.ts | 43 +++++++ 11 files changed, 177 insertions(+), 58 deletions(-) diff --git a/src/client/datascience/codeCssGenerator.ts b/src/client/datascience/codeCssGenerator.ts index e141640175e7..4c13499b6096 100644 --- a/src/client/datascience/codeCssGenerator.ts +++ b/src/client/datascience/codeCssGenerator.ts @@ -245,6 +245,17 @@ export class CodeCssGenerator implements ICodeCssGenerator { fontStyle: settings.fontStyle }); } + + // Same for string + // tslint:disable-next-line: possible-timing-attack + if (token === 'string') { + result.rules.push({ + token: 'punctuation.definition.string', + foreground: settings.foreground, + background: settings.background, + fontStyle: settings.fontStyle + }); + } } }); } diff --git a/src/datascience-ui/history-react/MainPanel.tsx b/src/datascience-ui/history-react/MainPanel.tsx index 048d0909d883..10554e242b12 100644 --- a/src/datascience-ui/history-react/MainPanel.tsx +++ b/src/datascience-ui/history-react/MainPanel.tsx @@ -8,10 +8,12 @@ import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; import { createDeferred, Deferred } from '../../client/common/utils/async'; +import { noop } from '../../client/common/utils/misc'; import { CellMatcher } from '../../client/datascience/cellMatcher'; import { generateMarkdownFromCodeLines } from '../../client/datascience/common'; import { HistoryMessages, IHistoryMapping } from '../../client/datascience/history/historyTypes'; import { CellState, ICell, IHistoryInfo, IJupyterVariable, IJupyterVariablesResponse } from '../../client/datascience/types'; +import { ErrorBoundary } from '../react-common/errorBoundary'; import { IMessageHandler, PostOffice } from '../react-common/postOffice'; import { getSettings, updateSettings } from '../react-common/settingsReactSide'; import { StyleInjector } from '../react-common/styleInjector'; @@ -50,18 +52,22 @@ export class MainPanel extends React.Component super(props); // Default state should show a busy message - this.state = { cellVMs: [], busy: true, undoStack: [], redoStack : [], submittedText: false, history: new InputHistory(), contentTop: 24 }; + this.state = { + cellVMs: [], + busy: true, + undoStack: [], + redoStack : [], + submittedText: false, + history: new InputHistory(), + contentTop: 24, + editCellVM: getSettings && getSettings().allowInput ? createEditableCellVM(1) : undefined + }; // Add test state if necessary if (!this.props.skipDefault) { this.state = generateTestState(this.inputBlockToggled); } - // Add a single empty cell if it's supported - if (getSettings && getSettings().allowInput) { - this.state.cellVMs.push(createEditableCellVM(1)); - } - // Create the ref to hold our variable explorer this.variableExplorerRef = React.createRef(); @@ -71,8 +77,10 @@ export class MainPanel extends React.Component // Setup the completion provider for monaco. We only need one this.completionProvider = new CompletionProvider(this.postOffice); - // Setup the tokenizer for monaco - initializeTokenizer(this.loadOnigasm, this.loadTmlanguage, this.tokenizerLoaded).ignoreErrors(); + // Setup the tokenizer for monaco if running inside of vscode + if (this.props.skipDefault) { + initializeTokenizer(this.loadOnigasm, this.loadTmlanguage, this.tokenizerLoaded).ignoreErrors(); + } } public componentWillMount() { @@ -107,7 +115,9 @@ export class MainPanel extends React.Component if (this.props.testMode) { this.renderCount = this.renderCount + 1; } + const baseTheme = this.computeBaseTheme(); + return (
darkChanged={this.darkChanged} monacoThemeChanged={this.monacoThemeChanged} ref={this.styleInjectorRef} /> - {this.renderInnerContent(baseTheme)} + {this.renderHeaderPanel(baseTheme)} + {this.renderContentPanel(baseTheme)} + {this.renderEditPanel(baseTheme)}
); } @@ -224,21 +236,70 @@ export class MainPanel extends React.Component // this.addCell(cell); // } - private renderInnerContent(baseTheme: string) { + private renderHeaderPanel(baseTheme: string) { + const headerProps = this.getHeaderProps(baseTheme); + return ( +
+
+ +
+
+ ); + } + + private renderContentPanel(baseTheme: string) { // Skip if the tokenizer isn't finished yet. It needs // to finish loading so our code editors work. if (!this.state.tokenizerLoaded && !this.props.testMode) { return null; } - // Otherwise render our cells and variable explorer. - const headerProps = this.getHeaderProps(baseTheme); + // Otherwise render our cells. const contentProps = this.getContentProps(baseTheme); + return ( +
+
+ +
+
+ ); + } + + private renderEditPanel(baseTheme: string) { + // Skip if the tokenizer isn't finished yet. It needs + // to finish loading so our code editors work. + if (!this.state.tokenizerLoaded || !this.state.editCellVM) { + return null; + } + + const maxOutputSize = getSettings().maxOutputSize; + const errorBackgroundColor = getSettings().errorBackgroundColor; + const actualErrorBackgroundColor = errorBackgroundColor ? errorBackgroundColor : '#FFFFFF'; + const maxTextSize = maxOutputSize && maxOutputSize < 10000 && maxOutputSize > 0 ? maxOutputSize : undefined; return ( -
- - +
+
+ + + +
); } @@ -296,12 +357,9 @@ export class MainPanel extends React.Component testMode: this.props.testMode, codeTheme: this.props.codeTheme, submittedText: this.state.submittedText, - saveEditCellRef: this.saveEditCellRef, gotoCellCode: this.gotoCellCode, deleteCell: this.deleteCell, - submitInput: this.submitInput, skipNextScroll: this.state.skipNextScroll ? true : false, - onCodeChange: this.codeChange, monacoTheme: this.state.monacoTheme }; } @@ -557,19 +615,7 @@ export class MainPanel extends React.Component cellVM = this.alterCellVM(cellVM, showInputs, !collapseInputs); if (cellVM) { - let newList : ICellViewModel[] = []; - - // Insert before the edit cell if we have one - const editCell = this.getEditCell(); - if (editCell) { - newList = [...this.state.cellVMs.filter(c => !c.editable), cellVM, editCell]; - - // Update execution count on the last cell - editCell.cell.data.execution_count = this.getInputExecutionCount(newList); - } else { - newList = [...this.state.cellVMs, cellVM]; - } - + const newList = [...this.state.cellVMs, cellVM]; this.setState({ cellVMs: newList, undoStack: this.pushStack(this.state.undoStack, this.state.cellVMs), @@ -584,12 +630,7 @@ export class MainPanel extends React.Component } private getEditCell() : ICellViewModel | undefined { - const editCells = this.state.cellVMs.filter(c => c.editable); - if (editCells && editCells.length === 1) { - return editCells[0]; - } - - return undefined; + return this.state.editCellVM; } private inputBlockToggled = (id: string) => { @@ -776,9 +817,6 @@ export class MainPanel extends React.Component // This should be from our last entry. Switch this entry to read only, and add a new item to our list let editCell = this.getEditCell(); if (editCell) { - // Save a copy of the ones without edits. - const withoutEdits = this.state.cellVMs.filter(c => !c.editable); - // Change this editable cell to not editable. editCell.cell.state = CellState.executing; editCell.cell.data.source = code; @@ -805,7 +843,8 @@ export class MainPanel extends React.Component // Stick in a new cell at the bottom that's editable and update our state // so that the last cell becomes busy this.setState({ - cellVMs: [...withoutEdits, editCell, createEditableCellVM(this.getInputExecutionCount(withoutEdits))], + cellVMs: [...this.state.cellVMs, editCell], + editCellVM: createEditableCellVM(this.getInputExecutionCount(this.state.cellVMs)), undoStack : this.pushStack(this.state.undoStack, this.state.cellVMs), redoStack: this.state.redoStack, skipNextScroll: false, diff --git a/src/datascience-ui/history-react/cell.css b/src/datascience-ui/history-react/cell.css index 0b5f5fc924f9..c412e05800d7 100644 --- a/src/datascience-ui/history-react/cell.css +++ b/src/datascience-ui/history-react/cell.css @@ -2,6 +2,9 @@ margin: 0px; padding: 2px; display: block; +} + +.cell-wrapper-noneditable { border-bottom-color: var(--override-widget-background, var(--vscode-editorGroupHeader-tabsBackground)); border-bottom-style: solid; border-bottom-width: 1px; diff --git a/src/datascience-ui/history-react/cell.tsx b/src/datascience-ui/history-react/cell.tsx index 9f2d1b5a3121..38293e3456b3 100644 --- a/src/datascience-ui/history-react/cell.tsx +++ b/src/datascience-ui/history-react/cell.tsx @@ -123,11 +123,12 @@ export class Cell extends React.Component { const allowsPlainInput = getSettings().showCellInputCode || this.props.cellVM.directInput || this.props.cellVM.editable; const shouldRender = allowsPlainInput || (results && results.length > 0); const cellOuterClass = this.props.cellVM.editable ? 'cell-outer-editable' : 'cell-outer'; + const cellWrapperClass = this.props.cellVM.editable ? 'cell-wrapper' : 'cell-wrapper cell-wrapper-noneditable'; // Only render if we are allowed to. if (shouldRender) { return ( -
+