diff --git a/src/client/pythonEnvironments/base/info/index.ts b/src/client/pythonEnvironments/base/info/index.ts index 11398217a512..c24cf8dc2aa3 100644 --- a/src/client/pythonEnvironments/base/info/index.ts +++ b/src/client/pythonEnvironments/base/info/index.ts @@ -4,6 +4,7 @@ import { Uri } from 'vscode'; import { Architecture } from '../../../common/utils/platform'; import { BasicVersionInfo, VersionInfo } from '../../../common/utils/version'; +import { arePathsSame } from '../../common/externalDependencies'; /** * IDs for the various supported Python environments. @@ -143,3 +144,31 @@ export type PythonEnvInfo = _PythonEnvInfo & { defaultDisplayName?: string; searchLocation?: Uri; }; + +/** + * Determine if the given infos correspond to the same env. + * + * @param environment1 - one of the two envs to compare + * @param environment2 - one of the two envs to compare + */ +export function areSameEnvironment( + environment1: PythonEnvInfo | string, + environment2: PythonEnvInfo | string, +): boolean { + let path1: string; + let path2: string; + if (typeof environment1 === 'string') { + path1 = environment1; + } else { + path1 = environment1.executable.filename; + } + if (typeof environment2 === 'string') { + path2 = environment2; + } else { + path2 = environment2.executable.filename; + } + if (arePathsSame(path1, path2)) { + return true; + } + return false; +} diff --git a/src/client/pythonEnvironments/base/locator.ts b/src/client/pythonEnvironments/base/locator.ts index 03eb206445bd..c6a77c473d05 100644 --- a/src/client/pythonEnvironments/base/locator.ts +++ b/src/client/pythonEnvironments/base/locator.ts @@ -92,7 +92,7 @@ export type PythonLocatorQuery = BasicPythonLocatorQuery & { searchLocations?: Uri[]; }; -type QueryForEvent = E extends PythonEnvsChangedEvent ? PythonLocatorQuery : BasicPythonLocatorQuery; +export type QueryForEvent = E extends PythonEnvsChangedEvent ? PythonLocatorQuery : BasicPythonLocatorQuery; /** * A single Python environment locator. diff --git a/src/client/pythonEnvironments/collection/environmentsReducer.ts b/src/client/pythonEnvironments/collection/environmentsReducer.ts new file mode 100644 index 000000000000..b4c80254a3f9 --- /dev/null +++ b/src/client/pythonEnvironments/collection/environmentsReducer.ts @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { cloneDeep, isEqual } from 'lodash'; +import { Event, EventEmitter } from 'vscode'; +import { traceVerbose } from '../../common/logger'; +import { createDeferred } from '../../common/utils/async'; +import { areSameEnvironment, PythonEnvInfo, PythonEnvKind } from '../base/info'; +import { + ILocator, IPythonEnvsIterator, PythonEnvUpdatedEvent, QueryForEvent, +} from '../base/locator'; +import { PythonEnvsChangedEvent } from '../base/watcher'; + +/** + * Combines duplicate environments received from the incoming locator into one and passes on unique environments + */ +export class PythonEnvsReducer implements ILocator { + public get onChanged(): Event { + return this.parentLocator.onChanged; + } + + constructor(private readonly parentLocator: ILocator) {} + + public async resolveEnv(env: string | PythonEnvInfo): Promise { + let environment: PythonEnvInfo | undefined; + const waitForUpdatesDeferred = createDeferred(); + const iterator = this.iterEnvs(); + iterator.onUpdated!((event) => { + if (event === null) { + waitForUpdatesDeferred.resolve(); + } else if (environment && areSameEnvironment(environment, event.new)) { + environment = event.new; + } + }); + let result = await iterator.next(); + while (!result.done) { + if (areSameEnvironment(result.value, env)) { + environment = result.value; + } + // eslint-disable-next-line no-await-in-loop + result = await iterator.next(); + } + if (!environment) { + return undefined; + } + await waitForUpdatesDeferred.promise; + return this.parentLocator.resolveEnv(environment); + } + + public iterEnvs(query?: QueryForEvent): IPythonEnvsIterator { + const didUpdate = new EventEmitter(); + const incomingIterator = this.parentLocator.iterEnvs(query); + const iterator: IPythonEnvsIterator = iterEnvsIterator(incomingIterator, didUpdate); + iterator.onUpdated = didUpdate.event; + return iterator; + } +} + +async function* iterEnvsIterator( + iterator: IPythonEnvsIterator, + didUpdate: EventEmitter, +): AsyncIterator { + const state = { + done: false, + pending: 0, + }; + const seen: PythonEnvInfo[] = []; + + if (iterator.onUpdated !== undefined) { + iterator.onUpdated((event) => { + if (event === null) { + state.done = true; + checkIfFinishedAndNotify(state, didUpdate); + } else { + const oldIndex = seen.findIndex((s) => areSameEnvironment(s, event.old)); + if (oldIndex !== -1) { + state.pending += 1; + resolveDifferencesInBackground(oldIndex, event.new, state, didUpdate, seen).ignoreErrors(); + } else { + // This implies a problem in a downstream locator + traceVerbose(`Expected already iterated env, got ${event.old}`); + } + } + }); + } + + let result = await iterator.next(); + while (!result.done) { + const currEnv = result.value; + const oldIndex = seen.findIndex((s) => areSameEnvironment(s, currEnv)); + if (oldIndex !== -1) { + state.pending += 1; + resolveDifferencesInBackground(oldIndex, currEnv, state, didUpdate, seen).ignoreErrors(); + } else { + // We haven't yielded a matching env so yield this one as-is. + yield currEnv; + seen.push(currEnv); + } + // eslint-disable-next-line no-await-in-loop + result = await iterator.next(); + } + if (iterator.onUpdated === undefined) { + state.done = true; + checkIfFinishedAndNotify(state, didUpdate); + } +} + +async function resolveDifferencesInBackground( + oldIndex: number, + newEnv: PythonEnvInfo, + state: { done: boolean; pending: number }, + didUpdate: EventEmitter, + seen: PythonEnvInfo[], +) { + const oldEnv = seen[oldIndex]; + const merged = mergeEnvironments(oldEnv, newEnv); + if (!isEqual(oldEnv, merged)) { + didUpdate.fire({ old: oldEnv, new: merged }); + seen[oldIndex] = merged; + } + state.pending -= 1; + checkIfFinishedAndNotify(state, didUpdate); +} + +/** + * When all info from incoming iterator has been received and all background calls finishes, notify that we're done + * @param state Carries the current state of progress + * @param didUpdate Used to notify when finished + */ +function checkIfFinishedAndNotify( + state: { done: boolean; pending: number }, + didUpdate: EventEmitter, +) { + if (state.done && state.pending === 0) { + didUpdate.fire(null); + didUpdate.dispose(); + } +} + +export function mergeEnvironments(environment: PythonEnvInfo, other: PythonEnvInfo): PythonEnvInfo { + const result = cloneDeep(environment); + // Preserve type information. + // Possible we identified environment as unknown, but a later provider has identified env type. + if (environment.kind === PythonEnvKind.Unknown && other.kind && other.kind !== PythonEnvKind.Unknown) { + result.kind = other.kind; + } + const props: (keyof PythonEnvInfo)[] = [ + 'version', + 'kind', + 'executable', + 'name', + 'arch', + 'distro', + 'defaultDisplayName', + 'searchLocation', + ]; + props.forEach((prop) => { + if (!result[prop] && other[prop]) { + // tslint:disable: no-any + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (result as any)[prop] = other[prop]; + } + }); + return result; +} diff --git a/src/test/pythonEnvironments/base/common.ts b/src/test/pythonEnvironments/base/common.ts index 6f56011f1f03..d1d3b21cc7cc 100644 --- a/src/test/pythonEnvironments/base/common.ts +++ b/src/test/pythonEnvironments/base/common.ts @@ -1,14 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { createDeferred, flattenIterator, iterable, mapToIterator } from '../../../client/common/utils/async'; +import { Event } from 'vscode'; +import { + createDeferred, flattenIterator, iterable, mapToIterator, +} from '../../../client/common/utils/async'; import { Architecture } from '../../../client/common/utils/platform'; import { PythonEnvInfo, PythonEnvKind, } from '../../../client/pythonEnvironments/base/info'; import { parseVersion } from '../../../client/pythonEnvironments/base/info/pythonVersion'; -import { IPythonEnvsIterator, Locator, PythonLocatorQuery } from '../../../client/pythonEnvironments/base/locator'; +import { + IPythonEnvsIterator, Locator, PythonEnvUpdatedEvent, PythonLocatorQuery, +} from '../../../client/pythonEnvironments/base/locator'; import { PythonEnvsChangedEvent } from '../../../client/pythonEnvironments/base/watcher'; export function createEnv( @@ -66,6 +71,7 @@ export class SimpleLocator extends Locator { resolve?: null | ((env: PythonEnvInfo) => Promise); before?: Promise; after?: Promise; + onUpdated?: Event; beforeEach?(e: PythonEnvInfo): Promise; afterEach?(e: PythonEnvInfo): Promise; onQuery?(query: PythonLocatorQuery | undefined, envs: PythonEnvInfo[]): Promise; @@ -83,7 +89,7 @@ export class SimpleLocator extends Locator { const deferred = this.deferred; const callbacks = this.callbacks; let envs = this.envs; - async function* iterator() { + const iterator: IPythonEnvsIterator = async function*() { if (callbacks?.onQuery !== undefined) { envs = await callbacks.onQuery(query, envs); } @@ -114,8 +120,9 @@ export class SimpleLocator extends Locator { await callbacks.after; } deferred.resolve(); - } - return iterator(); + }(); + iterator.onUpdated = this.callbacks?.onUpdated; + return iterator; } public async resolveEnv(env: string | PythonEnvInfo): Promise { const envInfo: PythonEnvInfo = typeof env === 'string' ? createEnv('', '', undefined, env) : env; diff --git a/src/test/pythonEnvironments/collection/environmentsReducer.unit.test.ts b/src/test/pythonEnvironments/collection/environmentsReducer.unit.test.ts new file mode 100644 index 000000000000..a05ebc772063 --- /dev/null +++ b/src/test/pythonEnvironments/collection/environmentsReducer.unit.test.ts @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { assert, expect } from 'chai'; +import { isEqual } from 'lodash'; +import * as path from 'path'; +import { EventEmitter } from 'vscode'; +import { PythonEnvInfo, PythonEnvKind } from '../../../client/pythonEnvironments/base/info'; +import { PythonEnvUpdatedEvent } from '../../../client/pythonEnvironments/base/locator'; +import { PythonEnvsChangedEvent } from '../../../client/pythonEnvironments/base/watcher'; +import { + mergeEnvironments, + PythonEnvsReducer, +} from '../../../client/pythonEnvironments/collection/environmentsReducer'; +import { sleep } from '../../core'; +import { createEnv, getEnvs, SimpleLocator } from '../base/common'; + +suite('Environments Reducer', () => { + suite('iterEnvs()', () => { + test('Iterator only yields unique environments', async () => { + const env1 = createEnv('env1', '3.5', PythonEnvKind.Venv, path.join('path', 'to', 'exec1')); + const env2 = createEnv('env2', '3.8', PythonEnvKind.Conda, path.join('path', 'to', 'exec2')); + const env3 = createEnv('env3', '2.7', PythonEnvKind.System, path.join('path', 'to', 'exec3')); + const env4 = createEnv('env4', '3.8.1', PythonEnvKind.Unknown, path.join('path', 'to', 'exec2')); // Same as env2 + const env5 = createEnv('env5', '3.5.12b1', PythonEnvKind.Venv, path.join('path', 'to', 'exec1')); // Same as env1 + const environmentsToBeIterated = [env1, env2, env3, env4, env5]; // Contains 3 unique environments + const parentLocator = new SimpleLocator(environmentsToBeIterated); + const reducer = new PythonEnvsReducer(parentLocator); + + const iterator = reducer.iterEnvs(); + const envs = await getEnvs(iterator); + + const expected = [env1, env2, env3]; + assert.deepEqual(envs, expected); + }); + + test('Single updates for multiple environments are sent correctly followed by the null event', async () => { + // Arrange + const env1 = createEnv('env1', '3.5', PythonEnvKind.Unknown, path.join('path', 'to', 'exec1')); + const env2 = createEnv('env2', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec2')); + const env3 = createEnv('env3', '2.7', PythonEnvKind.System, path.join('path', 'to', 'exec3')); + const env4 = createEnv('env4', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec2')); // Same as env2; + const env5 = createEnv('env5', '3.5.12b1', PythonEnvKind.Venv, path.join('path', 'to', 'exec1')); // Same as env1; + const environmentsToBeIterated = [env1, env2, env3, env4, env5]; // Contains 3 unique environments + const parentLocator = new SimpleLocator(environmentsToBeIterated); + const onUpdatedEvents: (PythonEnvUpdatedEvent | null)[] = []; + const reducer = new PythonEnvsReducer(parentLocator); + + const iterator = reducer.iterEnvs(); // Act + + // Assert + let { onUpdated } = iterator; + expect(onUpdated).to.not.equal(undefined, ''); + + // Arrange + onUpdated = onUpdated!; + onUpdated((e) => { + onUpdatedEvents.push(e); + }); + + // Act + await getEnvs(iterator); + await sleep(1); // Resolve pending calls in the background + + // Assert + const expectedUpdates = [ + { old: env2, new: mergeEnvironments(env2, env4) }, + { old: env1, new: mergeEnvironments(env1, env5) }, + null, + ]; + assert.deepEqual(expectedUpdates, onUpdatedEvents); + }); + + test('Multiple updates for the same environment are sent correctly followed by the null event', async () => { + // Arrange + const env1 = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const env2 = createEnv('env2', '3.8.1', PythonEnvKind.System, path.join('path', 'to', 'exec')); + const env3 = createEnv('env3', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec')); + const environmentsToBeIterated = [env1, env2, env3]; // All refer to the same environment + const parentLocator = new SimpleLocator(environmentsToBeIterated); + const onUpdatedEvents: (PythonEnvUpdatedEvent | null)[] = []; + const reducer = new PythonEnvsReducer(parentLocator); + + const iterator = reducer.iterEnvs(); // Act + + // Assert + let { onUpdated } = iterator; + expect(onUpdated).to.not.equal(undefined, ''); + + // Arrange + onUpdated = onUpdated!; + onUpdated((e) => { + onUpdatedEvents.push(e); + }); + + // Act + await getEnvs(iterator); + await sleep(1); // Resolve pending calls in the background + + // Assert + const env12 = mergeEnvironments(env1, env2); + const env123 = mergeEnvironments(env12, env3); + const expectedUpdates: (PythonEnvUpdatedEvent | null)[] = []; + if (isEqual(env12, env123)) { + expectedUpdates.push({ old: env1, new: env12 }, null); + } else { + expectedUpdates.push({ old: env1, new: env12 }, { old: env12, new: env123 }, null); + } + assert.deepEqual(onUpdatedEvents, expectedUpdates); + }); + + test('Updates to environments from the incoming iterator are passed on correctly followed by the null event', async () => { + // Arrange + const env1 = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const env2 = createEnv('env2', '3.8.1', PythonEnvKind.System, path.join('path', 'to', 'exec')); + const environmentsToBeIterated = [env1]; + const didUpdate = new EventEmitter(); + const parentLocator = new SimpleLocator(environmentsToBeIterated, { onUpdated: didUpdate.event }); + const onUpdatedEvents: (PythonEnvUpdatedEvent | null)[] = []; + const reducer = new PythonEnvsReducer(parentLocator); + + const iterator = reducer.iterEnvs(); // Act + + // Assert + let { onUpdated } = iterator; + expect(onUpdated).to.not.equal(undefined, ''); + + // Arrange + onUpdated = onUpdated!; + onUpdated((e) => { + onUpdatedEvents.push(e); + }); + + // Act + await getEnvs(iterator); + didUpdate.fire({ old: env1, new: env2 }); + didUpdate.fire(null); // It is essential for the incoming iterator to fire "null" event signifying it's done + await sleep(1); + + // Assert + const expectedUpdates = [{ old: env1, new: mergeEnvironments(env1, env2) }, null]; + assert.deepEqual(expectedUpdates, onUpdatedEvents); + didUpdate.dispose(); + }); + }); + + test('onChanged fires iff onChanged from locator manager fires', () => { + const parentLocator = new SimpleLocator([]); + const event1: PythonEnvsChangedEvent = {}; + const event2: PythonEnvsChangedEvent = { kind: PythonEnvKind.Unknown }; + const expected = [event1, event2]; + const reducer = new PythonEnvsReducer(parentLocator); + + const events: PythonEnvsChangedEvent[] = []; + reducer.onChanged((e) => events.push(e)); + + parentLocator.fire(event1); + parentLocator.fire(event2); + + assert.deepEqual(events, expected); + }); + + suite('resolveEnv()', () => { + test('Iterates environments from the reducer to get resolved environment, then calls into locator manager to resolve environment further and return it', async () => { + const env1 = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const env2 = createEnv('env2', '2.7', PythonEnvKind.System, path.join('path', 'to', 'exec3')); + const env3 = createEnv('env3', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec')); + const env4 = createEnv('env4', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec2')); + const env5 = createEnv('env5', '3.5.12b1', PythonEnvKind.Venv, path.join('path', 'to', 'exec1')); + const env6 = createEnv('env6', '3.8.1', PythonEnvKind.System, path.join('path', 'to', 'exec')); + const environmentsToBeIterated = [env1, env2, env3, env4, env5, env6]; // env1 env3 env6 are same + + const env13 = mergeEnvironments(env1, env3); + const env136 = mergeEnvironments(env13, env6); + const expectedResolvedEnv = createEnv('resolvedEnv', '3.8.1', PythonEnvKind.Conda, 'resolved/path/to/exec'); + const parentLocator = new SimpleLocator(environmentsToBeIterated, { + resolve: async (e: PythonEnvInfo) => { + if (isEqual(e, env136)) { + return expectedResolvedEnv; + } + return undefined; + }, + }); + const reducer = new PythonEnvsReducer(parentLocator); + + // Trying to resolve the environment corresponding to env1 env3 env6 + const expected = await reducer.resolveEnv(path.join('path', 'to', 'exec')); + + assert.deepEqual(expected, expectedResolvedEnv); + }); + + test("If the reducer isn't able to resolve environment, return undefined", async () => { + const env1 = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const env2 = createEnv('env2', '2.7', PythonEnvKind.System, path.join('path', 'to', 'exec3')); + const env3 = createEnv('env3', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec')); + const env4 = createEnv('env4', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec2')); + const env5 = createEnv('env5', '3.5.12b1', PythonEnvKind.Venv, path.join('path', 'to', 'exec1')); + const env6 = createEnv('env6', '3.8.1', PythonEnvKind.System, path.join('path', 'to', 'exec')); + const environmentsToBeIterated = [env1, env2, env3, env4, env5, env6]; // env1 env3 env6 are same + + const env13 = mergeEnvironments(env1, env3); + const env136 = mergeEnvironments(env13, env6); + const parentLocator = new SimpleLocator(environmentsToBeIterated, { + resolve: async (e: PythonEnvInfo) => { + if (isEqual(e, env136)) { + return createEnv('resolvedEnv', '3.8.1', PythonEnvKind.Conda, 'resolved/path/to/exec'); + } + return undefined; + }, + }); + const reducer = new PythonEnvsReducer(parentLocator); + + const expected = await reducer.resolveEnv(path.join('path', 'to', 'execNeverSeenBefore')); + + assert.deepEqual(expected, undefined); + }); + }); +});