Skip to content

Commit 0f23238

Browse files
eleanorjboydeleanorjboyd
authored
Result resolver feature branch (#21457)
fixes #21394
1 parent 1323a6a commit 0f23238

21 files changed

+2091
-984
lines changed

‎pythonFiles/unittestadapter/execution.py

-2
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,6 @@ def run_tests(
242242
try:
243243
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
244244
client_socket.connect(("localhost", run_test_ids_port_int))
245-
print(f"CLIENT: Server listening on port {run_test_ids_port_int}...")
246245
buffer = b""
247246

248247
while True:
@@ -263,7 +262,6 @@ def run_tests(
263262
buffer = b""
264263

265264
# Process the JSON data
266-
print(f"Received JSON data: {test_ids_from_buffer}")
267265
break
268266
except json.JSONDecodeError:
269267
# JSON decoding error, the complete JSON object is not yet received

‎src/client/testing/common/socketServer.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ export class UnitTestSocketServer extends EventEmitter implements IUnitTestSocke
123123
if ((socket as any).id) {
124124
destroyedSocketId = (socket as any).id;
125125
}
126-
this.log('socket disconnected', destroyedSocketId.toString());
126+
this.log('socket disconnected', destroyedSocketId?.toString());
127127
if (socket && socket.destroy) {
128128
socket.destroy();
129129
}

‎src/client/testing/testController/common/resultResolver.ts

+233
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode';
5+
import * as util from 'util';
6+
import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types';
7+
import { TestProvider } from '../../types';
8+
import { traceError, traceLog } from '../../../logging';
9+
import { Testing } from '../../../common/utils/localize';
10+
import { clearAllChildren, createErrorTestItem, getTestCaseNodes } from './testItemUtilities';
11+
import { sendTelemetryEvent } from '../../../telemetry';
12+
import { EventName } from '../../../telemetry/constants';
13+
import { splitLines } from '../../../common/stringUtils';
14+
import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils';
15+
16+
export class PythonResultResolver implements ITestResultResolver {
17+
testController: TestController;
18+
19+
testProvider: TestProvider;
20+
21+
public runIdToTestItem: Map<string, TestItem>;
22+
23+
public runIdToVSid: Map<string, string>;
24+
25+
public vsIdToRunId: Map<string, string>;
26+
27+
constructor(testController: TestController, testProvider: TestProvider, private workspaceUri: Uri) {
28+
this.testController = testController;
29+
this.testProvider = testProvider;
30+
31+
this.runIdToTestItem = new Map<string, TestItem>();
32+
this.runIdToVSid = new Map<string, string>();
33+
this.vsIdToRunId = new Map<string, string>();
34+
}
35+
36+
public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise<void> {
37+
const workspacePath = this.workspaceUri.fsPath;
38+
traceLog('Using result resolver for discovery');
39+
40+
const rawTestData = payload;
41+
if (!rawTestData) {
42+
// No test data is available
43+
return Promise.resolve();
44+
}
45+
46+
// Check if there were any errors in the discovery process.
47+
if (rawTestData.status === 'error') {
48+
const testingErrorConst =
49+
this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery;
50+
const { errors } = rawTestData;
51+
traceError(testingErrorConst, '\r\n', errors!.join('\r\n\r\n'));
52+
53+
let errorNode = this.testController.items.get(`DiscoveryError:${workspacePath}`);
54+
const message = util.format(
55+
`${testingErrorConst} ${Testing.seePythonOutput}\r\n`,
56+
errors!.join('\r\n\r\n'),
57+
);
58+
59+
if (errorNode === undefined) {
60+
const options = buildErrorNodeOptions(this.workspaceUri, message, this.testProvider);
61+
errorNode = createErrorTestItem(this.testController, options);
62+
this.testController.items.add(errorNode);
63+
}
64+
errorNode.error = message;
65+
} else {
66+
// Remove the error node if necessary,
67+
// then parse and insert test data.
68+
this.testController.items.delete(`DiscoveryError:${workspacePath}`);
69+
70+
if (rawTestData.tests) {
71+
// If the test root for this folder exists: Workspace refresh, update its children.
72+
// Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree.
73+
populateTestTree(this.testController, rawTestData.tests, undefined, this, token);
74+
} else {
75+
// Delete everything from the test controller.
76+
this.testController.items.replace([]);
77+
}
78+
}
79+
80+
sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, {
81+
tool: this.testProvider,
82+
failed: false,
83+
});
84+
return Promise.resolve();
85+
}
86+
87+
public resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise<void> {
88+
const rawTestExecData = payload;
89+
if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) {
90+
// Map which holds the subtest information for each test item.
91+
const subTestStats: Map<string, { passed: number; failed: number }> = new Map();
92+
93+
// iterate through payload and update the UI accordingly.
94+
for (const keyTemp of Object.keys(rawTestExecData.result)) {
95+
const testCases: TestItem[] = [];
96+
97+
// grab leaf level test items
98+
this.testController.items.forEach((i) => {
99+
const tempArr: TestItem[] = getTestCaseNodes(i);
100+
testCases.push(...tempArr);
101+
});
102+
103+
if (
104+
rawTestExecData.result[keyTemp].outcome === 'failure' ||
105+
rawTestExecData.result[keyTemp].outcome === 'passed-unexpected'
106+
) {
107+
const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? '';
108+
const traceback = splitLines(rawTraceback, {
109+
trim: false,
110+
removeEmptyEntries: true,
111+
}).join('\r\n');
112+
113+
const text = `${rawTestExecData.result[keyTemp].test} failed: ${
114+
rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome
115+
}\r\n${traceback}\r\n`;
116+
const message = new TestMessage(text);
117+
118+
// note that keyTemp is a runId for unittest library...
119+
const grabVSid = this.runIdToVSid.get(keyTemp);
120+
// search through freshly built array of testItem to find the failed test and update UI.
121+
testCases.forEach((indiItem) => {
122+
if (indiItem.id === grabVSid) {
123+
if (indiItem.uri && indiItem.range) {
124+
message.location = new Location(indiItem.uri, indiItem.range);
125+
runInstance.failed(indiItem, message);
126+
runInstance.appendOutput(fixLogLines(text));
127+
}
128+
}
129+
});
130+
} else if (
131+
rawTestExecData.result[keyTemp].outcome === 'success' ||
132+
rawTestExecData.result[keyTemp].outcome === 'expected-failure'
133+
) {
134+
const grabTestItem = this.runIdToTestItem.get(keyTemp);
135+
const grabVSid = this.runIdToVSid.get(keyTemp);
136+
if (grabTestItem !== undefined) {
137+
testCases.forEach((indiItem) => {
138+
if (indiItem.id === grabVSid) {
139+
if (indiItem.uri && indiItem.range) {
140+
runInstance.passed(grabTestItem);
141+
runInstance.appendOutput('Passed here');
142+
}
143+
}
144+
});
145+
}
146+
} else if (rawTestExecData.result[keyTemp].outcome === 'skipped') {
147+
const grabTestItem = this.runIdToTestItem.get(keyTemp);
148+
const grabVSid = this.runIdToVSid.get(keyTemp);
149+
if (grabTestItem !== undefined) {
150+
testCases.forEach((indiItem) => {
151+
if (indiItem.id === grabVSid) {
152+
if (indiItem.uri && indiItem.range) {
153+
runInstance.skipped(grabTestItem);
154+
runInstance.appendOutput('Skipped here');
155+
}
156+
}
157+
});
158+
}
159+
} else if (rawTestExecData.result[keyTemp].outcome === 'subtest-failure') {
160+
// split on " " since the subtest ID has the parent test ID in the first part of the ID.
161+
const parentTestCaseId = keyTemp.split(' ')[0];
162+
const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
163+
const data = rawTestExecData.result[keyTemp];
164+
// find the subtest's parent test item
165+
if (parentTestItem) {
166+
const subtestStats = subTestStats.get(parentTestCaseId);
167+
if (subtestStats) {
168+
subtestStats.failed += 1;
169+
} else {
170+
subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 });
171+
runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`));
172+
// clear since subtest items don't persist between runs
173+
clearAllChildren(parentTestItem);
174+
}
175+
const subtestId = keyTemp;
176+
const subTestItem = this.testController?.createTestItem(subtestId, subtestId);
177+
runInstance.appendOutput(fixLogLines(`${subtestId} Failed\r\n`));
178+
// create a new test item for the subtest
179+
if (subTestItem) {
180+
const traceback = data.traceback ?? '';
181+
const text = `${data.subtest} Failed: ${data.message ?? data.outcome}\r\n${traceback}\r\n`;
182+
runInstance.appendOutput(fixLogLines(text));
183+
parentTestItem.children.add(subTestItem);
184+
runInstance.started(subTestItem);
185+
const message = new TestMessage(rawTestExecData?.result[keyTemp].message ?? '');
186+
if (parentTestItem.uri && parentTestItem.range) {
187+
message.location = new Location(parentTestItem.uri, parentTestItem.range);
188+
}
189+
runInstance.failed(subTestItem, message);
190+
} else {
191+
throw new Error('Unable to create new child node for subtest');
192+
}
193+
} else {
194+
throw new Error('Parent test item not found');
195+
}
196+
} else if (rawTestExecData.result[keyTemp].outcome === 'subtest-success') {
197+
// split on " " since the subtest ID has the parent test ID in the first part of the ID.
198+
const parentTestCaseId = keyTemp.split(' ')[0];
199+
const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
200+
201+
// find the subtest's parent test item
202+
if (parentTestItem) {
203+
const subtestStats = subTestStats.get(parentTestCaseId);
204+
if (subtestStats) {
205+
subtestStats.passed += 1;
206+
} else {
207+
subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 });
208+
runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`));
209+
// clear since subtest items don't persist between runs
210+
clearAllChildren(parentTestItem);
211+
}
212+
const subtestId = keyTemp;
213+
const subTestItem = this.testController?.createTestItem(subtestId, subtestId);
214+
// create a new test item for the subtest
215+
if (subTestItem) {
216+
parentTestItem.children.add(subTestItem);
217+
runInstance.started(subTestItem);
218+
runInstance.passed(subTestItem);
219+
runInstance.appendOutput(fixLogLines(`${subtestId} Passed\r\n`));
220+
} else {
221+
throw new Error('Unable to create new child node for subtest');
222+
}
223+
} else {
224+
throw new Error('Parent test item not found');
225+
}
226+
}
227+
}
228+
}
229+
return Promise.resolve();
230+
}
231+
}
232+
233+
// had to switch the order of the original parameter since required param cannot follow optional.

‎src/client/testing/testController/common/server.ts

+38-5
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import {
99
IPythonExecutionFactory,
1010
SpawnOptions,
1111
} from '../../../common/process/types';
12-
import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging';
12+
import { traceError, traceInfo, traceLog } from '../../../logging';
1313
import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types';
1414
import { ITestDebugLauncher, LaunchOptions } from '../../common/types';
1515
import { UNITTEST_PROVIDER } from '../../common/constants';
@@ -24,6 +24,10 @@ export class PythonTestServer implements ITestServer, Disposable {
2424

2525
private ready: Promise<void>;
2626

27+
private _onRunDataReceived: EventEmitter<DataReceivedEvent> = new EventEmitter<DataReceivedEvent>();
28+
29+
private _onDiscoveryDataReceived: EventEmitter<DataReceivedEvent> = new EventEmitter<DataReceivedEvent>();
30+
2731
constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) {
2832
this.server = net.createServer((socket: net.Socket) => {
2933
let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data
@@ -48,11 +52,28 @@ export class PythonTestServer implements ITestServer, Disposable {
4852
rawData = rpcHeaders.remainingRawData;
4953
const rpcContent = jsonRPCContent(rpcHeaders.headers, rawData);
5054
const extractedData = rpcContent.extractedJSON;
55+
// do not send until we have the full content
5156
if (extractedData.length === Number(totalContentLength)) {
52-
// do not send until we have the full content
53-
traceVerbose(`Received data from test server: ${extractedData}`);
54-
this._onDataReceived.fire({ uuid, data: extractedData });
55-
this.uuids = this.uuids.filter((u) => u !== uuid);
57+
// if the rawData includes tests then this is a discovery request
58+
if (rawData.includes(`"tests":`)) {
59+
this._onDiscoveryDataReceived.fire({
60+
uuid,
61+
data: rpcContent.extractedJSON,
62+
});
63+
// if the rawData includes result then this is a run request
64+
} else if (rawData.includes(`"result":`)) {
65+
this._onRunDataReceived.fire({
66+
uuid,
67+
data: rpcContent.extractedJSON,
68+
});
69+
} else {
70+
traceLog(
71+
`Error processing test server request: request is not recognized as discovery or run.`,
72+
);
73+
this._onDataReceived.fire({ uuid: '', data: '' });
74+
return;
75+
}
76+
// this.uuids = this.uuids.filter((u) => u !== uuid); WHERE DOES THIS GO??
5677
buffer = Buffer.alloc(0);
5778
} else {
5879
break;
@@ -97,6 +118,18 @@ export class PythonTestServer implements ITestServer, Disposable {
97118
return uuid;
98119
}
99120

121+
public deleteUUID(uuid: string): void {
122+
this.uuids = this.uuids.filter((u) => u !== uuid);
123+
}
124+
125+
public get onRunDataReceived(): Event<DataReceivedEvent> {
126+
return this._onRunDataReceived.event;
127+
}
128+
129+
public get onDiscoveryDataReceived(): Event<DataReceivedEvent> {
130+
return this._onDiscoveryDataReceived.event;
131+
}
132+
100133
public dispose(): void {
101134
this.server.close();
102135
this._onDataReceived.dispose();

‎src/client/testing/testController/common/types.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,21 @@ export type TestCommandOptionsPytest = {
172172
*/
173173
export interface ITestServer {
174174
readonly onDataReceived: Event<DataReceivedEvent>;
175+
readonly onRunDataReceived: Event<DataReceivedEvent>;
176+
readonly onDiscoveryDataReceived: Event<DataReceivedEvent>;
175177
sendCommand(options: TestCommandOptions, runTestIdsPort?: string, callback?: () => void): Promise<void>;
176178
serverReady(): Promise<void>;
177179
getPort(): number;
178180
createUUID(cwd: string): string;
181+
deleteUUID(uuid: string): void;
182+
}
183+
export interface ITestResultResolver {
184+
runIdToVSid: Map<string, string>;
185+
runIdToTestItem: Map<string, TestItem>;
186+
vsIdToRunId: Map<string, string>;
187+
resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise<void>;
188+
resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise<void>;
179189
}
180-
181190
export interface ITestDiscoveryAdapter {
182191
// ** first line old method signature, second line new method signature
183192
discoverTests(uri: Uri): Promise<DiscoveredTestPayload>;
@@ -192,6 +201,7 @@ export interface ITestExecutionAdapter {
192201
uri: Uri,
193202
testIds: string[],
194203
debugBool?: boolean,
204+
runInstance?: TestRun,
195205
executionFactory?: IPythonExecutionFactory,
196206
debugLauncher?: ITestDebugLauncher,
197207
): Promise<ExecutionTestPayload>;

0 commit comments

Comments
 (0)