Allow execute on enter and Intellisense for native REPL with notebook UI (#23442)

"Smartly" allow execute on enter for the
microsoft#23235 experiment.
User should be able to execute when they press enter on text input box
of interactive window trigger from Python extension, whereas we would
"wait" and allow insertion of new line after detecting user's Python
command is not complete.

When the user finally types enter again on a blank line, we should just
proceed to execute whatever code, regardless of whether it is
complete/valid or not to replicate Python's original interactive REPL
behavior.

Basically creating Python command and registering that for keybinding of
'Enter'.
This would conditionally call interactive.execute which would then
eventually call our execute handler contributed from Python n
extension's REPL controller, or go ahead and insert,pass in Enter to the
text input box to allow user to type "complete" code.



This PR only intends to implement/add changes regarding execute on enter
logic, adding Intellisense support, and also adding things into
disposables so they can be properly disposed. Trying to also add setting
to allow toggling on/off to send Python command to Terminal or IW REPL
if the user is in experiment.


Handling of interrupt for windows should be on separate PR.
Test will be added later as separate PR.

---------

Co-authored-by: Courtney Webster <60238438+cwebster-99@users.noreply.github.com>
Co-authored-by: Karthik Nadig <kanadig@microsoft.com>

Author: 3 people

package.json

@@ -442,8 +442,7 @@
                             "pythonDiscoveryUsingWorkers",
                             "pythonTestAdapter",
                             "pythonREPLSmartSend",
-                            "pythonRecommendTensorboardExt",
-                            "pythonRunREPL"
+                            "pythonRecommendTensorboardExt"
                         ],
                         "enumDescriptions": [
                             "%python.experiments.All.description%",
@@ -453,8 +452,7 @@
                             "%python.experiments.pythonDiscoveryUsingWorkers.description%",
                             "%python.experiments.pythonTestAdapter.description%",
                             "%python.experiments.pythonREPLSmartSend.description%",
-                            "%python.experiments.pythonRecommendTensorboardExt.description%",
-                            "%python.experiments.pythonRunREPL.description%"
+                            "%python.experiments.pythonRecommendTensorboardExt.description%"
                         ]
                     },
                     "scope": "window",
@@ -472,8 +470,7 @@
                             "pythonTerminalEnvVarActivation",
                             "pythonDiscoveryUsingWorkers",
                             "pythonTestAdapter",
-                            "pythonREPLSmartSend",
-                            "pythonRunREPL"
+                            "pythonREPLSmartSend"
                         ],
                         "enumDescriptions": [
                             "%python.experiments.All.description%",
@@ -482,8 +479,7 @@
                             "%python.experiments.pythonTerminalEnvVarActivation.description%",
                             "%python.experiments.pythonDiscoveryUsingWorkers.description%",
                             "%python.experiments.pythonTestAdapter.description%",
-                            "%python.experiments.pythonREPLSmartSend.description%",
-                            "%python.experiments.pythonRunREPL.description%"
+                            "%python.experiments.pythonREPLSmartSend.description%"
                         ]
                     },
                     "scope": "window",
@@ -628,6 +624,15 @@
                     "scope": "resource",
                     "type": "boolean"
                 },
+                "python.REPL.sendToNativeREPL": {
+                    "default": false,
+                    "description": "%python.REPL.sendToNativeREPL.description%",
+                    "scope": "resource",
+                    "type": "boolean",
+                    "tags": [
+                        "experimental"
+                    ]
+                },
                 "python.testing.autoTestDiscoverOnSaveEnabled": {
                     "default": true,
                     "description": "%python.testing.autoTestDiscoverOnSaveEnabled.description%",
@@ -1108,7 +1113,22 @@
             {
                 "command": "python.execSelectionInTerminal",
                 "key": "shift+enter",
-                "when": "editorTextFocus && editorLangId == python && !findInputFocussed && !replaceInputFocussed && !jupyter.ownsSelection && !notebookEditorFocused && activeEditor != 'workbench.editor.interactive'"
+                "when": "!config.python.REPL.sendToNativeREPL && editorTextFocus && editorLangId == python && !findInputFocussed && !replaceInputFocussed && !jupyter.ownsSelection && !notebookEditorFocused && activeEditor != 'workbench.editor.interactive'"
+            },
+            {
+                "command": "python.execInREPL",
+                "key": "shift+enter",
+                "when": "config.python.REPL.sendToNativeREPL && activeEditor != 'workbench.editor.interactive'"
+            },
+            {
+                "command": "python.execREPLShiftEnter",
+                "key": "shift+enter",
+                "when": "activeEditor == 'workbench.editor.interactive' && config.interactiveWindow.executeWithShiftEnter"
+            },
+            {
+                "command": "python.execInREPLEnter",
+                "key": "enter",
+                "when": "!config.interactiveWindow.executeWithShiftEnter && activeEditor == 'workbench.editor.interactive'"
             },
             {
                 "command": "python.refreshTensorBoard",
@@ -1367,12 +1387,12 @@
                 {
                     "command": "python.execSelectionInTerminal",
                     "group": "Python",
-                    "when": "editorFocus && editorLangId == python && !virtualWorkspace && shellExecutionSupported"
+                    "when": "!config.python.REPL.sendToNativeREPL && editorFocus && editorLangId == python && !virtualWorkspace && shellExecutionSupported"
                 },
                 {
                     "command": "python.execInREPL",
                     "group": "Python",
-                    "when": "editorFocus && editorLangId == python && !virtualWorkspace && shellExecutionSupported && pythonRunREPL"
+                    "when": "editorFocus && editorLangId == python && !virtualWorkspace && shellExecutionSupported && config.python.REPL.sendToNativeREPL"
                 }
             ],
             "editor/title": [

package.nls.json

@@ -45,7 +45,6 @@
     "python.experiments.pythonTestAdapter.description": "Denotes the Python Test Adapter experiment.",
     "python.experiments.pythonREPLSmartSend.description": "Denotes the Python REPL Smart Send experiment.",
     "python.experiments.pythonRecommendTensorboardExt.description": "Denotes the Tensorboard Extension recommendation experiment.",
-    "python.experiments.pythonRunREPL.description": "Enables users to run code in interactive Python REPL.",
     "python.globalModuleInstallation.description": "Whether to install Python modules globally when not using an environment.",
     "python.languageServer.description": "Defines type of the language server.",
     "python.languageServer.defaultDescription": "Automatically select a language server: Pylance if installed and available, otherwise fallback to Jedi.",
@@ -63,6 +62,7 @@
     "python.pipenvPath.description": "Path to the pipenv executable to use for activation.",
     "python.poetryPath.description": "Path to the poetry executable.",
     "python.EnableREPLSmartSend.description": "Toggle Smart Send for the Python REPL. Smart Send enables sending the smallest runnable block of code to the REPL on Shift+Enter and moves the cursor accordingly.",
+    "python.REPL.sendToNativeREPL.description": "Toggle to send code to Python REPL instead of the terminal on execution. Turning this on will change the behavior for both Smart Send and Run Selection/Line in the Context Menu.",
     "python.tensorBoard.logDirectory.description": "Set this setting to your preferred TensorBoard log directory to skip log directory prompt when starting TensorBoard.",
     "python.tensorBoard.logDirectory.markdownDeprecationMessage": "Tensorboard support has been moved to the extension [Tensorboard extension](https://marketplace.visualstudio.com/items?itemName=ms-toolsai.tensorboard). Instead use the setting `tensorBoard.logDirectory`.",
     "python.tensorBoard.logDirectory.deprecationMessage": "Tensorboard support has been moved to the extension Tensorboard extension. Instead use the setting `tensorBoard.logDirectory`.",

python_files/python_server.py

@@ -1,11 +1,11 @@
 from typing import Dict, List, Optional, Union
-
 import sys
 import json
 import contextlib
 import io
 import traceback
 import uuid
+import ast
 
 STDIN = sys.stdin
 STDOUT = sys.stdout
@@ -88,6 +88,15 @@ def exec_function(user_input):
     return eval
 
 
+def check_valid_command(request):
+    try:
+        user_input = request["params"]
+        ast.parse(user_input[0])
+        send_response("True", request["id"])
+    except SyntaxError:
+        send_response("False", request["id"])
+
+
 def execute(request, user_globals):
     str_output = CustomIO("<stdout>", encoding="utf-8")
     str_error = CustomIO("<stderr>", encoding="utf-8")
@@ -160,6 +169,8 @@ def get_headers():
                 request_json = json.loads(request_text)
                 if request_json["method"] == "execute":
                     execute(request_json, USER_GLOBALS)
+                if request_json["method"] == "check_valid_command":
+                    check_valid_command(request_json)
                 elif request_json["method"] == "exit":
                     sys.exit(0)
 

src/client/common/application/commands.ts

@@ -39,6 +39,7 @@ interface ICommandNameWithoutArgumentTypeMapping {
     [Commands.Exec_Selection_In_Terminal]: [];
     [Commands.Exec_Selection_In_Django_Shell]: [];
     [Commands.Exec_In_REPL]: [];
+    [Commands.Exec_In_REPL_Enter]: [];
     [Commands.Create_Terminal]: [];
     [Commands.PickLocalProcess]: [];
     [Commands.ClearStorage]: [];

src/client/common/constants.ts

@@ -48,6 +48,8 @@ export namespace Commands {
     export const Exec_In_Separate_Terminal = 'python.execInDedicatedTerminal';
     export const Exec_In_REPL = 'python.execInREPL';
     export const Exec_Selection_In_Django_Shell = 'python.execSelectionInDjangoShell';
+    export const Exec_In_REPL_Enter = 'python.execInREPLEnter';
+    export const Exec_In_REPL_Shift_Enter = 'python.execREPLShiftEnter';
     export const Exec_Selection_In_Terminal = 'python.execSelectionInTerminal';
     export const GetSelectedInterpreterPath = 'python.interpreterPath';
     export const InstallJupyter = 'python.installJupyter';

src/client/common/experiments/groups.ts

@@ -30,8 +30,3 @@ export enum RecommendTensobardExtension {
 export enum CreateEnvOnPipInstallTrigger {
     experiment = 'pythonCreateEnvOnPipInstall',
 }
-
-// Experiment to enable running Python REPL using IW.
-export enum EnableRunREPL {
-    experiment = 'pythonRunREPL',
-}

src/client/common/types.ts

@@ -200,7 +200,7 @@ export interface ITerminalSettings {
 
 export interface IREPLSettings {
     readonly enableREPLSmartSend: boolean;
-    readonly enableIWREPL: boolean;
+    readonly sendToNativeREPL: boolean;
 }
 
 export interface IExperiments {

src/client/common/vscodeApis/windowApis.ts

@@ -21,6 +21,8 @@ import {
     TerminalShellExecutionStartEvent,
 } from 'vscode';
 import { createDeferred, Deferred } from '../utils/async';
+import { Resource } from '../types';
+import { getWorkspaceFolders } from './workspaceApis';
 
 export function showTextDocument(uri: Uri): Thenable<TextEditor> {
     return window.showTextDocument(uri);
@@ -238,3 +240,12 @@ export function createStepForwardEndNode<T>(deferred?: Deferred<T>, result?: T):
         undefined,
     );
 }
+
+export function getActiveResource(): Resource {
+    const editor = window.activeTextEditor;
+    if (editor && !editor.document.isUntitled) {
+        return editor.document.uri;
+    }
+    const workspaces = getWorkspaceFolders();
+    return Array.isArray(workspaces) && workspaces.length > 0 ? workspaces[0].uri : undefined;
+}

src/client/extensionActivation.ts

@@ -3,7 +3,7 @@
 
 'use strict';
 
-import { DebugConfigurationProvider, debug, languages, window, commands } from 'vscode';
+import { DebugConfigurationProvider, debug, languages, window } from 'vscode';
 
 import { registerTypes as activationRegisterTypes } from './activation/serviceRegistry';
 import { IExtensionActivationManager } from './activation/types';
@@ -16,7 +16,6 @@ import { IFileSystem } from './common/platform/types';
 import {
     IConfigurationService,
     IDisposableRegistry,
-    IExperimentService,
     IExtensions,
     IInterpreterPathService,
     ILogOutputChannel,
@@ -53,8 +52,7 @@ import { initializePersistentStateForTriggers } from './common/persistentState';
 import { logAndNotifyOnLegacySettings } from './logging/settingLogs';
 import { DebuggerTypeName } from './debugger/constants';
 import { StopWatch } from './common/utils/stopWatch';
-import { registerReplCommands } from './repl/replCommands';
-import { EnableRunREPL } from './common/experiments/groups';
+import { registerReplCommands, registerReplExecuteOnEnter, registerReplExecuteOnShiftEnter } from './repl/replCommands';
 
 export async function activateComponents(
     // `ext` is passed to any extra activation funcs.
@@ -109,16 +107,9 @@ export function activateFeatures(ext: ExtensionState, _components: Components):
         pathUtils,
     );
 
-    // Register native REPL context menu when in experiment
-    const experimentService = ext.legacyIOC.serviceContainer.get<IExperimentService>(IExperimentService);
-    commands.executeCommand('setContext', 'pythonRunREPL', false);
-    if (experimentService) {
-        const replExperimentValue = experimentService.inExperimentSync(EnableRunREPL.experiment);
-        if (replExperimentValue) {
-            registerReplCommands(ext.disposables, interpreterService);
-            commands.executeCommand('setContext', 'pythonRunREPL', true);
-        }
-    }
+    registerReplCommands(ext.disposables, interpreterService);
+    registerReplExecuteOnEnter(ext.disposables, interpreterService);
+    registerReplExecuteOnShiftEnter(ext.disposables);
 }
 
 /// //////////////////////////

src/client/repl/pythonServer.ts

@@ -6,27 +6,32 @@ import { EXTENSION_ROOT_DIR } from '../constants';
 import { traceError, traceLog } from '../logging';
 
 const SERVER_PATH = path.join(EXTENSION_ROOT_DIR, 'python_files', 'python_server.py');
+let serverInstance: PythonServer | undefined;
 
 export interface PythonServer extends Disposable {
     execute(code: string): Promise<string>;
     interrupt(): void;
     input(): void;
+    checkValidCommand(code: string): Promise<boolean>;
 }
 
 class PythonServerImpl implements Disposable {
+    private readonly disposables: Disposable[] = [];
+
     constructor(private connection: rpc.MessageConnection, private pythonServer: ch.ChildProcess) {
         this.initialize();
         this.input();
     }
 
     private initialize(): void {
-        this.connection.onNotification('log', (message: string) => {
-            console.log('Log:', message);
-        });
+        this.disposables.push(
+            this.connection.onNotification('log', (message: string) => {
+                console.log('Log:', message);
+            }),
+        );
         this.connection.listen();
     }
 
-    // Register input handler
     public input(): void {
         // Register input request handler
         this.connection.onRequest('input', async (request) => {
@@ -49,18 +54,32 @@ class PythonServerImpl implements Disposable {
     }
 
     public interrupt(): void {
+        // Passing SIGINT to interrupt only would work for Mac and Linux
         if (this.pythonServer.kill('SIGINT')) {
-            traceLog('Python server interrupted');
+            traceLog('Python REPL server interrupted');
+        }
+    }
+
+    public async checkValidCommand(code: string): Promise<boolean> {
+        const completeCode = await this.connection.sendRequest('check_valid_command', code);
+        if (completeCode === 'True') {
+            return new Promise((resolve) => resolve(true));
         }
+        return new Promise((resolve) => resolve(false));
     }
 
     public dispose(): void {
         this.connection.sendNotification('exit');
+        this.disposables.forEach((d) => d.dispose());
         this.connection.dispose();
     }
 }
 
 export function createPythonServer(interpreter: string[]): PythonServer {
+    if (serverInstance) {
+        return serverInstance;
+    }
+
     const pythonServer = ch.spawn(interpreter[0], [...interpreter.slice(1), SERVER_PATH]);
 
     pythonServer.stderr.on('data', (data) => {
@@ -76,6 +95,6 @@ export function createPythonServer(interpreter: string[]): PythonServer {
         new rpc.StreamMessageReader(pythonServer.stdout),
         new rpc.StreamMessageWriter(pythonServer.stdin),
     );
-
-    return new PythonServerImpl(connection, pythonServer);
+    serverInstance = new PythonServerImpl(connection, pythonServer);
+    return serverInstance;
 }