Allow filerting backticks in AI code completion (#14777)

* Allow filerting backticks in AI code completion

fixed #14461

Signed-off-by: Jonas Helming <jhelming@eclipsesource.com>

Author: JonasHelming

packages/ai-code-completion/src/browser/ai-code-completion-frontend-module.ts

@@ -22,6 +22,7 @@ import { FrontendApplicationContribution, KeybindingContribution, PreferenceCont
 import { Agent } from '@theia/ai-core';
 import { AICodeCompletionPreferencesSchema } from './ai-code-completion-preference';
 import { AICodeInlineCompletionsProvider } from './ai-code-inline-completion-provider';
+import { CodeCompletionPostProcessor, DefaultCodeCompletionPostProcessor } from './code-completion-postprocessor';
 
 export default new ContainerModule(bind => {
     bind(ILogger).toDynamicValue(ctx => {
@@ -36,4 +37,5 @@ export default new ContainerModule(bind => {
     bind(FrontendApplicationContribution).to(AIFrontendApplicationContribution);
     bind(KeybindingContribution).toService(AIFrontendApplicationContribution);
     bind(PreferenceContribution).toConstantValue({ schema: AICodeCompletionPreferencesSchema });
+    bind(CodeCompletionPostProcessor).to(DefaultCodeCompletionPostProcessor).inSingletonScope();
 });

packages/ai-code-completion/src/browser/ai-code-completion-preference.ts

@@ -20,6 +20,7 @@ import { AI_CORE_PREFERENCES_TITLE } from '@theia/ai-core/lib/browser/ai-core-pr
 export const PREF_AI_INLINE_COMPLETION_AUTOMATIC_ENABLE = 'ai-features.codeCompletion.automaticCodeCompletion';
 export const PREF_AI_INLINE_COMPLETION_EXCLUDED_EXTENSIONS = 'ai-features.codeCompletion.excludedFileExtensions';
 export const PREF_AI_INLINE_COMPLETION_MAX_CONTEXT_LINES = 'ai-features.codeCompletion.maxContextLines';
+export const PREF_AI_INLINE_COMPLETION_STRIP_BACKTICKS = 'ai-features.codeCompletion.stripBackticks';
 
 export const AICodeCompletionPreferencesSchema: PreferenceSchema = {
     type: 'object',
@@ -48,6 +49,13 @@ export const AICodeCompletionPreferencesSchema: PreferenceSchema = {
             Set this to -1 to use the full file as context without any line limit and 0 to only use the current line.',
             default: -1,
             minimum: -1
+        },
+        [PREF_AI_INLINE_COMPLETION_STRIP_BACKTICKS]: {
+            title: 'Strip Backticks from Inline Completions',
+            type: 'boolean',
+            description: 'Remove surrounding backticks from the code returned by some LLMs. If a backtick is detected, all content after the closing\
+             backtick is stripped as well. This setting helps ensure plain code is returned when language models use markdown-like formatting.',
+            default: true
         }
     }
 };

packages/ai-code-completion/src/browser/code-completion-agent.ts

@@ -23,6 +23,7 @@ import { inject, injectable, named } from '@theia/core/shared/inversify';
 import * as monaco from '@theia/monaco-editor-core';
 import { PREF_AI_INLINE_COMPLETION_MAX_CONTEXT_LINES } from './ai-code-completion-preference';
 import { PreferenceService } from '@theia/core/lib/browser';
+import { CodeCompletionPostProcessor } from './code-completion-postprocessor';
 
 export const CodeCompletionAgent = Symbol('CodeCompletionAgent');
 export interface CodeCompletionAgent extends Agent {
@@ -142,8 +143,10 @@ export class CodeCompletionAgentImpl implements CodeCompletionAgent {
                 response: completionText,
             });
 
+            const postProcessedCompletionText = this.postProcessor.postProcess(completionText);
+
             return {
-                items: [{ insertText: completionText }],
+                items: [{ insertText: postProcessedCompletionText }],
                 enableForwardStability: true,
             };
         } catch (e) {
@@ -175,6 +178,9 @@ export class CodeCompletionAgentImpl implements CodeCompletionAgent {
     @inject(PreferenceService)
     protected preferences: PreferenceService;
 
+    @inject(CodeCompletionPostProcessor)
+    protected postProcessor: CodeCompletionPostProcessor;
+
     id = 'Code Completion';
     name = 'Code Completion';
     description =
@@ -190,7 +196,23 @@ Finish the following code snippet.
 
 {{prefix}}[[MARKER]]{{suffix}}
 
-Only return the exact replacement for [[MARKER]] to complete the snippet.`,
+Only return the exact replacement for [[MARKER]] to complete the snippet.`
+        },
+        {
+            id: 'code-completion-prompt-next',
+            variantOf: 'code-completion-prompt',
+            template: `{{!-- Made improvements or adaptations to this prompt template? We’d love for you to share it with the community! Contribute back here:
+https://github.com/eclipse-theia/theia/discussions/new?category=prompt-template-contribution --}}
+## Code snippet
+\`\`\`
+{{ prefix }}[[MARKER]]{{ suffix }}
+\`\`\`
+
+## Meta Data
+- File: {{file}}
+- Language: {{language}}
+
+Replace [[MARKER]] with the exact code to complete the code snippet. Return only the replacement of [[MAKRER]] as plain text.`,
         },
     ];
     languageModelRequirements: LanguageModelRequirement[] = [

packages/ai-code-completion/src/browser/code-completion-postprocessor.spec.ts

@@ -0,0 +1,84 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { enableJSDOM } from '@theia/core/lib/browser/test/jsdom';
+let disableJSDOM = enableJSDOM();
+import { FrontendApplicationConfigProvider } from '@theia/core/lib/browser/frontend-application-config-provider';
+FrontendApplicationConfigProvider.set({});
+
+import { expect } from 'chai';
+import { DefaultCodeCompletionPostProcessor } from './code-completion-postprocessor';
+
+disableJSDOM();
+
+describe('CodeCompletionAgentImpl', () => {
+    let codeCompletionProcessor: DefaultCodeCompletionPostProcessor;
+    before(() => {
+        disableJSDOM = enableJSDOM();
+        codeCompletionProcessor = new DefaultCodeCompletionPostProcessor();
+    });
+
+    after(() => {
+        // Disable JSDOM after all tests
+        disableJSDOM();
+    });
+
+    describe('stripBackticks', () => {
+
+        it('should remove surrounding backticks and language (TypeScript)', () => {
+            const input = '```TypeScript\nconsole.log(\"Hello, World!\");```';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('console.log("Hello, World!");');
+        });
+
+        it('should remove surrounding backticks and language (md)', () => {
+            const input = '```md\nconsole.log(\"Hello, World!\");```';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('console.log("Hello, World!");');
+        });
+
+        it('should remove all text after second occurrence of backticks', () => {
+            const input = '```js\nlet x = 10;\n```\nTrailing text should be removed';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('let x = 10;');
+        });
+
+        it('should return the text unchanged if no surrounding backticks', () => {
+            const input = 'console.log(\"Hello, World!\");';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('console.log("Hello, World!");');
+        });
+
+        it('should remove surrounding backticks without language', () => {
+            const input = '```\nconsole.log(\"Hello, World!\");```';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('console.log("Hello, World!");');
+        });
+
+        it('should handle text starting with backticks but no second delimiter', () => {
+            const input = '```python\nprint(\"Hello, World!\")';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('print("Hello, World!")');
+        });
+
+        it('should handle multiple internal backticks correctly', () => {
+            const input = '```\nFoo```Bar```FooBar```';
+            const output = codeCompletionProcessor.stripBackticks(input);
+            expect(output).to.equal('Foo```Bar```FooBar');
+        });
+
+    });
+});

packages/ai-code-completion/src/browser/code-completion-postprocessor.ts

@@ -0,0 +1,48 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { PreferenceService } from '@theia/core/lib/browser';
+import { PREF_AI_INLINE_COMPLETION_STRIP_BACKTICKS } from './ai-code-completion-preference';
+
+export interface CodeCompletionPostProcessor {
+    postProcess(text: string): string;
+}
+export const CodeCompletionPostProcessor = Symbol('CodeCompletionPostProcessor');
+
+@injectable()
+export class DefaultCodeCompletionPostProcessor {
+
+    @inject(PreferenceService)
+    protected readonly preferenceService: PreferenceService;
+
+    public postProcess(text: string): string {
+        if (this.preferenceService.get<boolean>(PREF_AI_INLINE_COMPLETION_STRIP_BACKTICKS, true)) {
+            return this.stripBackticks(text);
+        }
+        return text;
+    }
+
+    public stripBackticks(text: string): string {
+        if (text.startsWith('```')) {
+            // Remove the first backticks and any language identifier
+            const startRemoved = text.slice(3).replace(/^\w*\n/, '');
+            const lastBacktickIndex = startRemoved.lastIndexOf('```');
+            return lastBacktickIndex !== -1 ? startRemoved.slice(0, lastBacktickIndex).trim() : startRemoved.trim();
+        }
+        return text;
+    }
+}