From c53f672b6173ebda530917ccb4e0c2f26f5c8f79 Mon Sep 17 00:00:00 2001 From: tsmithsz <84354541+tsmithsz@users.noreply.github.com> Date: Fri, 29 Aug 2025 10:22:49 -0700 Subject: [PATCH 01/10] fix: emit acceptedLineCount metric and AgenticCodeAccepted interaction type (#2167) --- .../agenticChat/agenticChatController.ts | 13 +++++++++++++ .../chat/telemetry/chatTelemetryController.ts | 4 +++- .../src/shared/telemetry/telemetryService.ts | 1 + .../src/shared/telemetry/types.ts | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index 101674a4d9..4a816317d2 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -2076,6 +2076,19 @@ export class AgenticChatController implements ChatHandlers { this.#abTestingAllocation?.experimentName, this.#abTestingAllocation?.userVariation ) + // Emit acceptedLineCount when write tool is used and code changes are accepted + const beforeLines = cachedToolUse?.fileChange?.before?.split('\n').length ?? 0 + const afterLines = doc?.getText()?.split('\n').length ?? 0 + const acceptedLineCount = afterLines - beforeLines + await this.#telemetryController.emitInteractWithMessageMetric( + tabId, + { + cwsprChatMessageId: chatResult.messageId ?? toolUse.toolUseId, + cwsprChatInteractionType: ChatInteractionType.AgenticCodeAccepted, + codewhispererCustomizationArn: this.#customizationArn, + }, + acceptedLineCount + ) await chatResultStream.writeResultBlock(chatResult) break case CodeReview.toolName: diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts index 8e9cb92624..580bcd4be2 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts @@ -421,10 +421,12 @@ export class ChatTelemetryController { public emitInteractWithMessageMetric( tabId: string, - metric: Omit + metric: Omit, + acceptedLineCount?: number ) { return this.#telemetryService.emitChatInteractWithMessage(metric, { conversationId: this.getConversationId(tabId), + acceptedLineCount, }) } diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts index 6f3ba52028..e5e8cbc471 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts @@ -66,6 +66,7 @@ export class TelemetryService { [ChatInteractionType.Upvote]: 'UPVOTE', [ChatInteractionType.Downvote]: 'DOWNVOTE', [ChatInteractionType.ClickBodyLink]: 'CLICK_BODY_LINK', + [ChatInteractionType.AgenticCodeAccepted]: 'AGENTIC_CODE_ACCEPTED', } constructor( diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts index bed82c2939..5bf9fea185 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts @@ -429,6 +429,7 @@ export enum ChatInteractionType { Upvote = 'upvote', Downvote = 'downvote', ClickBodyLink = 'clickBodyLink', + AgenticCodeAccepted = 'agenticCodeAccepted', } export enum ChatHistoryActionType { From 852b21b66f793102c52e35c2baec07a772e5134a Mon Sep 17 00:00:00 2001 From: Will Lo <96078566+Will-ShaoHua@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:28:28 -0700 Subject: [PATCH 02/10] fix: auto trigger should only respect previous decisions in the past 2mins (#2189) --- .../auto-trigger/autoTrigger.ts | 21 ++++++++++++------- .../inline-completion/codeWhispererServer.ts | 8 +++++-- .../session/sessionManager.ts | 7 +++++++ .../src/shared/telemetry/telemetryService.ts | 1 + 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts index d9dd7f16d5..adbbeb4662 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts @@ -177,7 +177,7 @@ type AutoTriggerParams = { char: string triggerType: string // Left as String intentionally to support future and unknown trigger types os: string - previousDecision: string + previousDecision: string | undefined ide: string lineNum: number } @@ -235,12 +235,19 @@ export const autoTrigger = ( const languageCoefficient = coefficients.languageCoefficient[fileContext.programmingLanguage.languageName] ?? 0 let previousDecisionCoefficient = 0 - if (previousDecision === 'Accept') { - previousDecisionCoefficient = coefficients.prevDecisionAcceptCoefficient - } else if (previousDecision === 'Reject') { - previousDecisionCoefficient = coefficients.prevDecisionRejectCoefficient - } else if (previousDecision === 'Discard' || previousDecision === 'Empty') { - previousDecisionCoefficient = coefficients.prevDecisionOtherCoefficient + switch (previousDecision) { + case 'Accept': + previousDecisionCoefficient = coefficients.prevDecisionAcceptCoefficient + break + case 'Reject': + previousDecisionCoefficient = coefficients.prevDecisionRejectCoefficient + break + case 'Discard': + case 'Empty': + previousDecisionCoefficient = coefficients.prevDecisionOtherCoefficient + break + default: + break } const ideCoefficient = coefficients.ideCoefficient[ide] ?? 0 diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/codeWhispererServer.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/codeWhispererServer.ts index 330cf54735..c869a534fd 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/codeWhispererServer.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/codeWhispererServer.ts @@ -239,7 +239,11 @@ export const CodewhispererServerFactory = : undefined const previousSession = completionSessionManager.getPreviousSession() - const previousDecision = previousSession?.getAggregatedUserTriggerDecision() ?? '' + // Only refer to decisions in the past 2 mins + const previousDecisionForClassifier = + previousSession && performance.now() - previousSession.decisionMadeTimestamp <= 2 * 60 * 1000 + ? previousSession.getAggregatedUserTriggerDecision() + : undefined let ideCategory: string | undefined = '' const initializeParams = lsp.getClientInitializeParams() if (initializeParams !== undefined) { @@ -280,7 +284,7 @@ export const CodewhispererServerFactory = char: triggerCharacters, // Add the character just inserted, if any, before the invication position ide: ideCategory ?? '', os: getNormalizeOsName(), - previousDecision, // The last decision by the user on the previous invocation + previousDecision: previousDecisionForClassifier, // The last decision by the user on the previous invocation triggerType: codewhispererAutoTriggerType, // The 2 trigger types currently influencing the Auto-Trigger are SpecialCharacter and Enter }, logging diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/session/sessionManager.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/session/sessionManager.ts index 34dbb12538..2a4af79ff6 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/session/sessionManager.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/session/sessionManager.ts @@ -60,6 +60,13 @@ export class CodeWhispererSession { suggestions: CachedSuggestion[] = [] suggestionsAfterRightContextMerge: InlineCompletionItemWithReferences[] = [] suggestionsStates = new Map() + private _decisionTimestamp = 0 + get decisionMadeTimestamp() { + return this._decisionTimestamp + } + set decisionMadeTimestamp(time: number) { + this._decisionTimestamp = time + } acceptedSuggestionId?: string = undefined responseContext?: ResponseContext triggerType: CodewhispererTriggerType diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts index e5e8cbc471..b11806a020 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts @@ -200,6 +200,7 @@ export class TelemetryService { removedIdeDiagnostics?: IdeDiagnostic[], streakLength?: number ) { + session.decisionMadeTimestamp = performance.now() if (this.enableTelemetryEventsToDestination) { const data: CodeWhispererUserTriggerDecisionEvent = { codewhispererSessionId: session.codewhispererSessionId || '', From 4fd0def2a7080921683dd712cae9c17a39d4372e Mon Sep 17 00:00:00 2001 From: Will Lo <96078566+Will-ShaoHua@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:28:54 -0700 Subject: [PATCH 03/10] chore: supplemental context log (#2185) --- .../src/shared/codeWhispererService.ts | 2 ++ .../crossFileContextUtil.ts | 10 ++++------ .../supplementalContextUtil.ts | 19 ++++++++++++++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts index 2304d81eea..984bb6da76 100644 --- a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts @@ -444,6 +444,8 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { "endpoint": ${this.codeWhispererEndpoint}, "predictionType": ${request.predictionTypes?.toString() ?? 'Not specified (COMPLETIONS)'}, "filename": ${request.fileContext.filename}, + "leftContextLength": ${request.fileContext.leftFileContent.length}, + rightContextLength: ${request.fileContext.rightFileContent.length}, "language": ${request.fileContext.programmingLanguage.languageName}, "supplementalContextCount": ${request.supplementalContexts?.length ?? 0}, "request.nextToken": ${request.nextToken}, diff --git a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts index 06bac73d54..91f02b9dc7 100644 --- a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts +++ b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts @@ -71,12 +71,13 @@ export async function fetchSupplementalContextForSrc( const supplementalContextConfig = getSupplementalContextConfig(document.languageId) if (supplementalContextConfig === undefined) { - return supplementalContextConfig + return undefined } - //TODO: add logic for other strategies once available + if (supplementalContextConfig === 'codemap') { return await codemapContext(document, position, workspace, cancellationToken, openTabFiles) } + return { supplementalContextItems: [], strategy: 'Empty' } } @@ -264,10 +265,7 @@ function getInputChunk(document: TextDocument, cursorPosition: Position, chunkSi * @returns specifically returning undefined if the langueage is not supported, * otherwise true/false depending on if the language is fully supported or not belonging to the user group */ -function getSupplementalContextConfig( - languageId: TextDocument['languageId'], - _userGroup?: any -): SupplementalContextStrategy | undefined { +function getSupplementalContextConfig(languageId: TextDocument['languageId']): SupplementalContextStrategy | undefined { return isCrossFileSupported(languageId) ? 'codemap' : undefined } diff --git a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/supplementalContextUtil.ts b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/supplementalContextUtil.ts index 8a5658188b..8e0902452f 100644 --- a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/supplementalContextUtil.ts +++ b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/supplementalContextUtil.ts @@ -91,7 +91,24 @@ export async function fetchSupplementalContext( strategy: supplementalContextValue.strategy, } - return truncateSupplementalContext(resBeforeTruncation) + const r = truncateSupplementalContext(resBeforeTruncation) + + let logstr = `@@supplemental context@@ +\tisUtg: ${r.isUtg}, +\tisProcessTimeout: ${r.isProcessTimeout}, +\tcontents.length: ${r.contentsLength}, +\tlatency: ${r.latency}, +\tstrategy: ${r.strategy}, +` + r.supplementalContextItems.forEach((item, index) => { + logstr += `\tChunk [${index}th]:\n` + logstr += `\t\tPath: ${item.filePath}\n` + logstr += `\t\tLength: ${item.content.length}\n` + logstr += `\t\tScore: ${item.score}\n` + }) + logging.info(logstr) + + return r } else { return undefined } From f4e2e6e885e665834a5d7b7cbb5f4ba4ff9bbb65 Mon Sep 17 00:00:00 2001 From: Will Lo <96078566+Will-ShaoHua@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:29:11 -0700 Subject: [PATCH 04/10] fix: should send classifier score after taking sigmoid (#2188) --- .../inline-completion/auto-trigger/autoTrigger.ts | 8 ++++++-- .../inline-completion/userTriggerDecision.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts index adbbeb4662..8990756dbf 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/auto-trigger/autoTrigger.ts @@ -229,7 +229,9 @@ export const autoTrigger = ( const triggerTypeCoefficient = coefficients.triggerTypeCoefficient[triggerType] ?? 0 const osCoefficient = coefficients.osCoefficient[os] ?? 0 + const charCoefficient = coefficients.charCoefficient[char] ?? 0 + const keyWordCoefficient = coefficients.charCoefficient[keyword] ?? 0 const languageCoefficient = coefficients.languageCoefficient[fileContext.programmingLanguage.languageName] ?? 0 @@ -281,11 +283,13 @@ export const autoTrigger = ( previousDecisionCoefficient + languageCoefficient + leftContextLengthCoefficient - const shouldTrigger = sigmoid(classifierResult) > TRIGGER_THRESHOLD + + const r = sigmoid(classifierResult) + const shouldTrigger = r > TRIGGER_THRESHOLD return { shouldTrigger, - classifierResult, + classifierResult: r, classifierThreshold: TRIGGER_THRESHOLD, } } diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/userTriggerDecision.test.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/userTriggerDecision.test.ts index c583815b6b..21c398d56a 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/userTriggerDecision.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/userTriggerDecision.test.ts @@ -145,7 +145,7 @@ describe('Telemetry', () => { }, } const EMPTY_RESULT = { items: [], sessionId: '' } - const classifierResult = getNormalizeOsName() !== 'Linux' ? 0.4114381148145918 : 0.46733811481459187 + const classifierResult = getNormalizeOsName() !== 'Linux' ? 0.6014326616203989 : 0.61475353067264 let features: TestFeatures let server: Server @@ -1251,7 +1251,7 @@ describe('Telemetry', () => { triggerType: 'AutoTrigger', autoTriggerType: 'SpecialCharacters', triggerCharacter: '(', - classifierResult: getNormalizeOsName() === 'Linux' ? 0.30173811481459184 : 0.2458381148145919, + classifierResult: getNormalizeOsName() === 'Linux' ? 0.5748673583477094 : 0.5611518554232429, classifierThreshold: 0.43, language: 'csharp', requestContext: { From b4975409a3ed518550290b72ac310895a293be4b Mon Sep 17 00:00:00 2001 From: Will Lo <96078566+Will-ShaoHua@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:29:44 -0700 Subject: [PATCH 05/10] perf: only process edit requests 1 at a time (#2187) --- .../inline-completion/editCompletionHandler.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/editCompletionHandler.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/editCompletionHandler.ts index 193521cce9..550de06708 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/editCompletionHandler.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/editCompletionHandler.ts @@ -45,6 +45,8 @@ export class EditCompletionHandler { private hasDocumentChangedSinceInvocation: boolean = false private readonly streakTracker: StreakTracker + private isInProgress = false + constructor( readonly logging: Logging, readonly clientMetadata: InitializeParams, @@ -89,6 +91,11 @@ export class EditCompletionHandler { params: InlineCompletionWithReferencesParams, token: CancellationToken ): Promise { + if (this.isInProgress) { + this.logging.info(`editCompletionHandler is WIP, skip the request`) + return EMPTY_RESULT + } + // On every new completion request close current inflight session. const currentSession = this.sessionManager.getCurrentSession() if (currentSession && currentSession.state == 'REQUESTING' && !params.partialResultToken) { @@ -118,6 +125,9 @@ export class EditCompletionHandler { return EMPTY_RESULT } + // Not ideally to rely on a state, should improve it and simply make it a debounced API + this.isInProgress = true + if (params.partialResultToken && currentSession) { // Close ACTIVE session. We shouldn't record Discard trigger decision for trigger with nextToken. if (currentSession && currentSession.state === 'ACTIVE') { @@ -152,10 +162,12 @@ export class EditCompletionHandler { ) } catch (error) { return this.handleSuggestionsErrors(error as Error, currentSession) + } finally { + this.isInProgress = false } } - return new Promise(async resolve => { + return new Promise(async resolve => { this.debounceTimeout = setTimeout(async () => { try { this.isWaiting = true @@ -185,6 +197,8 @@ export class EditCompletionHandler { this.hasDocumentChangedSinceInvocation = false } }, EDIT_DEBOUNCE_INTERVAL_MS) + }).finally(() => { + this.isInProgress = false }) } From 66742adfc44f33efbd8dd33b803000e08241e5ce Mon Sep 17 00:00:00 2001 From: atontb <104926752+atonaamz@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:12:15 -0700 Subject: [PATCH 06/10] feat: passing suggestionTypes and pluginVersion/lspVersion to STE (#2180) --- .../src/client/token/bearer-token-service.json | 13 +++++++++++++ .../token/codewhispererbearertokenclient.d.ts | 6 +++++- .../src/shared/telemetry/telemetryService.test.ts | 1 + .../src/shared/telemetry/telemetryService.ts | 5 ++++- .../src/shared/telemetryUtils.ts | 2 ++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/client/token/bearer-token-service.json b/server/aws-lsp-codewhisperer/src/client/token/bearer-token-service.json index 7e0533b0f5..58ae54fb3c 100644 --- a/server/aws-lsp-codewhisperer/src/client/token/bearer-token-service.json +++ b/server/aws-lsp-codewhisperer/src/client/token/bearer-token-service.json @@ -1890,6 +1890,10 @@ "type": "string", "enum": ["BLOCK", "LINE"] }, + "SuggestionType": { + "type": "string", + "enum": ["COMPLETIONS", "EDITS"] + }, "Completions": { "type": "list", "member": { @@ -6299,6 +6303,12 @@ }, "ideVersion": { "shape": "String" + }, + "pluginVersion": { + "shape": "String" + }, + "lspVersion": { + "shape": "String" } } }, @@ -6540,6 +6550,9 @@ }, "streakLength": { "shape": "UserTriggerDecisionEventStreakLengthInteger" + }, + "suggestionType": { + "shape": "SuggestionType" } } }, diff --git a/server/aws-lsp-codewhisperer/src/client/token/codewhispererbearertokenclient.d.ts b/server/aws-lsp-codewhisperer/src/client/token/codewhispererbearertokenclient.d.ts index 34aa384c14..8e64023479 100644 --- a/server/aws-lsp-codewhisperer/src/client/token/codewhispererbearertokenclient.d.ts +++ b/server/aws-lsp-codewhisperer/src/client/token/codewhispererbearertokenclient.d.ts @@ -549,6 +549,7 @@ declare namespace CodeWhispererBearerTokenClient { } export type CompletionContentString = string; export type CompletionType = "BLOCK"|"LINE"|string; + export type SuggestionType = "COMPLETIONS"|"EDITS"|string; export type Completions = Completion[]; export interface ConsoleState { region?: String; @@ -2014,6 +2015,8 @@ declare namespace CodeWhispererBearerTokenClient { product: UserContextProductString; clientId?: UUID; ideVersion?: String; + pluginVersion?: String; + lspVersion?: String; } export type UserContextProductString = string; export interface UserInputMessage { @@ -2127,6 +2130,7 @@ declare namespace CodeWhispererBearerTokenClient { addedCharacterCount?: UserTriggerDecisionEventAddedCharacterCountInteger; deletedCharacterCount?: UserTriggerDecisionEventDeletedCharacterCountInteger; streakLength?: UserTriggerDecisionEventStreakLengthInteger; + suggestionType?: SuggestionType; } export type UserTriggerDecisionEventAddedCharacterCountInteger = number; export type UserTriggerDecisionEventDeletedCharacterCountInteger = number; @@ -2182,4 +2186,4 @@ declare namespace CodeWhispererBearerTokenClient { } export = CodeWhispererBearerTokenClient; - \ No newline at end of file + \ No newline at end of file diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts index 04d1cf091c..b6a5592f5f 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts @@ -277,6 +277,7 @@ describe('TelemetryService', () => { addedIdeDiagnostics: undefined, removedIdeDiagnostics: undefined, streakLength: 0, + suggestionType: 'COMPLETIONS', }, }, optOutPreference: 'OPTIN', diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts index b11806a020..cd2ba18254 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts @@ -283,14 +283,17 @@ export class TelemetryService { addedIdeDiagnostics: addedIdeDiagnostics, removedIdeDiagnostics: removedIdeDiagnostics, streakLength: streakLength ?? 0, + suggestionType: isInlineEdit ? 'EDITS' : 'COMPLETIONS', } this.logging.info(`Invoking SendTelemetryEvent:UserTriggerDecisionEvent with: + "requestId": ${event.requestId} "suggestionState": ${event.suggestionState} "acceptedCharacterCount": ${event.acceptedCharacterCount} "addedCharacterCount": ${event.addedCharacterCount} "deletedCharacterCount": ${event.deletedCharacterCount} "streakLength": ${event.streakLength} - "firstCompletionDisplayLatency: ${event.recommendationLatencyMilliseconds}`) + "firstCompletionDisplayLatency: ${event.recommendationLatencyMilliseconds} + "suggestionType": ${event.suggestionType}`) return this.invokeSendTelemetryEvent({ userTriggerDecisionEvent: event, }) diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetryUtils.ts b/server/aws-lsp-codewhisperer/src/shared/telemetryUtils.ts index 49020a8822..e682263f7e 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetryUtils.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetryUtils.ts @@ -103,6 +103,8 @@ export const makeUserContextObject = ( product: product, clientId: initializeParams.initializationOptions?.aws?.clientInfo?.clientId, ideVersion: `ide=${ideVersion};plugin=${pluginVersion};lsp=${lspVersion}`, + pluginVersion: pluginVersion, + lspVersion: lspVersion, } if (userContext.ideCategory === 'UNKNOWN' || userContext.operatingSystem === 'UNKNOWN') { From ef7d7931954f5083e4a5c358e67c6dc652fa1a40 Mon Sep 17 00:00:00 2001 From: Jason Guo <81202082+jguoamz@users.noreply.github.com> Date: Fri, 29 Aug 2025 16:15:27 -0700 Subject: [PATCH 07/10] fix: compact UI is not updated correctly when multiple nudges are displayed (#2192) --- .../language-server/agenticChat/agenticChatController.ts | 1 + .../src/language-server/chat/chatSessionService.ts | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index 4a816317d2..4daa31db64 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -2707,6 +2707,7 @@ export class AgenticChatController implements ChatHandlers { session.setDeferredToolExecution(messageId, deferred.resolve, deferred.reject) this.#log(`Prompting for compaction approval for messageId: ${messageId}`) await deferred.promise + session.removeDeferredToolExecution(messageId) // Note: we want to overwrite the button block because it already exists in the stream. await resultStream.overwriteResultBlock(this.#getUpdateCompactConfirmResult(messageId), promptBlockId) } diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts index 92c7eb2c33..13c198a949 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts @@ -78,10 +78,17 @@ export class ChatSessionService { public getDeferredToolExecution(messageId: string): DeferredHandler | undefined { return this.#deferredToolExecution[messageId] } + public setDeferredToolExecution(messageId: string, resolve: any, reject: any) { this.#deferredToolExecution[messageId] = { resolve, reject } } + public removeDeferredToolExecution(messageId: string) { + if (messageId in this.#deferredToolExecution) { + delete this.#deferredToolExecution[messageId] + } + } + public getAllDeferredCompactMessageIds(): string[] { return Object.keys(this.#deferredToolExecution).filter(messageId => messageId.endsWith('_compact')) } From fd71e6cf3fc843242936564061061418edf83f56 Mon Sep 17 00:00:00 2001 From: tsmithsz <84354541+tsmithsz@users.noreply.github.com> Date: Tue, 2 Sep 2025 11:23:18 -0700 Subject: [PATCH 08/10] fix: fix calculation for num-lines contributed by the LLM (#2191) --- .../agenticChat/agenticChatController.ts | 5 +- .../utils/fileModificationMetrics.test.ts | 139 ++++++++++++++++++ .../utils/fileModificationMetrics.ts | 58 ++++++++ 3 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.test.ts create mode 100644 server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.ts diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index 4daa31db64..6c3f07fac2 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -166,6 +166,7 @@ import { FsWrite, FsWriteParams } from './tools/fsWrite' import { ExecuteBash, ExecuteBashParams } from './tools/executeBash' import { ExplanatoryParams, InvokeOutput, ToolApprovalException } from './tools/toolShared' import { validatePathBasic, validatePathExists, validatePaths as validatePathsSync } from './utils/pathValidation' +import { calculateModifiedLines } from './utils/fileModificationMetrics' import { GrepSearch, SanitizedRipgrepOutput } from './tools/grepSearch' import { FileSearch, FileSearchParams, isFileSearchParams } from './tools/fileSearch' import { FsReplace, FsReplaceParams } from './tools/fsReplace' @@ -2077,9 +2078,7 @@ export class AgenticChatController implements ChatHandlers { this.#abTestingAllocation?.userVariation ) // Emit acceptedLineCount when write tool is used and code changes are accepted - const beforeLines = cachedToolUse?.fileChange?.before?.split('\n').length ?? 0 - const afterLines = doc?.getText()?.split('\n').length ?? 0 - const acceptedLineCount = afterLines - beforeLines + const acceptedLineCount = calculateModifiedLines(toolUse, doc?.getText()) await this.#telemetryController.emitInteractWithMessageMetric( tabId, { diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.test.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.test.ts new file mode 100644 index 0000000000..900b569e7e --- /dev/null +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.test.ts @@ -0,0 +1,139 @@ +import { calculateModifiedLines } from './fileModificationMetrics' +import { ToolUse } from '@amzn/codewhisperer-streaming' +import { FS_WRITE, FS_REPLACE } from '../constants/toolConstants' +import * as assert from 'assert' + +describe('calculateModifiedLines', () => { + describe('FS_WRITE', () => { + it('should count lines for create command', () => { + const toolUse: ToolUse = { + toolUseId: 'test-1', + name: FS_WRITE, + input: { + command: 'create', + path: '/test/file.txt', + fileText: 'line1\nline2\nline3', + }, + } + const afterContent = 'line1\nline2\nline3' + + assert.strictEqual(calculateModifiedLines(toolUse, afterContent), 3) + }) + + it('should count lines for append command', () => { + const toolUse: ToolUse = { + toolUseId: 'test-2', + name: FS_WRITE, + input: { + command: 'append', + path: '/test/file.txt', + fileText: 'line4\nline5', + }, + } + + assert.strictEqual(calculateModifiedLines(toolUse), 2) + }) + + it('should handle empty content', () => { + const toolUse: ToolUse = { + toolUseId: 'test-3', + name: FS_WRITE, + input: { + command: 'create', + path: '/test/file.txt', + fileText: '', + }, + } + + assert.strictEqual(calculateModifiedLines(toolUse, ''), 0) + }) + }) + + describe('FS_REPLACE', () => { + it('should count replaced lines correctly (double counting)', () => { + const toolUse: ToolUse = { + toolUseId: 'test-4', + name: FS_REPLACE, + input: { + path: '/test/file.txt', + diffs: [ + { + oldStr: 'old line 1\nold line 2\nold line 3', + newStr: 'new line 1\nnew line 2\nnew line 3', + }, + ], + }, + } + + assert.strictEqual(calculateModifiedLines(toolUse), 6) + }) + + it('should count pure deletions', () => { + const toolUse: ToolUse = { + toolUseId: 'test-5', + name: FS_REPLACE, + input: { + path: '/test/file.txt', + diffs: [ + { + oldStr: 'line to delete 1\nline to delete 2', + newStr: '', + }, + ], + }, + } + + assert.strictEqual(calculateModifiedLines(toolUse), 2) + }) + + it('should count pure insertions', () => { + const toolUse: ToolUse = { + toolUseId: 'test-6', + name: FS_REPLACE, + input: { + path: '/test/file.txt', + diffs: [ + { + oldStr: '', + newStr: 'new line 1\nnew line 2', + }, + ], + }, + } + + assert.strictEqual(calculateModifiedLines(toolUse), 2) + }) + + it('should handle multiple diffs', () => { + const toolUse: ToolUse = { + toolUseId: 'test-7', + name: FS_REPLACE, + input: { + path: '/test/file.txt', + diffs: [ + { + oldStr: 'old line 1', + newStr: 'new line 1', + }, + { + oldStr: 'delete this line', + newStr: '', + }, + ], + }, + } + + assert.strictEqual(calculateModifiedLines(toolUse), 3) + }) + }) + + it('should return 0 for unknown tools', () => { + const toolUse: ToolUse = { + toolUseId: 'test-8', + name: 'unknownTool', + input: {}, + } + + assert.strictEqual(calculateModifiedLines(toolUse), 0) + }) +}) diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.ts new file mode 100644 index 0000000000..361c886607 --- /dev/null +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/utils/fileModificationMetrics.ts @@ -0,0 +1,58 @@ +import { ToolUse } from '@amzn/codewhisperer-streaming' +import { diffLines } from 'diff' +import { FsWriteParams } from '../tools/fsWrite' +import { FsReplaceParams } from '../tools/fsReplace' +import { FS_WRITE, FS_REPLACE } from '../constants/toolConstants' + +/** + * Counts the number of lines in text, handling different line endings + * @param text The text to count lines in + * @returns The number of lines + */ +function countLines(text?: string): number { + if (!text) return 0 + const parts = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n') + return parts.length && parts[parts.length - 1] === '' ? parts.length - 1 : parts.length +} + +/** + * Calculates the actual lines modified by analyzing file modification tools. + * @param toolUse The tool use object + * @param afterContent The content after the tool execution (for FS_WRITE create operations) + * @returns The total number of lines modified (added + removed) + */ +export function calculateModifiedLines(toolUse: ToolUse, afterContent?: string): number { + if (toolUse.name === FS_WRITE) { + const input = toolUse.input as unknown as FsWriteParams + + if (input.command === 'create') { + return countLines(afterContent ?? '') + } else if (input.command === 'append') { + return countLines(input.fileText) + } + } + + if (toolUse.name === FS_REPLACE) { + const input = toolUse.input as unknown as FsReplaceParams + let linesAdded = 0 + let linesRemoved = 0 + + for (const diff of input.diffs || []) { + const oldStr = diff.oldStr ?? '' + const newStr = diff.newStr ?? '' + + const changes = diffLines(oldStr, newStr) + + for (const change of changes) { + if (change.added) { + linesAdded += countLines(change.value) + } else if (change.removed) { + linesRemoved += countLines(change.value) + } + } + } + + return linesAdded + linesRemoved + } + return 0 +} From 94723d46073a1ea8211e7ae8f9dfce3fcb809604 Mon Sep 17 00:00:00 2001 From: Will Lo <96078566+Will-ShaoHua@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:36:57 -0700 Subject: [PATCH 09/10] revert: PR 2172 dedupe openTabs supplemental contexts (#2194) This reverts commit aa87ae2bd95edc1f38bf90f56093c5bf5ff18c53. --- .../supplementalContextUtil/crossFileContextUtil.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts index 91f02b9dc7..39e97a3b8c 100644 --- a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts +++ b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts @@ -210,17 +210,8 @@ export async function fetchOpenTabsContext( }) } - // Dedupe code chunks based on their filePath + content unique key - const seen = new Set() - const deduped = supplementalContexts.filter(item => { - const key = `${item.filePath}:${item.content}` - if (seen.has(key)) return false - seen.add(key) - return true - }) - // DO NOT send code chunk with empty content - return deduped.filter(item => item.content.trim().length !== 0) + return supplementalContexts.filter(item => item.content.trim().length !== 0) } function findBestKChunkMatches(chunkInput: Chunk, chunkReferences: Chunk[], k: number): Chunk[] { From 58f20649d345f159080006120e23cde559826df1 Mon Sep 17 00:00:00 2001 From: chungjac Date: Tue, 2 Sep 2025 12:56:21 -0700 Subject: [PATCH 10/10] fix: emit errorMessage in addMessage (#2197) * emit errorMessage in addMessage * fix: emit errorMessage in addMessage --- .../language-server/agenticChat/agenticChatController.ts | 6 +++--- .../chat/telemetry/chatTelemetryController.ts | 8 +++++++- .../src/shared/telemetry/telemetryService.test.ts | 1 + .../src/shared/telemetry/telemetryService.ts | 2 ++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index 6c3f07fac2..47739e77d1 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -3371,7 +3371,7 @@ export class AgenticChatController implements ChatHandlers { metric: Metric, agenticCodingMode: boolean ): Promise> { - const errorMessage = getErrorMsg(err) + const errorMessage = getErrorMsg(err) ?? GENERIC_ERROR_MS const requestID = getRequestID(err) ?? '' metric.setDimension('cwsprChatResponseCode', getHttpStatusCode(err) ?? 0) metric.setDimension('languageServerVersion', this.#features.runtime.serverInfo.version) @@ -3381,7 +3381,7 @@ export class AgenticChatController implements ChatHandlers { metric.metric.requestIds = [requestID] metric.metric.cwsprChatMessageId = errorMessageId metric.metric.cwsprChatConversationId = conversationId - await this.#telemetryController.emitAddMessageMetric(tabId, metric.metric, 'Failed') + await this.#telemetryController.emitAddMessageMetric(tabId, metric.metric, 'Failed', errorMessage) if (isUsageLimitError(err)) { if (this.#paidTierMode !== 'paidtier') { @@ -3426,7 +3426,7 @@ export class AgenticChatController implements ChatHandlers { tabId, metric.metric, requestID, - errorMessage ?? GENERIC_ERROR_MS, + errorMessage, agenticCodingMode ) } diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts index 580bcd4be2..797a1f640a 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts @@ -294,7 +294,12 @@ export class ChatTelemetryController { }) } - public emitAddMessageMetric(tabId: string, metric: Partial, result?: string) { + public emitAddMessageMetric( + tabId: string, + metric: Partial, + result?: string, + errorMessage?: string + ) { const conversationId = this.getConversationId(tabId) // Store the customization value associated with the message if (metric.cwsprChatMessageId && metric.codewhispererCustomizationArn) { @@ -349,6 +354,7 @@ export class ChatTelemetryController { requestIds: metric.requestIds, experimentName: metric.experimentName, userVariation: metric.userVariation, + errorMessage: errorMessage, } ) } diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts index b6a5592f5f..d66ada7707 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts @@ -859,6 +859,7 @@ describe('TelemetryService', () => { cwsprChatPinnedFileContextCount: undefined, cwsprChatPinnedFolderContextCount: undefined, cwsprChatPinnedPromptContextCount: undefined, + errorMessage: undefined, }, }) }) diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts index cd2ba18254..78182c076f 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts @@ -564,6 +564,7 @@ export class TelemetryService { requestIds?: string[] experimentName?: string userVariation?: string + errorMessage?: string }> ) { const timeBetweenChunks = params.timeBetweenChunks?.slice(0, 100) @@ -617,6 +618,7 @@ export class TelemetryService { requestIds: truncatedRequestIds, experimentName: additionalParams.experimentName, userVariation: additionalParams.userVariation, + errorMessage: additionalParams.errorMessage, }, }) }