Autocomplete: add providerModel to stage counter logger (sourcegrap…

…h#4699)
max-oyedele · Jun 28, 2024 · e17dceb · e17dceb
1 parent d69fe54
commit e17dceb
Show file tree

Hide file tree

Showing 3 changed files with 97 additions and 15 deletions.
diff --git a/vscode/src/completions/inline-completion-item-provider.ts b/vscode/src/completions/inline-completion-item-provider.ts
@@ -146,6 +146,8 @@ export class InlineCompletionItemProvider
             noInlineAccept: config.noInlineAccept ?? false,
         }
 
+        autocompleteStageCounterLogger.setProviderModel(config.providerConfig.model)
+
         if (this.config.completeSuggestWidgetSelection) {
             // This must be set to true, or else the suggest widget showing will suppress inline
             // completions. Note that the VS Code proposed API inlineCompletionsAdditions contains

diff --git a/vscode/src/services/autocomplete-stage-counter-logger.test.ts b/vscode/src/services/autocomplete-stage-counter-logger.test.ts
@@ -18,27 +18,19 @@ describe('AutocompleteStageCounter', () => {
         vi.clearAllTimers()
     })
 
-    it('returns initial state after LOG_INTERVAL', () => {
+    it('does not log empty counter events', () => {
+        logger.setProviderModel('test-model')
         vi.advanceTimersByTime(LOG_INTERVAL - 1)
 
         expect(recordSpy).not.toHaveBeenCalled()
 
         vi.advanceTimersByTime(1)
 
-        expect(recordSpy).toHaveBeenCalledWith('cody.completion.stageCounter', 'flush', {
-            metadata: {
-                preLastCandidate: 0,
-                preCache: 0,
-                preDebounce: 0,
-                preContextRetrieval: 0,
-                preNetworkRequest: 0,
-                preFinalCancellationCheck: 0,
-                preVisibilityCheck: 0,
-            },
-        })
+        expect(recordSpy).not.toHaveBeenCalled()
     })
 
     it('records state changes', () => {
+        logger.setProviderModel('test-model')
         logger.record('preLastCandidate')
         logger.record('preCache')
         logger.record('preDebounce')
@@ -59,10 +51,12 @@ describe('AutocompleteStageCounter', () => {
                 preFinalCancellationCheck: 1,
                 preVisibilityCheck: 1,
             },
+            privateMetadata: { providerModel: 'test-model' },
         })
     })
 
     it('resets state after flushing', () => {
+        logger.setProviderModel('test-model')
         logger.record('preLastCandidate')
         logger.record('preCache')
         logger.record('preCache')
@@ -79,6 +73,7 @@ describe('AutocompleteStageCounter', () => {
                 preFinalCancellationCheck: 0,
                 preVisibilityCheck: 0,
             },
+            privateMetadata: { providerModel: 'test-model' },
         })
 
         logger.record('preDebounce')
@@ -95,6 +90,72 @@ describe('AutocompleteStageCounter', () => {
                 preFinalCancellationCheck: 0,
                 preVisibilityCheck: 0,
             },
+            privateMetadata: { providerModel: 'test-model' },
+        })
+    })
+
+    it('includes providerModel in privateMetadata when set', () => {
+        logger.setProviderModel('test-model')
+        logger.record('preLastCandidate')
+
+        vi.advanceTimersByTime(LOG_INTERVAL)
+
+        expect(recordSpy).toHaveBeenCalledWith('cody.completion.stageCounter', 'flush', {
+            metadata: expect.any(Object),
+            privateMetadata: { providerModel: 'test-model' },
+        })
+    })
+
+    it('flushes when providerModel changes', () => {
+        logger.setProviderModel('model-1')
+        logger.record('preLastCandidate')
+
+        logger.setProviderModel('model-2')
+
+        expect(recordSpy).toHaveBeenCalledWith('cody.completion.stageCounter', 'flush', {
+            metadata: {
+                preLastCandidate: 1,
+                preCache: 0,
+                preDebounce: 0,
+                preContextRetrieval: 0,
+                preNetworkRequest: 0,
+                preFinalCancellationCheck: 0,
+                preVisibilityCheck: 0,
+            },
+            privateMetadata: { providerModel: 'model-1' },
+        })
+    })
+
+    it('does not flush when setting the same providerModel', () => {
+        logger.setProviderModel('test-model')
+        logger.record('preLastCandidate')
+
+        logger.setProviderModel('test-model')
+
+        expect(recordSpy).not.toHaveBeenCalled()
+    })
+
+    it('includes latest providerModel in flush after changes', () => {
+        logger.setProviderModel('model-1')
+        logger.record('preLastCandidate')
+
+        logger.setProviderModel('model-2')
+        logger.record('preCache')
+
+        vi.advanceTimersByTime(LOG_INTERVAL)
+
+        expect(recordSpy).toHaveBeenCalledTimes(2)
+        expect(recordSpy).toHaveBeenLastCalledWith('cody.completion.stageCounter', 'flush', {
+            metadata: {
+                preLastCandidate: 0,
+                preCache: 1,
+                preDebounce: 0,
+                preContextRetrieval: 0,
+                preNetworkRequest: 0,
+                preFinalCancellationCheck: 0,
+                preVisibilityCheck: 0,
+            },
+            privateMetadata: { providerModel: 'model-2' },
         })
     })
 })
diff --git a/vscode/src/services/autocomplete-stage-counter-logger.ts b/vscode/src/services/autocomplete-stage-counter-logger.ts
@@ -19,20 +19,34 @@ const INITIAL_STATE = {
 
 export class AutocompleteStageCounter implements vscode.Disposable {
     private nextTimeoutId: NodeJS.Timeout | null = null
+    private providerModel: string | null = null
     private currentState = { ...INITIAL_STATE }
 
     constructor() {
         this.nextTimeoutId = setTimeout(() => this.flush(), LOG_INTERVAL)
     }
 
+    public setProviderModel(providerModel: string): void {
+        // Flush the current counter on model change.
+        if (this.providerModel !== null && this.providerModel !== providerModel) {
+            this.flush()
+        }
+
+        this.providerModel = providerModel
+    }
+
     public flush(): void {
         this.nextTimeoutId = null
         const stateToLog = this.currentState
         this.currentState = { ...INITIAL_STATE }
 
-        telemetryRecorder.recordEvent('cody.completion.stageCounter', 'flush', {
-            metadata: stateToLog,
-        })
+        // Do not log empty counter events.
+        if (Object.values(stateToLog).some(count => count > 0)) {
+            telemetryRecorder.recordEvent('cody.completion.stageCounter', 'flush', {
+                metadata: stateToLog,
+                privateMetadata: { providerModel: this.providerModel },
+            })
+        }
 
         this.nextTimeoutId = setTimeout(() => this.flush(), LOG_INTERVAL)
     }
@@ -41,6 +55,11 @@ export class AutocompleteStageCounter implements vscode.Disposable {
      * Records the occurrence of a specific stage in the autocompletion generation pipeline.
      */
     public record(state: keyof typeof this.currentState): void {
+        if (!this.providerModel) {
+            // Do nothing if provider model is not set.
+            return
+        }
+
         this.currentState[state]++
     }