From b081608e06f7eedd542c63fab738050a525abcf5 Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Wed, 25 Sep 2024 15:54:44 -0700 Subject: [PATCH 1/4] fix exception --- agent/src/cli/command-bench/command-bench.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agent/src/cli/command-bench/command-bench.ts b/agent/src/cli/command-bench/command-bench.ts index aecb77880ef..12528add34c 100644 --- a/agent/src/cli/command-bench/command-bench.ts +++ b/agent/src/cli/command-bench/command-bench.ts @@ -14,6 +14,7 @@ import { isDefined, modelsService } from '@sourcegraph/cody-shared' import { sleep } from '../../../../vscode/src/completions/utils' import { setStaticResolvedConfigurationWithAuthCredentials } from '../../../../vscode/src/configuration' import { localStorage } from '../../../../vscode/src/services/LocalStorageProvider' +import { createOrUpdateTelemetryRecorderProvider } from '../../../../vscode/src/services/telemetry-v2' import { startPollyRecording } from '../../../../vscode/src/testutils/polly' import { dotcomCredentials } from '../../../../vscode/src/testutils/testing-credentials' import { allClientCapabilitiesEnabled } from '../../allClientCapabilitiesEnabled' @@ -352,6 +353,8 @@ export const benchCommand = new commander.Command('bench') async function evaluateWorkspace(options: CodyBenchOptions, recordingDirectory: string): Promise { console.log(`starting evaluation: fixture=${options.fixture.name} workspace=${options.workspace}`) + createOrUpdateTelemetryRecorderProvider(true) + const workspaceRootUri = vscode.Uri.from({ scheme: 'file', path: options.workspace }) const baseGlobalState: Record = {} From 7b63223f2fd47ee36c6dd3eee93474dc46a2335a Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Thu, 26 Sep 2024 10:03:07 -0700 Subject: [PATCH 2/4] emit yaml file with additional metadata, incorporate revision and timestamp into output filename --- .../strategy-chat-context-types.ts | 33 +++++++++ .../command-bench/strategy-chat-context.ts | 71 ++++++++++++++----- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/agent/src/cli/command-bench/strategy-chat-context-types.ts b/agent/src/cli/command-bench/strategy-chat-context-types.ts index 9119de4b042..6f7159660d1 100644 --- a/agent/src/cli/command-bench/strategy-chat-context-types.ts +++ b/agent/src/cli/command-bench/strategy-chat-context-types.ts @@ -4,6 +4,24 @@ import { parse } from 'csv-parse/sync' import { createObjectCsvWriter } from 'csv-writer' import { mkdirp } from 'fs-extra' import isError from 'lodash/isError' +import { stringify as yamlStringify } from 'yaml' + +export interface ClientOptions { + rewrite: boolean +} + +export interface EvalOutput { + evaluatedAt: string + clientOptions: ClientOptions + siteUserMetadata: { + url: string + version: string + username: string + userId: string + evaluatedFeatureFlags: Record + } + examples: ExampleOutput[] +} export interface EvalContextItem { repoName: string @@ -159,6 +177,21 @@ export async function readExamplesFromCSV(filePath: string): Promise<{ } } +/** + * Note: this mutates evalOutput to remove the content field from actualContext context items. + */ +export async function writeYAMLMetadata(outputFile: string, evalOutput: EvalOutput): Promise { + await mkdirp(path.dirname(outputFile)) + + for (const example of evalOutput.examples) { + for (const contextItem of example.actualContext) { + contextItem.content = undefined + } + } + + await fs.writeFile(outputFile, yamlStringify(evalOutput)) +} + export async function writeExamplesToCSV(outputFile: string, examples: ExampleOutput[]): Promise { await mkdirp(path.dirname(outputFile)) const csvWriter = createObjectCsvWriter({ diff --git a/agent/src/cli/command-bench/strategy-chat-context.ts b/agent/src/cli/command-bench/strategy-chat-context.ts index f8537f5db3d..0f2ecd26882 100644 --- a/agent/src/cli/command-bench/strategy-chat-context.ts +++ b/agent/src/cli/command-bench/strategy-chat-context.ts @@ -3,6 +3,7 @@ import { graphqlClient, isError } from '@sourcegraph/cody-shared' import type { RpcMessageHandler } from '../../jsonrpc-alias' import type { CodyBenchOptions } from './command-bench' import { + type ClientOptions, type EvalContextItem, type Example, type ExampleOutput, @@ -10,6 +11,7 @@ import { contextItemToString, readExamplesFromCSV, writeExamplesToCSV, + writeYAMLMetadata, } from './strategy-chat-context-types' export async function evaluateChatContextStrategy( @@ -17,35 +19,72 @@ export async function evaluateChatContextStrategy( options: CodyBenchOptions ): Promise { const inputFilename = options.fixture.customConfiguration?.['cody-bench.chatContext.inputFile'] + if (options.insecureTls) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + } + if (!inputFilename) { throw new Error( 'Missing cody-bench.chatContext.inputFile. To fix this problem, add "customConfiguration": { "cody-bench.chatContext.inputFile": "examples.csv" } to the cody-bench JSON config.' ) } - const outputFilename = options.fixture.customConfiguration?.['cody-bench.chatContext.outputFile'] - if (!outputFilename) { - throw new Error( - 'Missing cody-bench.chatContext.outputFile. To fix this problem, add "customConfiguration": { "cody-bench.chatContext.outputFile": "output.csv" } to the cody-bench JSON config.' - ) - } - const inputFile = path.join(options.workspace, inputFilename) - const outputFile = path.join(options.snapshotDirectory, outputFilename) + const inputBasename = path.basename(inputFilename).replace(/\.csv$/, '') - if (options.insecureTls) { - process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + const siteVersion = await graphqlClient.getSiteVersion() + if (isError(siteVersion)) { + throw siteVersion + } + const userInfo = await graphqlClient.getCurrentUserInfo() + if (isError(userInfo)) { + throw userInfo + } + const evaluatedFeatureFlags = await graphqlClient.getEvaluatedFeatureFlags() + if (isError(evaluatedFeatureFlags)) { + throw evaluatedFeatureFlags } + const shortSiteVersion = siteVersion.match(/-[0-9a-f]{7,40}$/) + ? siteVersion.match(/-([0-9a-f]{7,40})$/)?.[1] + : siteVersion + const currentTimestamp = new Date().toISOString() + + const outputBase = `${inputBasename}__${shortSiteVersion}` + const outputCSVFilename = `${outputBase}.csv` + const outputYAMLFilename = `${outputBase}.yaml` + + const inputFile = path.join(options.workspace, inputFilename) + const outputCSVFile = path.join(options.snapshotDirectory, outputCSVFilename) + const outputYAMLFile = path.join(options.snapshotDirectory, outputYAMLFilename) const { examples, ignoredRecords } = await readExamplesFromCSV(inputFile) - console.error(`ignoring ${ignoredRecords.length} malformed rows`) - if (!outputFile) { - throw new Error('no output file specified') + if (ignoredRecords.length > 0) { + console.log(`⚠ ignoring ${ignoredRecords.length} malformed rows`) } - await runContextCommand(examples, outputFile) + const rewrite = false // TODO(beyang): make configurable + + const outputs = await runContextCommand({ rewrite }, examples) + await writeExamplesToCSV(outputCSVFile, outputs) + await writeYAMLMetadata(outputYAMLFile, { + evaluatedAt: currentTimestamp, + clientOptions: { + rewrite, + }, + siteUserMetadata: { + url: options.srcEndpoint, + version: siteVersion, + username: userInfo?.username ?? '[none]', + userId: userInfo?.id ?? '[none]', + evaluatedFeatureFlags, + }, + examples: outputs, + }) } -async function runContextCommand(examples: Example[], outputFile: string): Promise { +async function runContextCommand( + ops: ClientOptions, // TODO(beyang) + examples: Example[] +): Promise { const exampleOutputs: ExampleOutput[] = [] for (const example of examples) { @@ -94,7 +133,7 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi }) } - await writeExamplesToCSV(outputFile, exampleOutputs) + return exampleOutputs } function contextOverlaps( From 8a81cdec5a813913bd3d2d738bb1c2b0102daca9 Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Thu, 26 Sep 2024 15:52:03 -0700 Subject: [PATCH 3/4] context internal bench > chat-context: add ClientOptions.rewrite --- .../command-bench/strategy-chat-context.ts | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/agent/src/cli/command-bench/strategy-chat-context.ts b/agent/src/cli/command-bench/strategy-chat-context.ts index 0f2ecd26882..f3cdd09d34a 100644 --- a/agent/src/cli/command-bench/strategy-chat-context.ts +++ b/agent/src/cli/command-bench/strategy-chat-context.ts @@ -1,5 +1,12 @@ import path from 'node:path' -import { graphqlClient, isError } from '@sourcegraph/cody-shared' +import { + PromptString, + SourcegraphCompletionsClient, + graphqlClient, + isError, +} from '@sourcegraph/cody-shared' +import { SourcegraphNodeCompletionsClient } from '../../../../vscode/src/completions/nodeClient' +import { rewriteKeywordQuery } from '../../../../vscode/src/local-context/rewrite-keyword-query' import type { RpcMessageHandler } from '../../jsonrpc-alias' import type { CodyBenchOptions } from './command-bench' import { @@ -22,7 +29,6 @@ export async function evaluateChatContextStrategy( if (options.insecureTls) { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' } - if (!inputFilename) { throw new Error( 'Missing cody-bench.chatContext.inputFile. To fix this problem, add "customConfiguration": { "cody-bench.chatContext.inputFile": "examples.csv" } to the cody-bench JSON config.' @@ -30,6 +36,12 @@ export async function evaluateChatContextStrategy( } const inputBasename = path.basename(inputFilename).replace(/\.csv$/, '') + const clientOptions: ClientOptions = options.fixture.customConfiguration?.[ + 'cody-bench.chatContext.clientOptions' + ] ?? { + rewrite: false, + } + const siteVersion = await graphqlClient.getSiteVersion() if (isError(siteVersion)) { throw siteVersion @@ -61,15 +73,11 @@ export async function evaluateChatContextStrategy( console.log(`⚠ ignoring ${ignoredRecords.length} malformed rows`) } - const rewrite = false // TODO(beyang): make configurable - - const outputs = await runContextCommand({ rewrite }, examples) + const outputs = await runContextCommand({ rewrite: clientOptions.rewrite }, examples) await writeExamplesToCSV(outputCSVFile, outputs) await writeYAMLMetadata(outputYAMLFile, { evaluatedAt: currentTimestamp, - clientOptions: { - rewrite, - }, + clientOptions, siteUserMetadata: { url: options.srcEndpoint, version: siteVersion, @@ -82,13 +90,14 @@ export async function evaluateChatContextStrategy( } async function runContextCommand( - ops: ClientOptions, // TODO(beyang) + clientOps: ClientOptions, examples: Example[] ): Promise { + const completionsClient = new SourcegraphNodeCompletionsClient() const exampleOutputs: ExampleOutput[] = [] for (const example of examples) { - const { targetRepoRevs, query, essentialContext } = example + const { targetRepoRevs, query: origQuery, essentialContext } = example const repoNames = targetRepoRevs.map(repoRev => repoRev.repoName) const repoIDNames = await graphqlClient.getRepoIds(repoNames, repoNames.length + 10) if (isError(repoIDNames)) { @@ -102,6 +111,15 @@ async function runContextCommand( ) } const repoIDs = repoIDNames.map(repoIDName => repoIDName.id) + + let query = origQuery + if (clientOps.rewrite) { + query = await rewriteKeywordQuery( + completionsClient, + PromptString.unsafe_fromUserQuery(origQuery) + ) + } + const resultsResp = await graphqlClient.contextSearch({ repoIDs, query, From 7372e652eeaadaf4c5f78c0819ccc3d3e09a1c45 Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Thu, 26 Sep 2024 16:06:39 -0700 Subject: [PATCH 4/4] track cody clientversion --- agent/src/cli/command-bench/strategy-chat-context-types.ts | 3 ++- agent/src/cli/command-bench/strategy-chat-context.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/agent/src/cli/command-bench/strategy-chat-context-types.ts b/agent/src/cli/command-bench/strategy-chat-context-types.ts index 6f7159660d1..abc0d6dd2e4 100644 --- a/agent/src/cli/command-bench/strategy-chat-context-types.ts +++ b/agent/src/cli/command-bench/strategy-chat-context-types.ts @@ -12,10 +12,11 @@ export interface ClientOptions { export interface EvalOutput { evaluatedAt: string + codyClientVersion: string clientOptions: ClientOptions siteUserMetadata: { url: string - version: string + sourcegraphVersion: string username: string userId: string evaluatedFeatureFlags: Record diff --git a/agent/src/cli/command-bench/strategy-chat-context.ts b/agent/src/cli/command-bench/strategy-chat-context.ts index f3cdd09d34a..df7cb174fb8 100644 --- a/agent/src/cli/command-bench/strategy-chat-context.ts +++ b/agent/src/cli/command-bench/strategy-chat-context.ts @@ -7,6 +7,7 @@ import { } from '@sourcegraph/cody-shared' import { SourcegraphNodeCompletionsClient } from '../../../../vscode/src/completions/nodeClient' import { rewriteKeywordQuery } from '../../../../vscode/src/local-context/rewrite-keyword-query' +import { version } from '../../../package.json' import type { RpcMessageHandler } from '../../jsonrpc-alias' import type { CodyBenchOptions } from './command-bench' import { @@ -74,13 +75,15 @@ export async function evaluateChatContextStrategy( } const outputs = await runContextCommand({ rewrite: clientOptions.rewrite }, examples) + const codyClientVersion = process.env.CODY_COMMIT ?? version await writeExamplesToCSV(outputCSVFile, outputs) await writeYAMLMetadata(outputYAMLFile, { evaluatedAt: currentTimestamp, clientOptions, + codyClientVersion, siteUserMetadata: { url: options.srcEndpoint, - version: siteVersion, + sourcegraphVersion: siteVersion, username: userInfo?.username ?? '[none]', userId: userInfo?.id ?? '[none]', evaluatedFeatureFlags,