Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Response Ops][Test] Debugging Flaky Tests #192892

Draft
wants to merge 24 commits into
base: main
Choose a base branch
from
Draft

[Response Ops][Test] Debugging Flaky Tests #192892

Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
09b7a7f
remove not necesary test + activate logs
jcger Sep 13, 2024
65265af
rename and deactivate custom env
jcger Sep 13, 2024
da2b8c3
perf logs in tests
jcger Sep 16, 2024
cb18b1f
remove unnecessary code
jcger Sep 16, 2024
86be37e
quick_checks exit 0
jcger Sep 16, 2024
7d7cdc6
remove integration and functional test
jcger Sep 16, 2024
473bc4f
fix ts error
jcger Sep 16, 2024
c8e8db0
fix ts warnings
jcger Sep 16, 2024
cb0a9d1
reduce amount of tests + wrong var
jcger Sep 16, 2024
7d5c733
better logging
jcger Sep 16, 2024
e5e3c11
run test onceo nly
jcger Sep 16, 2024
0443ef4
skip broken test + cleaup call
jcger Sep 16, 2024
1ae3d58
remove prob unnecessary step
jcger Sep 16, 2024
23629b7
add another cleanup
jcger Sep 16, 2024
4a15624
activate react query error logs
jcger Sep 17, 2024
1dde245
remove query log + add state change logs
jcger Sep 17, 2024
536d088
wrap render fn in act + cache staleTime 0
jcger Sep 17, 2024
2f2b4b4
remove stale time
jcger Sep 17, 2024
57d7377
add stale time
jcger Sep 17, 2024
928bb4c
memo AppWrapper
jcger Sep 18, 2024
e8727c0
rollback some changes and logs
jcger Sep 18, 2024
815f1b4
remove logs + provider obj
jcger Sep 18, 2024
4232af7
reduce amount of tests
jcger Sep 18, 2024
82181f2
run all again and add cleanup to failing test
jcger Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix ts warnings
  • Loading branch information
@jcger
jcger committed Sep 16, 2024
commit c8e8db0e731709f0e8b914014513d6973510ac9e
196 changes: 96 additions & 100 deletions .buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,15 @@
import * as Fs from 'fs';

import * as globby from 'globby';
import minimatch from 'minimatch';

// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
import { load as loadYaml } from 'js-yaml';

import { BuildkiteClient } from '../buildkite';
import { CiStatsClient, TestGroupRunOrderResponse } from './client';

import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
import { serverless, stateful } from '../../ftr_configs_manifests.json';
// import { serverless, stateful } from '../../ftr_configs_manifests.json';
import { expandAgentQueue } from '#pipeline-utils';

const ALL_FTR_MANIFEST_REL_PATHS = serverless.concat(stateful);
// const ALL_FTR_MANIFEST_REL_PATHS = serverless.concat(stateful);

type RunGroup = TestGroupRunOrderResponse['types'][0];

Expand Down Expand Up @@ -109,93 +105,93 @@ function getTrackedBranch(): string {
return branch;
}

function isObj(x: unknown): x is Record<string, unknown> {
return typeof x === 'object' && x !== null;
}

interface FtrConfigsManifest {
defaultQueue?: string;
disabled?: string[];
enabled?: Array<string | { [configPath: string]: { queue: string } }>;
}

function getEnabledFtrConfigs(patterns?: string[]) {
const configs: {
enabled: Array<string | { [configPath: string]: { queue: string } }>;
defaultQueue: string | undefined;
} = { enabled: [], defaultQueue: undefined };
const uniqueQueues = new Set<string>();

for (const manifestRelPath of ALL_FTR_MANIFEST_REL_PATHS) {
try {
const ymlData = loadYaml(Fs.readFileSync(manifestRelPath, 'utf8'));
if (!isObj(ymlData)) {
throw new Error('expected yaml file to parse to an object');
}
const manifest = ymlData as FtrConfigsManifest;

configs.enabled.push(...(manifest?.enabled ?? []));
if (manifest.defaultQueue) {
uniqueQueues.add(manifest.defaultQueue);
}
} catch (_) {
const error = _ instanceof Error ? _ : new Error(`${_} thrown`);
throw new Error(`unable to parse ${manifestRelPath} file: ${error.message}`);
}
}

try {
if (configs.enabled.length === 0) {
throw new Error('expected yaml files to have at least 1 "enabled" key');
}
if (uniqueQueues.size !== 1) {
throw Error(
`FTR manifest yml files should define the same 'defaultQueue', but found different ones: ${[
...uniqueQueues,
].join(' ')}`
);
}
configs.defaultQueue = uniqueQueues.values().next().value;

if (
!Array.isArray(configs.enabled) ||
!configs.enabled.every(
(p): p is string | { [configPath: string]: { queue: string } } =>
typeof p === 'string' ||
(isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string'))
)
) {
throw new Error(`expected "enabled" value to be an array of strings or objects shaped as:\n
- {configPath}:
queue: {queueName}`);
}
if (typeof configs.defaultQueue !== 'string') {
throw new Error('expected yaml file to have a string "defaultQueue" key');
}

const defaultQueue = configs.defaultQueue;
const ftrConfigsByQueue = new Map<string, string[]>();
for (const enabled of configs.enabled) {
const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0];
const queue = isObj(enabled) ? enabled[path].queue : defaultQueue;

if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) {
continue;
}

const group = ftrConfigsByQueue.get(queue);
if (group) {
group.push(path);
} else {
ftrConfigsByQueue.set(queue, [path]);
}
}
return { defaultQueue, ftrConfigsByQueue };
} catch (_) {
const error = _ instanceof Error ? _ : new Error(`${_} thrown`);
throw new Error(`unable to collect enabled FTR configs: ${error.message}`);
}
}
// function isObj(x: unknown): x is Record<string, unknown> {
// return typeof x === 'object' && x !== null;
// }
//
// interface FtrConfigsManifest {
// defaultQueue?: string;
// disabled?: string[];
// enabled?: Array<string | { [configPath: string]: { queue: string } }>;
// }

// function getEnabledFtrConfigs(patterns?: string[]) {
// const configs: {
// enabled: Array<string | { [configPath: string]: { queue: string } }>;
// defaultQueue: string | undefined;
// } = { enabled: [], defaultQueue: undefined };
// const uniqueQueues = new Set<string>();
//
// for (const manifestRelPath of ALL_FTR_MANIFEST_REL_PATHS) {
// try {
// const ymlData = loadYaml(Fs.readFileSync(manifestRelPath, 'utf8'));
// if (!isObj(ymlData)) {
// throw new Error('expected yaml file to parse to an object');
// }
// const manifest = ymlData as FtrConfigsManifest;
//
// configs.enabled.push(...(manifest?.enabled ?? []));
// if (manifest.defaultQueue) {
// uniqueQueues.add(manifest.defaultQueue);
// }
// } catch (_) {
// const error = _ instanceof Error ? _ : new Error(`${_} thrown`);
// throw new Error(`unable to parse ${manifestRelPath} file: ${error.message}`);
// }
// }
//
// try {
// if (configs.enabled.length === 0) {
// throw new Error('expected yaml files to have at least 1 "enabled" key');
// }
// if (uniqueQueues.size !== 1) {
// throw Error(
// `FTR manifest yml files should define the same 'defaultQueue', but found different ones: ${[
// ...uniqueQueues,
// ].join(' ')}`
// );
// }
// configs.defaultQueue = uniqueQueues.values().next().value;
//
// if (
// !Array.isArray(configs.enabled) ||
// !configs.enabled.every(
// (p): p is string | { [configPath: string]: { queue: string } } =>
// typeof p === 'string' ||
// (isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string'))
// )
// ) {
// throw new Error(`expected "enabled" value to be an array of strings or objects shaped as:\n
// - {configPath}:
// queue: {queueName}`);
// }
// if (typeof configs.defaultQueue !== 'string') {
// throw new Error('expected yaml file to have a string "defaultQueue" key');
// }
//
// const defaultQueue = configs.defaultQueue;
// const ftrConfigsByQueue = new Map<string, string[]>();
// for (const enabled of configs.enabled) {
// const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0];
// const queue = isObj(enabled) ? enabled[path].queue : defaultQueue;
//
// if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) {
// continue;
// }
//
// const group = ftrConfigsByQueue.get(queue);
// if (group) {
// group.push(path);
// } else {
// ftrConfigsByQueue.set(queue, [path]);
// }
// }
// return { defaultQueue, ftrConfigsByQueue };
// } catch (_) {
// const error = _ instanceof Error ? _ : new Error(`${_} thrown`);
// throw new Error(`unable to collect enabled FTR configs: ${error.message}`);
// }
// }

export async function pickTestGroupRunOrder() {
const bk = new BuildkiteClient();
Expand Down Expand Up @@ -291,13 +287,13 @@ export async function pickTestGroupRunOrder() {
})
: [];

const jestIntegrationConfigs = LIMIT_CONFIG_TYPE.includes('integration')
? globby.sync(['**/jest.integration.config.js', '!**/__fixtures__/**'], {
cwd: process.cwd(),
absolute: false,
ignore: DISABLED_JEST_CONFIGS,
})
: [];
// const jestIntegrationConfigs = LIMIT_CONFIG_TYPE.includes('integration')
// ? globby.sync(['**/jest.integration.config.js', '!**/__fixtures__/**'], {
// cwd: process.cwd(),
// absolute: false,
// ignore: DISABLED_JEST_CONFIGS,
// })
// : [];

// if (!ftrConfigsByQueue.size && !jestUnitConfigs.length && !jestIntegrationConfigs.length) {
// throw new Error('unable to find any unit, integration, or FTR configs');
Expand Down