From 4a59b49917f25f9933965cff90ac064c6127bfb2 Mon Sep 17 00:00:00 2001 From: Katerina Date: Wed, 4 Sep 2024 13:07:42 +0300 Subject: [PATCH 1/2] [APM][Refactor] Rename create_assets_es_client to create_entities_es_client (#192061) ## Summary Refactoring: Rename assets to entities --- .../create_entities_es_client.ts} | 0 .../routes/custom_dashboards/get_entities_with_dashboards.ts | 2 +- .../apm/server/routes/custom_dashboards/route.ts | 2 +- .../apm/server/routes/entities/get_entities.ts | 2 +- .../routes/entities/get_service_entities_history_metrics.ts | 2 +- .../routes/entities/get_service_entities_history_timeseries.ts | 2 +- .../apm/server/routes/entities/get_service_entity_summary.ts | 2 +- .../apm/server/routes/entities/get_service_latest_entity.ts | 2 +- .../apm/server/routes/entities/services/get_service_entities.ts | 2 +- .../apm/server/routes/entities/services/routes.ts | 2 +- .../routes/historical_data/has_historical_entities_data.ts | 2 +- .../apm/server/routes/historical_data/route.ts | 2 +- .../observability_solution/apm/server/routes/services/route.ts | 2 +- 13 files changed, 12 insertions(+), 12 deletions(-) rename x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/{create_assets_es_client/create_assets_es_clients.ts => create_entities_es_client/create_entities_es_client.ts} (100%) diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client.ts similarity index 100% rename from x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts rename to x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client.ts diff --git a/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/get_entities_with_dashboards.ts b/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/get_entities_with_dashboards.ts index 0d5b443ae6d458..df1f785216367b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/get_entities_with_dashboards.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/get_entities_with_dashboards.ts @@ -9,7 +9,7 @@ import { kqlQuery, termQuery } from '@kbn/observability-plugin/server'; import { estypes } from '@elastic/elasticsearch'; import { SERVICE_NAME } from '../../../common/es_fields/apm'; import { SavedApmCustomDashboard } from '../../../common/custom_dashboards'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; function getSearchRequest(filters: estypes.QueryDslQueryContainer[]) { return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/route.ts index 4fcc1508715d16..750bacb4ba4558 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/custom_dashboards/route.ts @@ -14,7 +14,7 @@ import { getCustomDashboards } from './get_custom_dashboards'; import { getServicesWithDashboards } from './get_services_with_dashboards'; import { getApmEventClient } from '../../lib/helpers/get_apm_event_client'; import { rangeRt } from '../default_api_types'; -import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { getEntitiesWithDashboards } from './get_entities_with_dashboards'; const serviceDashboardSaveRoute = createApmServerRoute({ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts index a69a906454ff74..e3a40b698eb130 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts @@ -14,7 +14,7 @@ import { } from '../../../common/es_fields/apm'; import { FIRST_SEEN, LAST_SEEN, ENTITY, ENTITY_TYPE } from '../../../common/es_fields/entities'; import { environmentQuery } from '../../../common/utils/environment_query'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { getServiceEntitiesHistoryMetrics } from './get_service_entities_history_metrics'; import { EntitiesRaw, EntityType, ServiceEntities } from './types'; import { isFiniteNumber } from '../../../common/utils/is_finite_number'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_metrics.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_metrics.ts index 606c1748aa0a0c..5355cfd8b563c2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_metrics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_metrics.ts @@ -16,7 +16,7 @@ import { ENTITY_METRICS_THROUGHPUT, LAST_SEEN, } from '../../../common/es_fields/entities'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; interface Params { entitiesESClient: EntitiesESClient; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_timeseries.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_timeseries.ts index 6e969b6022bf21..73811a544701b7 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_timeseries.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_timeseries.ts @@ -17,7 +17,7 @@ import { LAST_SEEN, } from '../../../common/es_fields/entities'; import { SERVICE_NAME } from '../../../common/es_fields/apm'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { environmentQuery } from '../../../common/utils/environment_query'; interface Params { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entity_summary.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entity_summary.ts index 37e3ddef9ecc0d..84886272ba54a2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entity_summary.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entity_summary.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { withApmSpan } from '../../utils/with_apm_span'; import { getServiceLatestEntity } from './get_service_latest_entity'; import { ServiceEntities } from './types'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_latest_entity.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_latest_entity.ts index f4a4e4d1a81a45..71cead9a173019 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_latest_entity.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_latest_entity.ts @@ -14,7 +14,7 @@ import { import { ENTITY, ENTITY_TYPE } from '../../../common/es_fields/entities'; import { environmentQuery } from '../../../common/utils/environment_query'; import { isFiniteNumber } from '../../../common/utils/is_finite_number'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { entitiesRangeQuery } from './get_entities'; import { EntitiesRaw, EntityType, ServiceEntities } from './types'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts index 0f90afb214cbfe..084fbbe4389522 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts @@ -7,7 +7,7 @@ import { errors } from '@elastic/elasticsearch'; import { Logger } from '@kbn/core/server'; import { WrappedElasticsearchClientError } from '@kbn/observability-plugin/server'; -import { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { withApmSpan } from '../../../utils/with_apm_span'; import { getEntities } from '../get_entities'; import { calculateAvgMetrics } from '../utils/calculate_avg_metrics'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts index 027bd5bd09c982..fad2e495e8c6e2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts @@ -9,7 +9,7 @@ import { jsonRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { EntityServiceListItem } from '../../../../common/entities/types'; import { environmentQuery } from '../../../../common/utils/environment_query'; -import { createEntitiesESClient } from '../../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { createEntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { createApmServerRoute } from '../../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../../default_api_types'; import { getServiceEntities } from './get_service_entities'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_entities_data.ts b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_entities_data.ts index 13c6ebcf7bfd97..f245c27a99036d 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_entities_data.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_entities_data.ts @@ -7,7 +7,7 @@ import { WrappedElasticsearchClientError } from '@kbn/observability-plugin/server'; import { Logger } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; export async function hasEntitiesData(entitiesESClient: EntitiesESClient, logger: Logger) { const params = { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/route.ts index 484590a00a36e8..94b7d40b84c2dc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/route.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { getApmEventClient } from '../../lib/helpers/get_apm_event_client'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { hasHistoricalAgentData } from './has_historical_agent_data'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts index f80c58c19ed93d..1d4cbd0cef1c3d 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts @@ -80,7 +80,7 @@ import { import { getThroughput, ServiceThroughputResponse } from './get_throughput'; import { getServiceEntitySummary } from '../entities/get_service_entity_summary'; import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; -import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients'; +import { createEntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; const servicesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/services', From 00230cfe0c23ccbdbbe217f549f64a45f24429a6 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 4 Sep 2024 12:12:27 +0200 Subject: [PATCH 2/2] [CI] use kibana_install_dir from built artifacts (#191909) ## Summary I noticed recently, that the step "Copying ES Snapshot cache" got cancelled due to a lost agent. On closer inspection, it leads to the changed file (`.buildkite/scripts/steps/functional/osquery_cypress.sh`) where with a note, there's a kibana rebuild. Unless there's something special in that rebuild, we should be using the artifacts built in the build step. So let's follow the TODO notes, and use the downloaded artifacts - this should save ~6m per step where the osquery tests were warmed up. ## Before From [on-merge#49584](https://buildkite.com/elastic/kibana-on-merge/builds/49584), running the pre-PR version: - You can see, after the copying of es snapshot cache, the command to build kibana's plugins is ran, taking 6+ minutes Screenshot 2024-09-02 at 16 25 43 ## After This is on the [PR build](https://buildkite.com/elastic/kibana-pull-request/builds/231455#0191b2cc-f8be-410d-860d-59ab9f984477): - You can see, the es snapshot cache update is only 6 sec, after that, we can get going with the testing Screenshot 2024-09-02 at 16 28 10 ## Results On a primitive comparison between the PR build (https://buildkite.com/elastic/kibana-pull-request/builds/231505) and an on-merge run (https://buildkite.com/elastic/kibana-on-merge/builds/49598) we've **saved on average 9 minutes per runs** of `Osquery Cypress Tests`. We can only say more adequate savings once we compare on-merge with on-merge, but it can be estimated this would be **16 x 9 minutes = 144 minutes** (16 = 8 osquery cypress + 8 osquery serverless cypress) per on-merges. --- .buildkite/scripts/steps/functional/osquery_cypress.sh | 6 +----- .buildkite/scripts/steps/functional/osquery_cypress_burn.sh | 5 ++--- .../scripts/steps/functional/security_serverless_osquery.sh | 6 +----- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.buildkite/scripts/steps/functional/osquery_cypress.sh b/.buildkite/scripts/steps/functional/osquery_cypress.sh index 9ba7cabc4d1cb5..0ae47edd8c8e7f 100755 --- a/.buildkite/scripts/steps/functional/osquery_cypress.sh +++ b/.buildkite/scripts/steps/functional/osquery_cypress.sh @@ -4,11 +4,7 @@ set -euo pipefail source .buildkite/scripts/steps/functional/common.sh -# TODO: remove the line below to use build artifacts for tests. -# in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location -# by exporting a var like: -# export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} -node scripts/build_kibana_platform_plugins.js +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} export JOB=kibana-osquery-cypress diff --git a/.buildkite/scripts/steps/functional/osquery_cypress_burn.sh b/.buildkite/scripts/steps/functional/osquery_cypress_burn.sh index 8068dccac64f32..bebd391d952546 100755 --- a/.buildkite/scripts/steps/functional/osquery_cypress_burn.sh +++ b/.buildkite/scripts/steps/functional/osquery_cypress_burn.sh @@ -2,10 +2,9 @@ set -euo pipefail -source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/functional/common.sh -.buildkite/scripts/bootstrap.sh -node scripts/build_kibana_platform_plugins.js +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} export JOB=kibana-osquery-cypress diff --git a/.buildkite/scripts/steps/functional/security_serverless_osquery.sh b/.buildkite/scripts/steps/functional/security_serverless_osquery.sh index 27ed0c79871ee3..9844222e7f8698 100755 --- a/.buildkite/scripts/steps/functional/security_serverless_osquery.sh +++ b/.buildkite/scripts/steps/functional/security_serverless_osquery.sh @@ -4,11 +4,7 @@ set -euo pipefail source .buildkite/scripts/steps/functional/common.sh -# TODO: remove the line below to use build artifacts for tests. -# in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location -# by exporting a var like: -# export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} -node scripts/build_kibana_platform_plugins.js +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} export JOB=kibana-osquery-cypress-serverless