FIPS FTR Overrides and test skips (elastic#192053)

## Summary Kibana requires security to be enabled and a platinum or better license to run in FIPS mode. Since not all FTR configs assume these conditions will be enabled, we cant run every test. So these failing tests will be skipped when these overrides are enforced. This does not mean that the functionality is not supported in FIPS mode. ## What is the point? Running these tests in FIPS mode is not necessarily to check that the functionality works as expected, it is to make sure Kibana does not crash due to unsupported algorithm usage (`md4`, `md5`, etc). When running in FIPS mode, Node will throw an `unsupported envelope function` error (with FIPS enabled) if it encounters an unsupported algorithm, so the more lines of code covered, the more assurance we can have that features will work in FIPS mode. ## Nature of the changes To skip a test, a `tag` is added: `this.tags('skipFIPS')` `this.tags` is only available for `describe('description', function() {...});` There should not be any logical changes, just tests wrapped in an extra block. I tried to make the wording in the new `describe` block "flow" 😅 if you prefer different wording in the new `describe` block - please add a change! --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Nikita Indik <nikita.indik@elastic.co>
kibanamachine · Sep 10, 2024 · 5ec0cb0 · 5ec0cb0
1 parent 2d40fa3
commit 5ec0cb0
Show file tree

Hide file tree

Showing 103 changed files with 1,919 additions and 1,595 deletions.
diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh
@@ -135,6 +135,8 @@ export GH_REPO=github.com/elastic/kibana
 FTR_ENABLE_FIPS_AGENT=false
 if [[ "${KBN_ENABLE_FIPS:-}" == "true" ]] || is_pr_with_label "ci:enable-fips-agent"; then
   FTR_ENABLE_FIPS_AGENT=true
+  ES_SECURITY_ENABLED=true
+  export ES_SECURITY_ENABLED
   # used by FIPS agents to link FIPS OpenSSL modules
   export OPENSSL_MODULES=$HOME/openssl/lib/ossl-modules
 

diff --git a/packages/kbn-ftr-common-functional-ui-services/services/security/test_user.ts b/packages/kbn-ftr-common-functional-ui-services/services/security/test_user.ts
@@ -54,6 +54,18 @@ export class TestUser extends FtrService {
       return;
     }
 
+    /*
+     * When running in FIPS mode, security must be enabled. Many suites expect that there will be no authc/authz.
+     * Test user is configured to get `defaultRole` which is being overridden in `fips_overrides.ts` to the most privileged
+     * roles available so that more tests can be run successfully
+     */
+    if (process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() === 'true') {
+      this.log.debug(
+        `FTR is running in FIPS mode and does not allow for Test User's roles to be overridden`
+      );
+      return;
+    }
+
     this.log.debug(`set roles = ${roles}`);
     await this.user.create(TEST_USER_NAME, {
       password: TEST_USER_PASSWORD,
@@ -79,7 +91,6 @@ export class TestUser extends FtrService {
 export async function createTestUserService(ctx: FtrProviderContext, role: Role, user: User) {
   const log = ctx.getService('log');
   const config = ctx.getService('config');
-
   const enabled =
     !config
       .get('esTestCluster.serverArgs')

diff --git a/packages/kbn-test/src/functional_tests/lib/fips_overrides.ts b/packages/kbn-test/src/functional_tests/lib/fips_overrides.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+// This will only apply overrides when running in FIPS mode
+export function applyFipsOverrides(vars: any) {
+  vars.esTestCluster.license = 'trial';
+
+  const skipTags = vars.suiteTags?.exclude ?? [];
+  skipTags.push('skipFIPS');
+  vars.suiteTags = {
+    ...vars.suiteTags,
+    exclude: skipTags,
+  };
+
+  vars.security = {
+    ...vars.security,
+    /*
+     * When running in FIPS mode, security must be enabled. Many suites expect that there will be no authc/authz.
+     * Test user's roles are set to `defaultRoles`, the most privileged roles are added here
+     *  so that more tests can be run successfully
+     */
+    defaultRoles: ['superuser', 'kibana_admin', 'system_indices_superuser'],
+  };
+
+  const newServerArgs = vars.esTestCluster.serverArgs.filter(
+    (arg: string) => arg !== 'xpack.security.enabled=false'
+  );
+  newServerArgs.push('xpack.security.enabled=true');
+
+  const selfTypedBasicLicenseIndex = newServerArgs.indexOf(
+    `xpack.license.self_generated.type=basic`
+  );
+  if (selfTypedBasicLicenseIndex > -1) {
+    newServerArgs[selfTypedBasicLicenseIndex] = `xpack.license.self_generated.type=trial`;
+  }
+
+  vars.esTestCluster.serverArgs = newServerArgs;
+
+  return vars;
+}
diff --git a/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts b/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts
@@ -14,7 +14,8 @@ import { REPO_ROOT } from '@kbn/repo-info';
 import { ToolingLog } from '@kbn/tooling-log';
 import { withProcRunner } from '@kbn/dev-proc-runner';
 
-import { readConfigFile } from '../../functional_test_runner';
+import { applyFipsOverrides } from '../lib/fips_overrides';
+import { Config, readConfigFile } from '../../functional_test_runner';
 
 import { checkForEnabledTestsInFtrConfig, runFtr } from '../lib/run_ftr';
 import { runElasticsearch } from '../lib/run_elasticsearch';
@@ -68,7 +69,18 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
         log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, path)}`);
       }
 
-      const config = await readConfigFile(log, options.esVersion, path, settingOverrides);
+      let config: Config;
+      if (process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() !== 'true') {
+        config = await readConfigFile(log, options.esVersion, path, settingOverrides);
+      } else {
+        config = await readConfigFile(
+          log,
+          options.esVersion,
+          path,
+          settingOverrides,
+          applyFipsOverrides
+        );
+      }
 
       const hasTests = await checkForEnabledTestsInFtrConfig({
         config,

diff --git a/packages/kbn-test/src/functional_tests/start_servers/start_servers.ts b/packages/kbn-test/src/functional_tests/start_servers/start_servers.ts
@@ -16,7 +16,8 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { withProcRunner } from '@kbn/dev-proc-runner';
 import { getTimeReporter } from '@kbn/ci-stats-reporter';
 
-import { readConfigFile } from '../../functional_test_runner';
+import { applyFipsOverrides } from '../lib/fips_overrides';
+import { Config, readConfigFile } from '../../functional_test_runner';
 import { runElasticsearch } from '../lib/run_elasticsearch';
 import { runKibanaServer } from '../lib/run_kibana_server';
 import { StartServerOptions } from './flags';
@@ -28,7 +29,12 @@ export async function startServers(log: ToolingLog, options: StartServerOptions)
   const reportTime = getTimeReporter(log, 'scripts/functional_tests_server');
 
   await withProcRunner(log, async (procs) => {
-    const config = await readConfigFile(log, options.esVersion, options.config);
+    let config: Config;
+    if (process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() !== 'true') {
+      config = await readConfigFile(log, options.esVersion, options.config);
+    } else {
+      config = await readConfigFile(log, options.esVersion, options.config, {}, applyFipsOverrides);
+    }
 
     const shutdownEs = await runElasticsearch({
       config,

diff --git a/test/functional/apps/dashboard/group4/dashboard_listing.ts b/test/functional/apps/dashboard/group4/dashboard_listing.ts
@@ -237,7 +237,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       });
     });
 
-    describe('insights', () => {
+    describe('insights', function () {
+      this.tags('skipFIPS');
       const DASHBOARD_NAME = 'Insights Dashboard';
 
       before(async () => {

diff --git a/test/functional/apps/discover/group6/_view_mode_toggle.ts b/test/functional/apps/discover/group6/_view_mode_toggle.ts
@@ -88,11 +88,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           await testSubjects.missingOrFail('discoverErrorCalloutTitle');
         });
 
-        it('should not show Patterns tab (basic license)', async () => {
-          await testSubjects.missingOrFail('dscViewModePatternAnalysisButton');
-          await retry.try(async () => {
-            const documentTab = await testSubjects.find('dscViewModeDocumentButton');
-            expect(await documentTab.getAttribute('aria-selected')).to.be('true');
+        describe('Patterns tab (basic license)', function () {
+          this.tags('skipFIPS');
+
+          it('should not show', async function () {
+            await testSubjects.missingOrFail('dscViewModePatternAnalysisButton');
+            await retry.try(async () => {
+              const documentTab = await testSubjects.find('dscViewModeDocumentButton');
+              expect(await documentTab.getAttribute('aria-selected')).to.be('true');
+            });
           });
         });
 

diff --git a/test/functional/apps/discover/group8/_default_route.ts b/test/functional/apps/discover/group8/_default_route.ts
@@ -22,6 +22,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const queryBar = getService('queryBar');
 
   describe('discover as defaultRoute', function () {
+    this.tags('skipFIPS');
     before(async function () {
       await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
       await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');

diff --git a/test/functional/apps/kibana_overview/_analytics.ts b/test/functional/apps/kibana_overview/_analytics.ts
@@ -36,19 +36,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
     const apps = ['dashboards', 'discover', 'canvas', 'maps', 'ml'];
 
-    it('should display Analytics apps cards', async () => {
-      const kbnOverviewAppsCards = await find.allByCssSelector('.kbnOverviewApps__item');
-      expect(kbnOverviewAppsCards.length).to.be(apps.length);
+    describe('Analytics apps cards', function () {
+      this.tags('skipFIPS');
+      it('should display ', async () => {
+        const kbnOverviewAppsCards = await find.allByCssSelector('.kbnOverviewApps__item');
+        expect(kbnOverviewAppsCards.length).to.be(apps.length);
 
-      const verifyImageUrl = async (el: WebElementWrapper, imgName: string) => {
-        const image = await el.findByCssSelector('img');
-        const imageUrl = (await image.getAttribute('src')) ?? '';
-        expect(imageUrl.includes(imgName)).to.be(true);
-      };
+        const verifyImageUrl = async (el: WebElementWrapper, imgName: string) => {
+          const image = await el.findByCssSelector('img');
+          const imageUrl = (await image.getAttribute('src')) ?? '';
+          expect(imageUrl.includes(imgName)).to.be(true);
+        };
 
-      for (let i = 0; i < apps.length; i++) {
-        await verifyImageUrl(kbnOverviewAppsCards[i], `kibana_${apps[i]}_light.svg`);
-      }
+        for (let i = 0; i < apps.length; i++) {
+          await verifyImageUrl(kbnOverviewAppsCards[i], `kibana_${apps[i]}_light.svg`);
+        }
+      });
     });
 
     it('click on a card should lead to the appropriate app', async () => {

diff --git a/test/interactive_setup_api_integration/tests/enrollment_flow.ts b/test/interactive_setup_api_integration/tests/enrollment_flow.ts
@@ -21,7 +21,7 @@ export default function (context: FtrProviderContext) {
   const config = context.getService('config');
 
   describe('Interactive setup APIs - Enrollment flow', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     let kibanaVerificationCode: string;
     let elasticsearchCaFingerprint: string;

diff --git a/test/interactive_setup_api_integration/tests/manual_configuration_flow.ts b/test/interactive_setup_api_integration/tests/manual_configuration_flow.ts
@@ -20,7 +20,7 @@ export default function (context: FtrProviderContext) {
   const config = context.getService('config');
 
   describe('Interactive setup APIs - Manual configuration flow', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     let kibanaVerificationCode: string;
     let elasticsearchCaCertificate: string;

diff --git a/test/interactive_setup_api_integration/tests/manual_configuration_flow_without_tls.ts b/test/interactive_setup_api_integration/tests/manual_configuration_flow_without_tls.ts
@@ -19,7 +19,7 @@ export default function (context: FtrProviderContext) {
   const config = context.getService('config');
 
   describe('Interactive setup APIs - Manual configuration flow without TLS', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     let kibanaVerificationCode: string;
     before(async () => {

diff --git a/test/interactive_setup_functional/tests/enrollment_token.ts b/test/interactive_setup_functional/tests/enrollment_token.ts
@@ -23,7 +23,7 @@ export default function ({ getService }: FtrProviderContext) {
   const log = getService('log');
 
   describe('Interactive Setup Functional Tests (Enrollment token)', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     const elasticsearchConfig = config.get('servers.elasticsearch');
     let verificationCode: string;

diff --git a/test/interactive_setup_functional/tests/manual_configuration.ts b/test/interactive_setup_functional/tests/manual_configuration.ts
@@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) {
   const log = getService('log');
 
   describe('Interactive Setup Functional Tests (Manual configuration)', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     let verificationCode: string;
     before(async function () {

diff --git a/test/interactive_setup_functional/tests/manual_configuration_without_security.ts b/test/interactive_setup_functional/tests/manual_configuration_without_security.ts
@@ -20,7 +20,7 @@ export default function ({ getService, getPageObject }: FtrProviderContext) {
   const log = getService('log');
 
   describe('Interactive Setup Functional Tests (Manual configuration without Security)', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     let verificationCode: string;
     before(async function () {

diff --git a/test/interactive_setup_functional/tests/manual_configuration_without_tls.ts b/test/interactive_setup_functional/tests/manual_configuration_without_tls.ts
@@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) {
   const log = getService('log');
 
   describe('Interactive Setup Functional Tests (Manual configuration without TLS)', function () {
-    this.tags('skipCloud');
+    this.tags(['skipCloud', 'skipFIPS']);
 
     let verificationCode: string;
     before(async function () {

diff --git a/test/server_integration/http/platform/status.ts b/test/server_integration/http/platform/status.ts
@@ -32,7 +32,8 @@ export default function ({ getService }: FtrProviderContext) {
       .set('kbn-xsrf', 'xxx')
       .expect(200);
 
-  describe('status service', () => {
+  describe('status service', function () {
+    this.tags('skipFIPS');
     // This test must come first because the timeout only applies to the initial emission
     it("returns a timeout for status check that doesn't emit after 30s", async () => {
       let aStatus = await getStatus('statusPluginA');

diff --git a/x-pack/test/alerting_api_integration/basic/tests/index.ts b/x-pack/test/alerting_api_integration/basic/tests/index.ts
@@ -13,6 +13,7 @@ export default function alertingApiIntegrationTests({
   getService,
 }: FtrProviderContext) {
   describe('alerting api integration basic license', function () {
+    this.tags('skipFIPS');
     loadTestFile(require.resolve('./actions'));
     loadTestFile(require.resolve('./alerts'));
   });

diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/alerts_base.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/alerts_base.ts
@@ -38,7 +38,8 @@ export function alertTests({ getService }: FtrProviderContext, space: Space) {
       .then((response: SupertestResponse) => response.body);
   }
 
-  describe('alerts', () => {
+  describe('alerts', function () {
+    this.tags('skipFIPS');
     let alertUtils: AlertUtils;
     let indexRecordActionId: string;
     const authorizationIndex = '.kibana-test-authorization';