Add experiment for px-k8ssandra to perf tool (pixie-io#1718)

Summary: Adds perf experiment for `px-k8ssandra` to new
`k8ssandraExperimentSuite`

The perf tool does not currently install the
[cert-manager](https://cert-manager.io/docs/installation/helm/)
automatically, which is required for px-k8ssandra, so we have to spin up
a cluster, install the cert-manager and run the perf tool with
`--use-local-cluster`. For this reason, this PR creates a new
`k8ssandraExperimentSuite` to avoid polluting nightly.

```sh
bazel run //src/e2e_test/perf_tool -- run --suite=k8ssandra --experiment_name=px-k8ssandra --commit_sha=$(git rev-parse HEAD) --api_key=$API_KEY --cloud_addr=$PL_CLOUD_ADDR --use_local_cluster
```

Another issue is that the perf tool currently does not properly clean up
the demo; instead of running `px demo delete`, it just tries to delete
the namespace. Since the k8ssandra demo uses CRDs, this could lead to
orphaned resources or lingering finalizers that might complicate
subsequent tests or deployments. A workaround is to comment out the
lines in workload.go that delete the namespace and manually run `px demo
delete`. Note that this is not an issue if perf tool deletes the entire
cluster after a perf run (which it usually does).

Type of change: /kind feature

Test Plan: Ran the perf tool and verified that data showed up for
`px-k8ssandra`

---------

Signed-off-by: Benjamin Kilimnik <bkilimnik@pixielabs.ai>

src/e2e_test/perf_tool/pkg/deploy/workload.go

@@ -145,6 +145,8 @@ func (w *workloadImpl) WaitForHealthCheck(ctx context.Context, clusterCtx *clust
 
 // Close stops the workload.
 func (w *workloadImpl) Close() error {
+	// TODO(@benkilimnik): Run demo delete for demos that use CRDs (e.g. px-k8ssandra)
+	// current approach can lead to orphaned resources or lingering finalizers that might complicate subsequent tests or deployments.
 	for ns := range w.namespacesToDelete {
 		log.WithField("workload", w.name).WithField("namespace", ns).Trace("deleting workload namespace")
 		od := k8s.ObjectDeleter{

src/e2e_test/perf_tool/pkg/suites/experiments.go

@@ -82,6 +82,56 @@ func HTTPLoadTestExperiment(
 	return e
 }
 
+// Perf test for px-k8ssandra
+func K8ssandraExperiment(
+	metricPeriod time.Duration,
+	predeployDur time.Duration,
+	dur time.Duration,
+) *pb.ExperimentSpec {
+	e := &pb.ExperimentSpec{
+		VizierSpec: VizierWorkload(),
+		WorkloadSpecs: []*pb.WorkloadSpec{
+			K8ssandraWorkload(),
+		},
+		MetricSpecs: []*pb.MetricSpec{
+			ProcessStatsMetrics(metricPeriod),
+			// Stagger the second query a little bit because of query stability issues.
+			HeapMetrics(metricPeriod + (2 * time.Second)),
+		},
+		RunSpec: &pb.RunSpec{
+			Actions: []*experimentpb.ActionSpec{
+				{
+					Type: experimentpb.START_VIZIER,
+				},
+				{
+					Type: experimentpb.START_METRIC_RECORDERS,
+				},
+				{
+					Type:     experimentpb.BURNIN,
+					Duration: types.DurationProto(predeployDur),
+				},
+				{
+					Type: experimentpb.START_WORKLOADS,
+				},
+				{
+					Type:     experimentpb.RUN,
+					Duration: types.DurationProto(dur),
+				},
+				{
+					// Make sure metric recorders are stopped before vizier/workloads.
+					Type: experimentpb.STOP_METRIC_RECORDERS,
+				},
+			},
+		},
+		ClusterSpec: DefaultCluster,
+	}
+	e = addTags(e,
+		"workload/k8ssandra",
+		"parameter/default/",
+	)
+	return e
+}
+
 // SockShopExperiment is an experiment that runs all of sock shop as a workload.
 func SockShopExperiment(
 	metricPeriod time.Duration,

src/e2e_test/perf_tool/pkg/suites/suites.go

@@ -32,6 +32,7 @@ type ExperimentSuite func() map[string]*pb.ExperimentSpec
 var ExperimentSuiteRegistry = map[string]ExperimentSuite{
 	"nightly":   nightlyExperimentSuite,
 	"http-grid": httpGridSuite,
+	"k8ssandra": k8ssandraExperimentSuite,
 }
 
 func nightlyExperimentSuite() map[string]*pb.ExperimentSpec {
@@ -53,6 +54,25 @@ func nightlyExperimentSuite() map[string]*pb.ExperimentSpec {
 	return exps
 }
 
+// Added separate experiment suite for k8ssandra because the perf tool does not currently install the cert-manager
+// automatically, which is required for px-k8ssandra.
+// To run this experiment, we have to spin up a cluster, install the cert-manager, and
+// run the perf tool with --use-local-cluster.
+// Tags are added to properly display results in the perf dashboard.
+// TODO(@benkilimnik): move to nightly once cert-manager is installed automatically or perf tool workflow changes.
+func k8ssandraExperimentSuite() map[string]*pb.ExperimentSpec {
+	defaultMetricPeriod := 30 * time.Second
+	preDur := 5 * time.Minute
+	dur := 40 * time.Minute
+	exps := map[string]*pb.ExperimentSpec{
+		"px-k8ssandra": K8ssandraExperiment(defaultMetricPeriod, preDur, dur),
+	}
+	for _, e := range exps {
+		addTags(e, "suite/k8ssandra")
+	}
+	return exps
+}
+
 func httpGridSuite() map[string]*pb.ExperimentSpec {
 	defaultMetricPeriod := 30 * time.Second
 	preDur := 5 * time.Minute

src/e2e_test/perf_tool/pkg/suites/workloads.go

@@ -141,6 +141,28 @@ func SockShopWorkload() *pb.WorkloadSpec {
 		Healthchecks: HTTPHealthChecks("px-sock-shop", true),
 	}
 }
+func K8ssandraWorkload() *pb.WorkloadSpec {
+	return &pb.WorkloadSpec{
+		Name: "px-python-demo",
+		DeploySteps: []*pb.DeployStep{
+			{
+				DeployType: &pb.DeployStep_Px{
+					Px: &pb.PxCLIDeploy{
+						Args: []string{
+							"demo",
+							"deploy",
+							"px-k8ssandra",
+						},
+						Namespaces: []string{
+							"px-k8ssandra",
+						},
+					},
+				},
+			},
+		},
+		Healthchecks: HTTPHealthChecks("px-k8ssandra", true),
+	}
+}
 
 // OnlineBoutiqueWorkload returns the WorkloadSpec to deploy online boutique.
 func OnlineBoutiqueWorkload() *pb.WorkloadSpec {