Skip to content

Commit

Permalink
add support for defining TTL for image credentials cache (#5660)
Browse files Browse the repository at this point in the history
  • Loading branch information
dejanzele committed Jul 17, 2024
1 parent bb5f7bd commit 4e3477f
Show file tree
Hide file tree
Showing 13 changed files with 502 additions and 79 deletions.
7 changes: 6 additions & 1 deletion cmd/api-server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ import (
"syscall"
"time"

corev1 "k8s.io/api/core/v1"

"github.com/kubeshop/testkube/pkg/cache"

"github.com/nats-io/nats.go"
"google.golang.org/grpc/metadata"
"google.golang.org/protobuf/types/known/emptypb"
Expand Down Expand Up @@ -487,7 +491,7 @@ func main() {
inspector := imageinspector.NewInspector(
cfg.TestkubeRegistry,
imageinspector.NewSkopeoFetcher(),
imageinspector.NewSecretFetcher(secretClient),
imageinspector.NewSecretFetcher(secretClient, cache.NewInMemoryCache[*corev1.Secret](), imageinspector.WithSecretCacheTTL(cfg.TestkubeImageCredentialsCacheTTL)),
inspectorStorages...,
)

Expand Down Expand Up @@ -515,6 +519,7 @@ func main() {
features,
cfg.TestkubeDefaultStorageClassName,
cfg.WhitelistedContainers,
cfg.TestkubeImageCredentialsCacheTTL,
)
if err != nil {
exitOnError("Creating container executor", err)
Expand Down
1 change: 1 addition & 0 deletions cmd/tcl/testworkflow-toolkit/spawn/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ func CreateBaseMachine() expressions.Machine {
"images.toolkit": env.Config().Images.Toolkit,
"images.persistence.enabled": strconv.FormatBool(env.Config().Images.InspectorPersistenceEnabled),
"images.persistence.key": env.Config().Images.InspectorPersistenceCacheKey,
"images.cache.ttl": env.Config().Images.ImageCredentialsCacheTTL.String(),
}),
)
}
Expand Down
6 changes: 5 additions & 1 deletion cmd/testworkflow-toolkit/env/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ import (
"fmt"
"math"

corev1 "k8s.io/api/core/v1"

"github.com/kubeshop/testkube/pkg/cache"

"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

Expand Down Expand Up @@ -58,7 +62,7 @@ func ImageInspector() imageinspector.Inspector {
return imageinspector.NewInspector(
Config().System.DefaultRegistry,
imageinspector.NewSkopeoFetcher(),
imageinspector.NewSecretFetcher(secretClient),
imageinspector.NewSecretFetcher(secretClient, cache.NewInMemoryCache[*corev1.Secret](), imageinspector.WithSecretCacheTTL(Config().Images.ImageCredentialsCacheTTL)),
inspectorStorages...,
)
}
Expand Down
9 changes: 5 additions & 4 deletions cmd/testworkflow-toolkit/env/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,11 @@ type envSystemConfig struct {
}

type envImagesConfig struct {
Init string `envconfig:"TESTKUBE_TW_INIT_IMAGE"`
Toolkit string `envconfig:"TESTKUBE_TW_TOOLKIT_IMAGE"`
InspectorPersistenceEnabled bool `envconfig:"TK_IMG_P" default:"false"`
InspectorPersistenceCacheKey string `envconfig:"TK_IMG_PK"`
Init string `envconfig:"TESTKUBE_TW_INIT_IMAGE"`
Toolkit string `envconfig:"TESTKUBE_TW_TOOLKIT_IMAGE"`
InspectorPersistenceEnabled bool `envconfig:"TK_IMG_P" default:"false"`
InspectorPersistenceCacheKey string `envconfig:"TK_IMG_PK"`
ImageCredentialsCacheTTL time.Duration `envconfig:"TK_IMG_CRED_TTL" default:"0"`
}

type featuresConfig struct {
Expand Down
59 changes: 31 additions & 28 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,36 +82,39 @@ type Config struct {
TestkubeProTLSSecret string `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""`
TestkubeProRunnerCustomCASecret string `envconfig:"TESTKUBE_PRO_RUNNER_CUSTOM_CA_SECRET" default:""`
TestkubeWatcherNamespaces string `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""`
GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
TestkubeRegistry string `envconfig:"TESTKUBE_REGISTRY" default:""`
TestkubePodStartTimeout time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"`
CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""`
TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"`
TestkubeClusterName string `envconfig:"TESTKUBE_CLUSTER_NAME" default:""`
CompressArtifacts bool `envconfig:"COMPRESSARTIFACTS" default:"false"`
TestkubeHelmchartVersion string `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""`
DebugListenAddr string `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"`
EnableDebugServer bool `envconfig:"ENABLE_DEBUG_SERVER" default:"false"`
EnableSecretsEndpoint bool `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"`
EnableListingAllSecrets bool `envconfig:"ENABLE_LISTING_ALL_SECRETS" default:"false"`
SecretCreationPrefix string `envconfig:"SECRET_CREATION_PREFIX" default:"testkube-"`
DisableMongoMigrations bool `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"`
Debug bool `envconfig:"DEBUG" default:"false"`
Trace bool `envconfig:"TRACE" default:"false"`
EnableImageDataPersistentCache bool `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"`
ImageDataPersistentCacheKey string `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"`
LogServerGrpcAddress string `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"`
LogServerSecure bool `envconfig:"LOG_SERVER_SECURE" default:"false"`
LogServerSkipVerify bool `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"`
LogServerCertFile string `envconfig:"LOG_SERVER_CERT_FILE" default:""`
LogServerKeyFile string `envconfig:"LOG_SERVER_KEY_FILE" default:""`
LogServerCAFile string `envconfig:"LOG_SERVER_CA_FILE" default:""`
DisableSecretCreation bool `envconfig:"DISABLE_SECRET_CREATION" default:"false"`
TestkubeExecutionNamespaces string `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""`
TestkubeDefaultStorageClassName string `envconfig:"TESTKUBE_DEFAULT_STORAGE_CLASS_NAME" default:""`
GlobalWorkflowTemplateName string `envconfig:"TESTKUBE_GLOBAL_WORKFLOW_TEMPLATE_NAME" default:""`
EnableK8sEvents bool `envconfig:"ENABLE_K8S_EVENTS" default:"true"`
// TestkubeImageCredentialsCacheTTL is the duration for which the image pull credentials should be cached provided as a Go duration string.
// If set to 0, the cache is disabled.
TestkubeImageCredentialsCacheTTL time.Duration `envconfig:"TESTKUBE_IMAGE_CREDENTIALS_CACHE_TTL" default:"30m"`
GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""`
TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"`
TestkubeClusterName string `envconfig:"TESTKUBE_CLUSTER_NAME" default:""`
CompressArtifacts bool `envconfig:"COMPRESSARTIFACTS" default:"false"`
TestkubeHelmchartVersion string `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""`
DebugListenAddr string `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"`
EnableDebugServer bool `envconfig:"ENABLE_DEBUG_SERVER" default:"false"`
EnableSecretsEndpoint bool `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"`
EnableListingAllSecrets bool `envconfig:"ENABLE_LISTING_ALL_SECRETS" default:"false"`
SecretCreationPrefix string `envconfig:"SECRET_CREATION_PREFIX" default:"testkube-"`
DisableMongoMigrations bool `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"`
Debug bool `envconfig:"DEBUG" default:"false"`
Trace bool `envconfig:"TRACE" default:"false"`
EnableImageDataPersistentCache bool `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"`
ImageDataPersistentCacheKey string `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"`
LogServerGrpcAddress string `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"`
LogServerSecure bool `envconfig:"LOG_SERVER_SECURE" default:"false"`
LogServerSkipVerify bool `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"`
LogServerCertFile string `envconfig:"LOG_SERVER_CERT_FILE" default:""`
LogServerKeyFile string `envconfig:"LOG_SERVER_KEY_FILE" default:""`
LogServerCAFile string `envconfig:"LOG_SERVER_CA_FILE" default:""`
DisableSecretCreation bool `envconfig:"DISABLE_SECRET_CREATION" default:"false"`
TestkubeExecutionNamespaces string `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""`
TestkubeDefaultStorageClassName string `envconfig:"TESTKUBE_DEFAULT_STORAGE_CLASS_NAME" default:""`
GlobalWorkflowTemplateName string `envconfig:"TESTKUBE_GLOBAL_WORKFLOW_TEMPLATE_NAME" default:""`
EnableK8sEvents bool `envconfig:"ENABLE_K8S_EVENTS" default:"true"`

// DEPRECATED: Use TestkubeProAPIKey instead
TestkubeCloudAPIKey string `envconfig:"TESTKUBE_CLOUD_API_KEY" default:""`
Expand Down
33 changes: 33 additions & 0 deletions pkg/cache/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package cache

import (
"context"
"math"
"time"

"github.com/pkg/errors"
)

var (
ErrNotFound = errors.New("item not found")
)

type Cache[T any] interface {
// Get retrieves the cached value for the given key.
// If the key is not found or expired, the method should return ErrNotFound.
Get(ctx context.Context, key string) (T, error)
// Set stores the value in the cache with the given key.
// If ttl is 0, the item should not be cached and this method should return no error.
Set(ctx context.Context, key string, value T, ttl time.Duration) error
}

// IsCacheMiss returns true if the error is a cache miss error.
// This is a helper function to determine so users don't have to compare errors manually.
func IsCacheMiss(err error) bool {
return errors.Is(err, ErrNotFound)
}

// InfiniteTTL returns a time.Duration that represents an infinite TTL.
func InfiniteTTL() time.Duration {
return math.MaxInt64
}
71 changes: 71 additions & 0 deletions pkg/cache/inmem.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package cache

import (
"context"
"sync"
"time"

"github.com/pkg/errors"
)

type item[T any] struct {
value T
expiresAt *time.Time
}

// timeGetter is a function that returns the current time.
type timeGetter func() time.Time

type InMemoryCache[T any] struct {
cache sync.Map
timeGetter timeGetter
}

// NewInMemoryCache creates a new in-memory cache.
// The underlying cache implementation uses a sync.Map so it is thread-safe.
func NewInMemoryCache[T any]() *InMemoryCache[T] {
return &InMemoryCache[T]{
timeGetter: time.Now,
}
}

func (c *InMemoryCache[T]) Get(ctx context.Context, key string) (T, error) {
var defaultVal T
rawItem, ok := c.cache.Load(key)
if !ok {
return defaultVal, ErrNotFound
}
i, ok := rawItem.(*item[T])
if !ok {
return defaultVal, errors.New("unexpected item type found in cache")
}

if i.expiresAt != nil && i.expiresAt.Before(time.Now()) {
c.cache.Delete(key)
return defaultVal, ErrNotFound
}

return i.value, nil
}

func (c *InMemoryCache[T]) Set(ctx context.Context, key string, value T, ttl time.Duration) error {
if ttl < 0 {
return errors.New("ttl must be greater than 0")
}
if ttl == 0 {
return nil
}

i := &item[T]{
value: value,
}
if ttl > 0 {
expiresAt := c.timeGetter().Add(ttl)
i.expiresAt = &expiresAt
}
c.cache.Store(key, i)

return nil
}

var _ Cache[any] = &InMemoryCache[any]{}
Loading

0 comments on commit 4e3477f

Please sign in to comment.