SPM-1813: lru cache utilization metrics

evaluator/metrics.go

@@ -54,11 +54,40 @@ var (
 		Name:      "two_evaluations_interval_hours",
 		Buckets:   []float64{1, 2, 6, 24, 72, 168},
 	})
+
+	packageCacheCnt = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Help:      "How many packages hit/miss package cache",
+		Namespace: "patchman_engine",
+		Subsystem: "evaluator",
+		Name:      "package_cache",
+	}, []string{"type", "by"})
+
+	packageCacheGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Help:      "Package cache size",
+		Namespace: "patchman_engine",
+		Subsystem: "evaluator",
+		Name:      "package_cache_size",
+	}, []string{"by"})
+
+	vmaasCacheCnt = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Help:      "How many vmaas checksums hit/miss cache",
+		Namespace: "patchman_engine",
+		Subsystem: "evaluator",
+		Name:      "vmaas_cache",
+	}, []string{"type"})
+
+	vmaasCacheGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+		Help:      "VMaaS cache size",
+		Namespace: "patchman_engine",
+		Subsystem: "evaluator",
+		Name:      "vmaas_cache_size",
+	})
 )
 
 func RunMetrics() {
 	prometheus.MustRegister(evaluationCnt, updatesCnt, evaluationDuration, evaluationPartDuration,
-		uploadEvaluationDelay, twoEvaluationsInterval)
+		uploadEvaluationDelay, twoEvaluationsInterval, packageCacheCnt, packageCacheGauge,
+		vmaasCacheCnt, vmaasCacheGauge)
 
 	// create web app
 	app := gin.New()

evaluator/package_cache.go

@@ -44,6 +44,11 @@ func NewPackageCache(enabled bool, preload bool, size int, nameSize int) *Packag
 	c.size = size
 	c.nameSize = nameSize
 
+	packageCacheGauge.WithLabelValues("id").Set(0)
+	packageCacheGauge.WithLabelValues("nevra").Set(0)
+	packageCacheGauge.WithLabelValues("name").Set(0)
+	packageCacheGauge.WithLabelValues("nameByID").Set(0)
+
 	if c.enabled {
 		var err error
 		c.byID, err = lru.New(c.size)
@@ -120,6 +125,7 @@ func (c *PackageCache) GetByID(id int64) (*PackageCacheMetadata, bool) {
 	if c.enabled {
 		val, ok := c.byID.Get(id)
 		if ok {
+			packageCacheCnt.WithLabelValues("hit", "id").Inc()
 			utils.LogTrace("id", id, "PackageCache.GetByID cache hit")
 			metadata := val.(*PackageCacheMetadata)
 			return metadata, true
@@ -132,6 +138,7 @@ func (c *PackageCache) GetByID(id int64) (*PackageCacheMetadata, bool) {
 		utils.LogTrace("id", id, "PackageCache.GetByID read from db")
 		return metadata, true
 	}
+	packageCacheCnt.WithLabelValues("miss", "id").Inc()
 	utils.LogTrace("id", id, "PackageCache.GetByID not found")
 	return nil, false
 }
@@ -140,6 +147,7 @@ func (c *PackageCache) GetByNevra(nevra string) (*PackageCacheMetadata, bool) {
 	if c.enabled {
 		val, ok := c.byNevra.Get(nevra)
 		if ok {
+			packageCacheCnt.WithLabelValues("hit", "nevra").Inc()
 			utils.LogTrace("nevra", nevra, "PackageCache.GetByNevra cache hit")
 			metadata := val.(*PackageCacheMetadata)
 			return metadata, true
@@ -152,6 +160,7 @@ func (c *PackageCache) GetByNevra(nevra string) (*PackageCacheMetadata, bool) {
 		utils.LogTrace("nevra", nevra, "PackageCache.GetByNevra read from db")
 		return metadata, true
 	}
+	packageCacheCnt.WithLabelValues("miss", "nevra").Inc()
 	utils.LogTrace("nevra", nevra, "PackageCache.GetByNevra not found")
 	return nil, false
 }
@@ -160,6 +169,7 @@ func (c *PackageCache) GetLatestByName(name string) (*PackageCacheMetadata, bool
 	if c.enabled {
 		val, ok := c.latestByName.Get(name)
 		if ok {
+			packageCacheCnt.WithLabelValues("hit", "name").Inc()
 			utils.LogTrace("name", name, "PackageCache.GetLatestByName cache hit")
 			metadata := val.(*PackageCacheMetadata)
 			return metadata, true
@@ -172,6 +182,7 @@ func (c *PackageCache) GetLatestByName(name string) (*PackageCacheMetadata, bool
 		utils.LogTrace("name", name, "PackageCache.GetLatestByName read from db")
 		return metadata, true
 	}
+	packageCacheCnt.WithLabelValues("miss", "name").Inc()
 	utils.LogTrace("name", name, "PackageCache.GetLatestByName not found")
 	return nil, false
 }
@@ -180,6 +191,7 @@ func (c *PackageCache) GetNameByID(id int64) (string, bool) {
 	if c.enabled {
 		val, ok := c.nameByID.Get(id)
 		if ok {
+			packageCacheCnt.WithLabelValues("hit", "nameByID").Inc()
 			utils.LogTrace("id", id, "PackageCache.GetNameByID cache hit")
 			metadata := val.(*PackageCacheMetadata)
 			return metadata.Name, true
@@ -192,6 +204,7 @@ func (c *PackageCache) GetNameByID(id int64) (string, bool) {
 		utils.LogTrace("id", id, "PackageCache.GetNameByID read from db")
 		return metadata.Name, true
 	}
+	packageCacheCnt.WithLabelValues("miss", "nameByID").Inc()
 	utils.LogTrace("id", id, "PackageCache.GetNameByID not found")
 	return "", false
 }
@@ -205,6 +218,7 @@ func (c *PackageCache) Add(pkg *PackageCacheMetadata) {
 
 func (c *PackageCache) addByID(pkg *PackageCacheMetadata) {
 	evicted := c.byID.Add(pkg.ID, pkg)
+	packageCacheGauge.WithLabelValues("id").Inc()
 	utils.LogTrace("byID", pkg.ID, "evicted", evicted, "PackageCache.addByID")
 }
 
@@ -218,6 +232,7 @@ func (c *PackageCache) addByNevra(pkg *PackageCacheMetadata) {
 	}
 	nevraString := nevra.StringE(true)
 	evicted := c.byNevra.Add(nevraString, pkg)
+	packageCacheGauge.WithLabelValues("nevra").Inc()
 	utils.LogTrace("byNevra", nevraString, "evicted", evicted, "PackageCache.addByNevra")
 }
 
@@ -231,6 +246,7 @@ func (c *PackageCache) addLatestByName(pkg *PackageCacheMetadata) {
 		// if there is no record yet
 		// or it has older EVR we have to replace it
 		evicted := c.latestByName.Add(pkg.Name, pkg)
+		packageCacheGauge.WithLabelValues("name").Inc()
 		utils.LogTrace("latestByName", pkg.Name, "evicted", evicted, "PackageCache.addLatestByName")
 	}
 }
@@ -239,6 +255,7 @@ func (c *PackageCache) addNameByID(pkg *PackageCacheMetadata) {
 	ok, evicted := c.nameByID.ContainsOrAdd(pkg.NameID, pkg)
 	if !ok {
 		// name was not there and we've added it
+		packageCacheGauge.WithLabelValues("nameByID").Inc()
 		utils.LogTrace("nameByID", pkg.NameID, "evicted", evicted, "PackageCache.addNameByID")
 	}
 }

evaluator/vmaas_cache.go

@@ -27,6 +27,7 @@ func NewVmaasPackageCache(enabled bool, size int, checkDuration time.Duration) *
 	c.size = size
 	c.validity = vmaas_sync.GetLastSync(vmaas_sync.VmaasExported)
 	c.checkDuration = checkDuration
+	vmaasCacheGauge.Set(0)
 
 	if c.enabled {
 		var err error
@@ -43,23 +44,27 @@ func (c *VmaasCache) Get(checksum *string) (*vmaas.UpdatesV2Response, bool) {
 	if c.enabled && checksum != nil {
 		val, ok := c.data.Get(checksum)
 		if ok {
+			vmaasCacheCnt.WithLabelValues("hit").Inc()
 			utils.LogTrace("checksum", checksum, "VmaasCache.Get cache hit")
 			response := val.(*vmaas.UpdatesV2Response)
 			return response, true
 		}
 	}
+	vmaasCacheCnt.WithLabelValues("miss").Inc()
 	return nil, false
 }
 
 func (c *VmaasCache) Add(checksum *string, response *vmaas.UpdatesV2Response) {
 	if c.enabled && checksum != nil {
+		vmaasCacheGauge.Inc()
 		c.data.Add(checksum, response)
 	}
 }
 
 func (c *VmaasCache) Reset(ts *types.Rfc3339TimestampWithZ) {
 	c.data.Purge()
 	c.validity = ts
+	vmaasCacheGauge.Set(0)
 }
 
 func (c *VmaasCache) CheckValidity() {

manager/controllers/advisory_detail.go

@@ -235,6 +235,7 @@ func pkgsV2topkgsV1(pkgsV2 packagesV2) packagesV1 {
 }
 
 func initAdvisoryDetailCache() *lru.Cache {
+	middlewares.AdvisoryDetailGauge.Set(0)
 	if !enableAdvisoryDetailCache {
 		return nil
 	}
@@ -280,9 +281,11 @@ func tryGetAdvisoryFromCacheV2(advisoryName string) *AdvisoryDetailResponseV2 {
 
 	val, ok := advisoryDetailCacheV2.Get(advisoryName)
 	if !ok {
+		middlewares.AdvisoryDetailCnt.WithLabelValues("miss").Inc()
 		return nil
 	}
 	resp := val.(AdvisoryDetailResponseV2)
+	middlewares.AdvisoryDetailCnt.WithLabelValues("hit").Inc()
 	return &resp
 }
 
@@ -291,6 +294,7 @@ func tryAddAdvisoryToCacheV2(advisoryName string, resp *AdvisoryDetailResponseV2
 		return
 	}
 	evicted := advisoryDetailCacheV2.Add(advisoryName, *resp)
+	middlewares.AdvisoryDetailGauge.Inc()
 	utils.LogDebug("evictedV2", evicted, "advisoryName", advisoryName, "saved to cache")
 }
 

manager/middlewares/prometheus.go

@@ -29,9 +29,24 @@ var callerSourceCnt = prometheus.NewCounterVec(prometheus.CounterOpts{
 	Name:      "caller_source",
 }, []string{"source", "account"})
 
+var AdvisoryDetailCnt = prometheus.NewCounterVec(prometheus.CounterOpts{
+	Help:      "How many advisories hit/miss cache",
+	Namespace: "patchman_engine",
+	Subsystem: "manager",
+	Name:      "advisory_detail_cache",
+}, []string{"type"})
+
+var AdvisoryDetailGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+	Help:      "Advisory detail cache size",
+	Namespace: "patchman_engine",
+	Subsystem: "manager",
+	Name:      "advisory_detail_cache_size",
+})
+
 // Create and configure Prometheus middleware to expose metrics
 func Prometheus() *ginprometheus.Prometheus {
-	prometheus.MustRegister(serviceErrorCnt, requestDurations, callerSourceCnt)
+	prometheus.MustRegister(serviceErrorCnt, requestDurations, callerSourceCnt,
+		AdvisoryDetailCnt, AdvisoryDetailGauge)
 
 	p := ginprometheus.NewPrometheus("patchman_engine")
 	p.MetricsPath = utils.Cfg.MetricsPath