dns: configuring a basic key value store to persist to disk

CODEOWNERS

@@ -196,6 +196,8 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/formatter/req_without_query @dio @tsaarni
 # IP address input matcher
 /*/extensions/matching/input_matchers/ip @aguinet @snowp
+# Key Value store
+/*/extensions/key_value @alyssawilk @ryantheoptimist
 
 # Contrib
 /contrib/exe/ @mattklein123 @lizan

api/BUILD

@@ -97,6 +97,7 @@ proto_library(
         "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
         "//envoy/extensions/clusters/redis/v3:pkg",
         "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg",
+        "//envoy/extensions/common/key_value/v3:pkg",
         "//envoy/extensions/common/matching/v3:pkg",
         "//envoy/extensions/common/ratelimit/v3:pkg",
         "//envoy/extensions/common/tap/v3:pkg",
@@ -187,6 +188,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
+        "//envoy/extensions/key_value/file_based/v3:pkg",
         "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg",
         "//envoy/extensions/matching/input_matchers/consistent_hashing/v3:pkg",
         "//envoy/extensions/matching/input_matchers/ip/v3:pkg",

api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD

@@ -9,6 +9,7 @@ api_proto_package(
         "//envoy/annotations:pkg",
         "//envoy/config/cluster/v3:pkg",
         "//envoy/config/core/v3:pkg",
+        "//envoy/extensions/common/key_value/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
     ],
 )

api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto

@@ -6,6 +6,7 @@ import "envoy/config/cluster/v3/cluster.proto";
 import "envoy/config/core/v3/address.proto";
 import "envoy/config/core/v3/extension.proto";
 import "envoy/config/core/v3/resolver.proto";
+import "envoy/extensions/common/key_value/v3/config.proto";
 
 import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
@@ -31,7 +32,7 @@ message DnsCacheCircuitBreakers {
 
 // Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview
 // <arch_overview_http_dynamic_forward_proxy>` for more information.
-// [#next-free-field: 13]
+// [#next-free-field: 14]
 message DnsCacheConfig {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig";
@@ -138,4 +139,8 @@ message DnsCacheConfig {
   // Setting this timeout will ensure that queries succeed or fail within the specified time frame
   // and are then retried using the standard refresh rates. Defaults to 5s if not set.
   google.protobuf.Duration dns_query_timeout = 11 [(validate.rules).duration = {gt {}}];
+
+  // [#not-implemented-hide:]
+  // Configuration to flush the DNS cache to long term storage.
+  key_value.v3.KeyValueStoreConfig key_value_config = 13;
 }

api/envoy/extensions/common/key_value/v3/BUILD

@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "//envoy/config/core/v3:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+    ],
+)

api/envoy/extensions/common/key_value/v3/config.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package envoy.extensions.common.key_value.v3;
+
+import "envoy/config/core/v3/extension.proto";
+
+import "google/protobuf/any.proto";
+import "google/protobuf/duration.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.common.key_value.v3";
+option java_outer_classname = "ConfigProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Key Value Store storage plugin]
+
+// [#alpha:]
+// This shared configuration for Envoy key value stores.
+message KeyValueStoreConfig {
+  // [#extension-category: envoy.common.key_value]
+  config.core.v3.TypedExtensionConfig config = 1 [(validate.rules).message = {required: true}];
+}

api/envoy/extensions/key_value/file_based/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/key_value/file_based/v3/config.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package envoy.extensions.key_value.file_based.v3;
+
+import "google/protobuf/duration.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.key_value.file_based.v3";
+option java_outer_classname = "ConfigProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: File Based Key Value Store storage plugin]
+
+// [#alpha:]
+// [#extension: envoy.key_value.file_based]
+// This is configuration to flush a key value store out to disk.
+message FileBasedKeyValueStoreConfig {
+  // The filename to read the keys and values from, and write the keys and
+  // values to.
+  string filename = 1 [(validate.rules).string = {min_len: 1}];
+
+  // The interval at which the key value store should be flushed to the file.
+  google.protobuf.Duration flush_interval = 2;
+}

api/versioning/BUILD

@@ -49,6 +49,7 @@ proto_library(
         "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
         "//envoy/extensions/clusters/redis/v3:pkg",
         "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg",
+        "//envoy/extensions/common/key_value/v3:pkg",
         "//envoy/extensions/common/matching/v3:pkg",
         "//envoy/extensions/common/ratelimit/v3:pkg",
         "//envoy/extensions/common/tap/v3:pkg",
@@ -139,6 +140,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
+        "//envoy/extensions/key_value/file_based/v3:pkg",
         "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg",
         "//envoy/extensions/matching/input_matchers/consistent_hashing/v3:pkg",
         "//envoy/extensions/matching/input_matchers/ip/v3:pkg",

docs/root/api-v3/common_messages/common_messages.rst

@@ -20,12 +20,14 @@ Common messages
   ../config/core/v3/socket_option.proto
   ../config/core/v3/udp_socket_config.proto
   ../config/core/v3/substitution_format_string.proto
+  ../extensions/common/key_value/v3/config.proto
   ../extensions/common/ratelimit/v3/ratelimit.proto
   ../extensions/filters/common/fault/v3/fault.proto
   ../extensions/network/socket_interface/v3/default_socket_interface.proto
   ../extensions/common/matching/v3/extension_matcher.proto
   ../extensions/filters/common/dependency/v3/dependency.proto
   ../extensions/filters/common/matcher/action/v3/skip_action.proto
+  ../extensions/key_value/file_based/v3/config.proto
   ../extensions/matching/input_matchers/consistent_hashing/v3/consistent_hashing.proto
   ../extensions/matching/input_matchers/ip/v3/ip.proto
   ../extensions/matching/common_inputs/environment_variable/v3/input.proto

envoy/common/BUILD

@@ -75,6 +75,8 @@ envoy_cc_library(
     name = "key_value_store_interface",
     hdrs = ["key_value_store.h"],
     deps = [
+        "//envoy/protobuf:message_validator_interface",
+        "//envoy/registry",
     ],
 )
 

envoy/common/key_value_store.h

@@ -1,6 +1,10 @@
 #pragma once
 
 #include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/filesystem/filesystem.h"
+#include "envoy/protobuf/message_validator.h"
 
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
@@ -37,6 +41,45 @@ class KeyValueStore {
    * Flushes the store to long term storage.
    */
   virtual void flush() PURE;
+
+  // Returns for the iterate function.
+  enum class Iterate { Continue, Break };
+
+  /**
+   * Callback when calling iterate() in a key value store.
+   * @param key is the key for a given entry
+   * @param value is the value for a given entry
+   * @return Iterate::Continue to continue iteration, or Iterate::Break to stop.
+   */
+  using ConstIterateCb = std::function<Iterate(const std::string& key, const std::string& value)>;
+
+  /**
+   * Iterate over a key value store.
+   * @param cb supplies the iteration callback.
+   */
+  virtual void iterate(ConstIterateCb cb) const PURE;
+};
+
+using KeyValueStorePtr = std::unique_ptr<KeyValueStore>;
+
+// A factory for creating key value stores.
+class KeyValueStoreFactory : public Envoy::Config::TypedFactory {
+public:
+  /**
+   * Function to create KeyValueStores from the specified config.
+   * @param cb supplies the key value store configuration
+   * @param validation_visitor the configuration validator
+   * @dispatcher the dispatcher for the thread, for flush alarms.
+   * @file_system the file system.
+   * @return a new key value store.
+   */
+  virtual KeyValueStorePtr createStore(const Protobuf::Message& config,
+                                       ProtobufMessage::ValidationVisitor& validation_visitor,
+                                       Event::Dispatcher& dispatcher,
+                                       Filesystem::Instance& file_system) PURE;
+
+  // @brief the category of the key value store for factory registration.
+  std::string category() const override { return "envoy.common.key_value"; }
 };
 
 } // namespace Envoy

envoy/registry/BUILD

@@ -13,6 +13,7 @@ envoy_cc_library(
     hdrs = ["registry.h"],
     deps = [
         "//source/common/common:assert_lib",
+        "//source/common/common:minimal_logger_lib",
         "//source/common/protobuf:utility_lib",
         "//source/extensions/common:utility_lib",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",

generated_api_shadow/BUILD

@@ -97,6 +97,7 @@ proto_library(
         "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
         "//envoy/extensions/clusters/redis/v3:pkg",
         "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg",
+        "//envoy/extensions/common/key_value/v3:pkg",
         "//envoy/extensions/common/matching/v3:pkg",
         "//envoy/extensions/common/ratelimit/v3:pkg",
         "//envoy/extensions/common/tap/v3:pkg",
@@ -187,6 +188,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
+        "//envoy/extensions/key_value/file_based/v3:pkg",
         "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg",
         "//envoy/extensions/matching/input_matchers/consistent_hashing/v3:pkg",
         "//envoy/extensions/matching/input_matchers/ip/v3:pkg",