Change debug-tools command to get-column-counts (Consensys#5960)

The debug-tool has been updated to output the counts of all columns, along with the actual column name, and no longer filters 0's from output. Also cleaned up some of the functions that were used in the debug-tool. Signed-off-by: Paul Harris <paul.harris@consensys.net>
mcdee · Jul 21, 2022 · a03d77d · a03d77d
1 parent 3d4bdbb
commit a03d77d
Show file tree

Hide file tree

Showing 13 changed files with 47 additions and 163 deletions.
diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java
@@ -102,11 +102,6 @@ public interface Database extends AutoCloseable {
   @MustBeClosed
   Stream<SignedBeaconBlock> streamFinalizedBlocks(UInt64 startSlot, UInt64 endSlot);
 
-  @MustBeClosed
-  Stream<SignedBeaconBlock> streamHotBlocks();
-
-  long countUnblindedFinalizedBlocks();
-
   @MustBeClosed
   Stream<Map.Entry<Bytes32, BlockCheckpoints>> streamBlockCheckpoints();
 
@@ -133,11 +128,7 @@ public interface Database extends AutoCloseable {
 
   void storeVotes(Map<UInt64, VoteTracker> votes);
 
-  long countBlindedBlocks();
-
-  long countExecutionPayloads();
-
-  long countNonCanonicalSlots();
+  Map<String, Long> getColumnCounts();
 
   void migrate();
 }
diff --git a/...e/src/main/java/tech/pegasys/teku/storage/server/kvstore/BlindedBlockKvStoreDatabase.java b/...e/src/main/java/tech/pegasys/teku/storage/server/kvstore/BlindedBlockKvStoreDatabase.java
@@ -123,17 +123,6 @@ public Optional<SignedBeaconBlock> getHotBlock(final Bytes32 blockRoot) {
     return getUnblindedBlock(dao.getBlindedBlock(blockRoot));
   }
 
-  @Override
-  @MustBeClosed
-  public Stream<SignedBeaconBlock> streamHotBlocks() {
-    return dao.streamBlindedHotBlocks().map(this::getUnblindedBlock);
-  }
-
-  @Override
-  public long countUnblindedFinalizedBlocks() {
-    return 0;
-  }
-
   @Override
   @MustBeClosed
   public Stream<SignedBeaconBlock> streamFinalizedBlocks(
@@ -241,8 +230,8 @@ public Optional<SignedBeaconBlock> getEarliestAvailableBlock() {
   }
 
   @Override
-  public long countBlindedBlocks() {
-    return dao.countBlindedBlocks();
+  public Map<String, Long> getColumnCounts() {
+    return dao.getColumnCounts();
   }
 
   @Override

diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java
@@ -462,18 +462,6 @@ public void storeVotes(final Map<UInt64, VoteTracker> votes) {
     }
   }
 
-  @Override
-  public long countExecutionPayloads() {
-    try (final Stream<?> stream = dao.streamExecutionPayloads()) {
-      return stream.count();
-    }
-  }
-
-  @Override
-  public long countNonCanonicalSlots() {
-    return dao.countNonCanonicalSlots();
-  }
-
   @Override
   public void close() throws Exception {
     dao.close();

diff --git a/...src/main/java/tech/pegasys/teku/storage/server/kvstore/UnblindedBlockKvStoreDatabase.java b/...src/main/java/tech/pegasys/teku/storage/server/kvstore/UnblindedBlockKvStoreDatabase.java
@@ -124,17 +124,6 @@ public Optional<SignedBeaconBlock> getHotBlock(final Bytes32 blockRoot) {
     return dao.getHotBlock(blockRoot);
   }
 
-  @Override
-  @MustBeClosed
-  public Stream<SignedBeaconBlock> streamHotBlocks() {
-    return dao.streamHotBlocks();
-  }
-
-  @Override
-  public long countUnblindedFinalizedBlocks() {
-    return dao.countUnblindedFinalizedBlocks();
-  }
-
   @Override
   @MustBeClosed
   public Stream<SignedBeaconBlock> streamFinalizedBlocks(
@@ -203,8 +192,8 @@ protected Optional<SignedBeaconBlock> getFinalizedBlock(final Bytes32 root) {
   }
 
   @Override
-  public long countBlindedBlocks() {
-    return 0;
+  public Map<String, Long> getColumnCounts() {
+    return dao.getColumnCounts();
   }
 
   @Override

diff --git a/...src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java b/...src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java
@@ -18,6 +18,7 @@
 import com.google.errorprone.annotations.MustBeClosed;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -330,11 +331,6 @@ public long countNonCanonicalSlots() {
     return db.size(schema.getColumnNonCanonicalRootsBySlot());
   }
 
-  @Override
-  public long countBlindedBlocks() {
-    return db.size(schema.getColumnBlindedBlocksByRoot());
-  }
-
   @Override
   @MustBeClosed
   public Stream<SignedBeaconBlock> streamFinalizedBlocks(
@@ -343,14 +339,6 @@ public Stream<SignedBeaconBlock> streamFinalizedBlocks(
         .map(ColumnEntry::getValue);
   }
 
-  @Override
-  public long countUnblindedFinalizedBlocks() {
-    try (Stream<ColumnEntry<Bytes, Bytes>> entries =
-        db.streamRaw(schema.getColumnSlotsByFinalizedRoot())) {
-      return entries.count();
-    }
-  }
-
   @Override
   public Optional<UInt64> getEarliestBlindedBlockSlot() {
     return db.getFirstEntry(schema.getColumnFinalizedBlockRootBySlot()).map(ColumnEntry::getKey);
@@ -376,25 +364,11 @@ public Optional<Bytes32> getFinalizedBlockRootAtSlot(final UInt64 slot) {
     return db.get(schema.getColumnFinalizedBlockRootBySlot(), slot);
   }
 
-  @Override
-  @MustBeClosed
-  public Stream<Bytes> streamExecutionPayloads() {
-    return db.stream(schema.getColumnExecutionPayloadByPayloadHash()).map(ColumnEntry::getValue);
-  }
-
   @Override
   public Optional<SignedBeaconBlock> getBlindedBlock(final Bytes32 root) {
     return db.get(schema.getColumnBlindedBlocksByRoot(), root);
   }
 
-  @Override
-  @MustBeClosed
-  public Stream<SignedBeaconBlock> streamBlindedHotBlocks() {
-    return db.stream(schema.getColumnHotBlockCheckpointEpochsByRoot())
-        .map(ColumnEntry::getKey)
-        .flatMap(root -> getBlindedBlock(root).stream());
-  }
-
   @Override
   public Optional<Bytes> getExecutionPayload(final Bytes32 root) {
     return db.get(schema.getColumnExecutionPayloadByPayloadHash(), root);
@@ -425,6 +399,13 @@ public Optional<UInt64> getOptimisticTransitionBlockSlot() {
     return db.get(schema.getOptimisticTransitionBlockSlot());
   }
 
+  @Override
+  public Map<String, Long> getColumnCounts() {
+    final Map<String, Long> columnCounts = new LinkedHashMap<>();
+    schema.getColumnMap().forEach((k, v) -> columnCounts.put(k, db.size(v)));
+    return columnCounts;
+  }
+
   @Override
   public Optional<? extends SignedBeaconBlock> getNonCanonicalBlock(final Bytes32 root) {
     return db.get(schema.getColumnNonCanonicalBlocksByRoot(), root);

diff --git a/...n/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java b/...n/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java
@@ -15,6 +15,7 @@
 
 import com.google.errorprone.annotations.MustBeClosed;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -202,43 +203,18 @@ public long countNonCanonicalSlots() {
     return finalizedDao.countNonCanonicalSlots();
   }
 
-  @Override
-  public long countBlindedBlocks() {
-    return finalizedDao.countBlindedBlocks();
-  }
-
   @Override
   @MustBeClosed
   public Stream<SignedBeaconBlock> streamFinalizedBlocks(
       final UInt64 startSlot, final UInt64 endSlot) {
     return finalizedDao.streamFinalizedBlocks(startSlot, endSlot);
   }
 
-  @Override
-  public long countUnblindedFinalizedBlocks() {
-    return finalizedDao.countUnblindedFinalizedBlocks();
-  }
-
-  @Override
-  @MustBeClosed
-  public Stream<Bytes> streamExecutionPayloads() {
-    return finalizedDao.streamExecutionPayloads();
-  }
-
   @Override
   public Optional<SignedBeaconBlock> getBlindedBlock(final Bytes32 root) {
     return finalizedDao.getBlindedBlock(root);
   }
 
-  @Override
-  @MustBeClosed
-  public Stream<SignedBeaconBlock> streamBlindedHotBlocks() {
-    return hotDao
-        .streamBlockCheckpoints()
-        .map(Map.Entry::getKey)
-        .flatMap(blockRoot -> getBlindedBlock(blockRoot).stream());
-  }
-
   @Override
   public Optional<Bytes> getExecutionPayload(final Bytes32 root) {
     return finalizedDao.getExecutionPayload(root);
@@ -285,6 +261,13 @@ public Optional<UInt64> getOptimisticTransitionBlockSlot() {
     return finalizedDao.getOptimisticTransitionBlockSlot();
   }
 
+  @Override
+  public Map<String, Long> getColumnCounts() {
+    final HashMap<String, Long> result = new LinkedHashMap<>(hotDao.getColumnCounts());
+    result.putAll(finalizedDao.getColumnCounts());
+    return result;
+  }
+
   @Override
   public Optional<? extends SignedBeaconBlock> getNonCanonicalBlock(final Bytes32 root) {
     return finalizedDao.getNonCanonicalBlock(root);

diff --git a/...n/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoBlinded.java b/...n/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoBlinded.java
@@ -40,13 +40,8 @@ public interface KvStoreCombinedDaoBlinded extends KvStoreCombinedDaoCommon {
 
   List<SignedBeaconBlock> getBlindedNonCanonicalBlocksAtSlot(UInt64 slot);
 
-  long countBlindedBlocks();
-
   Optional<SignedBeaconBlock> getBlindedBlock(Bytes32 root);
 
-  @MustBeClosed
-  Stream<SignedBeaconBlock> streamBlindedHotBlocks();
-
   Optional<Bytes> getExecutionPayload(Bytes32 root);
 
   Optional<UInt64> getEarliestBlindedBlockSlot();

diff --git a/...in/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoCommon.java b/...in/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoCommon.java
@@ -20,7 +20,6 @@
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
-import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.ethereum.pow.api.DepositsFromBlockEvent;
 import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent;
@@ -76,11 +75,10 @@ public interface KvStoreCombinedDaoCommon extends AutoCloseable {
 
   long countNonCanonicalSlots();
 
-  @MustBeClosed
-  Stream<Bytes> streamExecutionPayloads();
-
   Optional<UInt64> getOptimisticTransitionBlockSlot();
 
+  Map<String, Long> getColumnCounts();
+
   interface CombinedUpdaterCommon extends HotUpdaterCommon, FinalizedUpdaterCommon {}
 
   interface HotUpdaterCommon extends AutoCloseable {

diff --git a/...java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoUnblinded.java b/...java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoUnblinded.java
@@ -56,8 +56,6 @@ public interface KvStoreCombinedDaoUnblinded extends KvStoreCombinedDaoCommon {
   @MustBeClosed
   Stream<SignedBeaconBlock> streamFinalizedBlocks(UInt64 startSlot, UInt64 endSlot);
 
-  long countUnblindedFinalizedBlocks();
-
   Optional<UInt64> getSlotForFinalizedBlockRoot(Bytes32 blockRoot);
 
   Optional<UInt64> getSlotForFinalizedStateRoot(Bytes32 stateRoot);

diff --git a/.../main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4FinalizedKvStoreDao.java b/.../main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4FinalizedKvStoreDao.java
@@ -15,6 +15,7 @@
 
 import com.google.errorprone.annotations.MustBeClosed;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -118,11 +119,6 @@ public long countUnblindedFinalizedBlocks() {
     }
   }
 
-  @MustBeClosed
-  public Stream<Bytes> streamExecutionPayloads() {
-    return db.stream(schema.getColumnExecutionPayloadByPayloadHash()).map(ColumnEntry::getValue);
-  }
-
   public Optional<SignedBeaconBlock> getBlindedBlock(final Bytes32 root) {
     return db.get(schema.getColumnBlindedBlocksByRoot(), root);
   }
@@ -215,6 +211,12 @@ public Set<Bytes32> getNonCanonicalBlockRootsAtSlot(final UInt64 slot) {
     return db.get(schema.getColumnNonCanonicalRootsBySlot(), slot).orElseGet(HashSet::new);
   }
 
+  public Map<String, Long> getColumnCounts() {
+    final Map<String, Long> columnCounts = new HashMap<>();
+    schema.getColumnMap().forEach((k, v) -> columnCounts.put(k, db.size(v)));
+    return columnCounts;
+  }
+
   static class V4FinalizedUpdater implements FinalizedUpdaterBlinded, FinalizedUpdaterUnblinded {
     private final KvStoreTransaction transaction;
     private final KvStoreAccessor db;

diff --git a/...ge/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4HotKvStoreDao.java b/...ge/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4HotKvStoreDao.java
@@ -14,6 +14,7 @@
 package tech.pegasys.teku.storage.server.kvstore.dataaccess;
 
 import com.google.errorprone.annotations.MustBeClosed;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -171,6 +172,12 @@ public Map<String, KvStoreVariable<?>> getVariableMap() {
     return schema.getVariableMap();
   }
 
+  public Map<String, Long> getColumnCounts() {
+    final Map<String, Long> columnCounts = new LinkedHashMap<>();
+    schema.getColumnMap().forEach((k, v) -> columnCounts.put(k, db.size(v)));
+    return columnCounts;
+  }
+
   static class V4HotUpdater implements HotUpdaterBlinded, HotUpdaterUnblinded {
 
     private final KvStoreTransaction transaction;

diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java
@@ -16,6 +16,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -125,16 +126,6 @@ public Stream<SignedBeaconBlock> streamFinalizedBlocks(
     return Stream.empty();
   }
 
-  @Override
-  public Stream<SignedBeaconBlock> streamHotBlocks() {
-    return Stream.empty();
-  }
-
-  @Override
-  public long countUnblindedFinalizedBlocks() {
-    return 0;
-  }
-
   @Override
   public Stream<Map.Entry<Bytes32, BlockCheckpoints>> streamBlockCheckpoints() {
     return Stream.empty();
@@ -187,18 +178,8 @@ public void addDepositsFromBlockEvent(final DepositsFromBlockEvent event) {}
   public void storeVotes(final Map<UInt64, VoteTracker> votes) {}
 
   @Override
-  public long countBlindedBlocks() {
-    return 0;
-  }
-
-  @Override
-  public long countExecutionPayloads() {
-    return 0;
-  }
-
-  @Override
-  public long countNonCanonicalSlots() {
-    return 0;
+  public Map<String, Long> getColumnCounts() {
+    return new HashMap<>();
   }
 
   @Override