Merge pull request google#1061 from AlCutter/split_smt_and_verifiable_map

Split out VerifiableMap from SparseMerkleTree.

Author: AlCutter

.gitignore

@@ -58,6 +58,7 @@
 /cpp/merkletree/serial_hasher_test
 /cpp/merkletree/sparse_merkle_tree_test
 /cpp/merkletree/tree_hasher_test
+/cpp/merkletree/verifiable_map_test
 /cpp/monitor/database_test
 /cpp/monitoring/counter_test
 /cpp/monitoring/gauge_test

Makefile.am

@@ -89,6 +89,7 @@ TESTS = \
 	cpp/merkletree/serial_hasher_test \
 	cpp/merkletree/sparse_merkle_tree_test \
 	cpp/merkletree/tree_hasher_test \
+	cpp/merkletree/verifiable_map_test \
 	cpp/monitor/database_test \
 	cpp/monitoring/counter_test \
 	cpp/monitoring/gauge_test \
@@ -163,6 +164,7 @@ cpp_libcore_a_SOURCES = \
 	cpp/merkletree/serial_hasher.cc \
 	cpp/merkletree/sparse_merkle_tree.cc \
 	cpp/merkletree/tree_hasher.cc \
+	cpp/merkletree/verifiable_map.cc \
 	cpp/monitoring/gcm/exporter.cc \
 	cpp/monitoring/monitoring.cc \
 	cpp/monitoring/prometheus/exporter.cc \
@@ -773,6 +775,14 @@ cpp_merkletree_tree_hasher_test_SOURCES = \
 	cpp/util/util.cc \
 	cpp/merkletree/tree_hasher_test.cc
 
+cpp_merkletree_verifiable_map_test_LDADD = \
+	cpp/libcore.a \
+	cpp/libtest.a \
+	$(libevent_LIBS)
+cpp_merkletree_verifiable_map_test_SOURCES = \
+	cpp/util/util.cc \
+	cpp/merkletree/verifiable_map_test.cc
+
 cpp_util_sync_task_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \

cpp/merkletree/sparse_merkle_tree.cc

@@ -45,17 +45,11 @@ void SparseMerkleTree::EnsureHaveLevel(size_t level) {
 }
 
 
-void SparseMerkleTree::SetLeafByHash(const string& key, const string& data) {
-  serial_hasher_->Reset();
-  serial_hasher_->Update(key);
-  return SetLeaf(PathFromBytes(serial_hasher_->Final()), data);
-}
-
-
 void SparseMerkleTree::SetLeaf(const Path& path, const string& data) {
   CHECK_EQ(treehasher_.DigestSize(), path.size());
   // Mark the tree dirty:
   root_hash_.clear();
+  string leaf_hash(treehasher_.HashLeaf(data));
 
   IndexType node_index(0);
   for (int depth(0); depth <= kDigestSizeBits; ++depth) {
@@ -64,29 +58,29 @@ void SparseMerkleTree::SetLeaf(const Path& path, const string& data) {
     auto it(tree_[depth].find(node_index));
     if (it == tree_[depth].end()) {
       CHECK(tree_[depth]
-                .emplace(make_pair(node_index, TreeNode(path, data)))
+                .emplace(make_pair(node_index, TreeNode(path, leaf_hash)))
                 .second);
       return;
     } else if (it->second.type_ == TreeNode::INTERNAL) {
       // Mark the internal node hash dirty
-      it->second.hash_.reset();
-    } else if (it->second.leaf_->path_ == path) {
+      it->second.hash_.clear();
+    } else if (*it->second.path_ == path) {
       // replacement
       CHECK_EQ(TreeNode::LEAF, it->second.type_);
-      it->second.leaf_->value_ = data;
+      it->second.hash_ = std::move(leaf_hash);
       return;
     } else {
       // restructure: push the existing node down a level and replace this one
       // with an INTERNAL node
       CHECK_LT(depth, kDigestSizeBits);
       EnsureHaveLevel(depth + 1);
       IndexType child_index((node_index << 1) +
-                            PathBit(it->second.leaf_->path_, depth + 1));
+                            PathBit(*it->second.path_, depth + 1));
       CHECK(tree_[depth + 1]
                 .emplace(make_pair(child_index, std::move(it->second)))
                 .second);
       it->second.type_ = TreeNode::INTERNAL;
-      it->second.leaf_.reset();
+      it->second.hash_.clear();
     }
     node_index <<= 1;
   }
@@ -129,22 +123,21 @@ string SparseMerkleTree::CalculateSubtreeHash(size_t depth, IndexType index) {
   if (it != tree_[depth].end()) {
     switch (it->second.type_) {
       case TreeNode::INTERNAL: {
-        if (it->second.hash_) {
-          return *(it->second.hash_);
+        if (!it->second.hash_.empty()) {
+          return it->second.hash_;
         }
         IndexType left_child_index(index << 1);
         const string left(CalculateSubtreeHash(depth + 1, left_child_index));
         const string right(
             CalculateSubtreeHash(depth + 1, left_child_index + 1));
-        it->second.hash_.reset(
-            new string(treehasher_.HashChildren(left, right)));
-        return *(it->second.hash_);
+        it->second.hash_.assign(treehasher_.HashChildren(left, right));
+        return it->second.hash_;
       }
 
       case TreeNode::LEAF: {
-        string ret(treehasher_.HashLeaf(it->second.leaf_->value_));
+        string ret(it->second.hash_);
         for (int i(kDigestSizeBits - 1); i > depth; --i) {
-          if (PathBit(it->second.leaf_->path_, i) == 0) {
+          if (PathBit(*(it->second.path_), i) == 0) {
             ret = treehasher_.HashChildren(ret, null_hashes_->at(i));
           } else {
             ret = treehasher_.HashChildren(null_hashes_->at(i), ret);
@@ -178,29 +171,28 @@ std::vector<string> SparseMerkleTree::InclusionProof(const Path& path) {
 
 string SparseMerkleTree::TreeNode::DebugString() const {
   ostringstream os;
-  os << "[TreeNode ";
+  os << "[TreeNode type: ";
   switch (type_) {
     case INTERNAL:
-      os << "INTL hash: ";
-      if (hash_) {
-        os << util::ToBase64(*hash_);
-      } else {
-        os << "(unset)";
-      }
+      os << "I";
       break;
     case LEAF:
-      os << "LEAF leaf: ";
-      os << leaf_->DebugString();
+      os << "L";
       break;
   }
-  os << "]";
-  return os.str();
-}
 
+  os << " hash: ";
+  if (!hash_.empty()) {
+    os << util::ToBase64(hash_);
+  } else {
+    os << "(unset)";
+  }
 
-string SparseMerkleTree::Leaf::DebugString() const {
-  ostringstream os;
-  os << "[Leaf path: " << path_ << " value: " << value_ << "]";
+  if (path_) {
+    os << " path: ";
+    os << *path_;
+  }
+  os << "]";
   return os.str();
 }
 

cpp/merkletree/sparse_merkle_tree.h

@@ -20,6 +20,7 @@ class SerialHasher;
 // Visible out here because it's useful for testing too.
 const std::vector<std::string>* GetNullHashes(const TreeHasher& hasher);
 
+
 // Implementation of a Sparse Merkle Tree.
 //
 // The design is inspired by the tree described in
@@ -146,13 +147,6 @@ class SparseMerkleTree {
     return treehasher_.HashLeaf(data);
   }
 
-  // Add a new leaf to the hash tree. Stores the hash of the leaf data in the
-  // tree structure at path |Hash(key)|, does not store the data itself.
-  //
-  // @param data Binary input blob
-  // @param path Binary path of node to set.
-  virtual void SetLeafByHash(const std::string& key, const std::string& data);
-
   // Add a new leaf to the hash tree. Stores the hash of the leaf data in the
   // tree structure, does not store the data itself.
   //
@@ -188,34 +182,19 @@ class SparseMerkleTree {
   // TODO(alcutter): BIGNUM probably.
   typedef uint64_t IndexType;
 
-  struct Leaf {
-    Leaf(const Path& path, const std::string& value)
-        : path_(path), value_(value) {
-    }
-
-    std::string DebugString() const;
-
-    const Path path_;
-    std::string value_;
-  };
-
   struct TreeNode {
-    TreeNode() : type_(INTERNAL), hash_(nullptr) {
-    }
-    TreeNode(const std::string& hash)
-        : type_(INTERNAL), hash_(new std::string(hash)) {
+    TreeNode(const std::string& hash) : type_(INTERNAL), hash_(hash) {
     }
 
-    TreeNode(const Path& path, const std::string& leaf_value)
-        : type_(LEAF), leaf_(new Leaf(path, leaf_value)) {
+    TreeNode(const Path& path, const std::string& leaf_hash)
+        : type_(LEAF), path_(new Path(path)), hash_(leaf_hash) {
     }
 
     std::string DebugString() const;
 
     enum { INTERNAL, LEAF } type_;
-    // TODO(alcutter): sort this out
-    std::unique_ptr<std::string> hash_;
-    std::unique_ptr<Leaf> leaf_;
+    std::unique_ptr<Path> path_;
+    std::string hash_;
   };
 
   std::string CalculateSubtreeHash(size_t depth, IndexType index);
@@ -260,4 +239,13 @@ inline int PathBit(const SparseMerkleTree::Path& path, size_t bit) {
   return (path[bit / 8] & (1 << (7 - bit % 8))) == 0 ? 0 : 1;
 }
 
+
+struct PathHasher {
+  size_t operator()(const SparseMerkleTree::Path& p) const {
+    return std::hash<std::string>()(
+        std::string(reinterpret_cast<const char*>(p.data()), p.size()));
+  }
+};
+
+
 #endif  // CERT_TRANS_MERKLETREE_SPARSE_MERKLE_TREE_H

cpp/merkletree/sparse_merkle_tree_test.cc

@@ -15,6 +15,7 @@
 namespace {
 
 using cert_trans::ScopedBIGNUM;
+using std::fill;
 using std::lower_bound;
 using std::map;
 using std::mt19937;
@@ -192,6 +193,13 @@ class SparseMerkleTreeTest : public testing::Test {
     return ret;
   }
 
+  SparseMerkleTree::Path PathFromString(const string& s) {
+    SparseMerkleTree::Path ret;
+    CHECK_LE(s.size(), ret.size());
+    fill(copy(s.begin(), s.end(), ret.begin()), ret.end(), 0);
+    return ret;
+  }
+
   TreeHasher tree_hasher_;
   SparseMerkleTree tree_;
   mt19937 rand_;
@@ -311,22 +319,22 @@ TEST_F(SparseMerkleTreeTest, DISABLED_SMTMemTest) {
 }
 
 
-TEST_F(SparseMerkleTreeTest, TestSetLeafByHash) {
+TEST_F(SparseMerkleTreeTest, TestSetLeaf) {
   tree_.CurrentRoot();
   LOG(INFO) << "Tree@0:";
   LOG(INFO) << tree_.Dump();
 
-  tree_.SetLeafByHash("one", "one");
+  tree_.SetLeaf(PathFromString("one"), "one");
   tree_.CurrentRoot();
   LOG(INFO) << "Tree@1:";
   LOG(INFO) << tree_.Dump();
 
-  tree_.SetLeafByHash("two", "two");
+  tree_.SetLeaf(PathFromString("two"), "two");
   tree_.CurrentRoot();
   LOG(INFO) << "Tree@2:";
   LOG(INFO) << tree_.Dump();
 
-  tree_.SetLeafByHash("three", "three");
+  tree_.SetLeaf(PathFromString("three"), "three");
   tree_.CurrentRoot();
   LOG(INFO) << "Tree@3:";
   LOG(INFO) << tree_.Dump();

cpp/merkletree/verifiable_map.cc

@@ -0,0 +1,50 @@
+#include <array>
+#include <string>
+
+#include "merkletree/verifiable_map.h"
+
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+using util::Status;
+using util::StatusOr;
+
+
+namespace cert_trans {
+
+
+VerifiableMap::VerifiableMap(SerialHasher* hasher)
+    : hasher_model_(CHECK_NOTNULL(hasher)->Create()), merkle_tree_(hasher) {
+}
+
+
+void VerifiableMap::Set(const string& key, const string& value) {
+  const SparseMerkleTree::Path path(PathFromKey(key));
+  merkle_tree_.SetLeaf(path, value);
+  values_[path] = value;
+}
+
+
+StatusOr<string> VerifiableMap::Get(const string& key) const {
+  const SparseMerkleTree::Path path(PathFromKey(key));
+  const auto it(values_.find(path));
+  if (it == values_.end()) {
+    return Status(util::error::NOT_FOUND, "No such entry.");
+  }
+  return it->second;
+}
+
+
+vector<string> VerifiableMap::InclusionProof(const string& key) {
+  return merkle_tree_.InclusionProof(PathFromKey(key));
+}
+
+
+SparseMerkleTree::Path VerifiableMap::PathFromKey(const string& key) const {
+  unique_ptr<SerialHasher> h(hasher_model_->Create());
+  h->Update(key);
+  return PathFromBytes(h->Final());
+}
+
+}  // namespace cert_trans

cpp/merkletree/verifiable_map.h

@@ -0,0 +1,45 @@
+#ifndef CERT_TRANS_MERKLETREE_VERIFIABLE_MAP_H_
+#define CERT_TRANS_MERKLETREE_VERIFIABLE_MAP_H_
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "base/macros.h"
+#include "merkletree/sparse_merkle_tree.h"
+#include "util/statusor.h"
+
+namespace cert_trans {
+
+
+// Implements a Verifiable Map using a SparseMerkleTree and hashmap.
+class VerifiableMap {
+ public:
+  VerifiableMap(SerialHasher* hasher);
+
+  std::string CurrentRoot() {
+    return merkle_tree_.CurrentRoot();
+  }
+
+  void Set(const std::string& key, const std::string& value);
+
+  util::StatusOr<std::string> Get(const std::string& key) const;
+
+  std::vector<std::string> InclusionProof(const std::string& key);
+
+ private:
+  SparseMerkleTree::Path PathFromKey(const std::string& key) const;
+
+  std::unique_ptr<SerialHasher> hasher_model_;
+  SparseMerkleTree merkle_tree_;
+
+  // TODO(alcutter): allow arbitrary stores here.
+  std::unordered_map<SparseMerkleTree::Path, std::string, PathHasher> values_;
+
+  DISALLOW_COPY_AND_ASSIGN(VerifiableMap);
+};
+
+
+}  // namespace cert_trans
+
+#endif  // CERT_TRANS_MERKLETREE_VERIFIABLE_MAP_H_