Compare commits

...

3 Commits

Author SHA1 Message Date
ValarDragon
1302db7dac Update types package, fix proof's aunt handling 2018-08-30 22:43:49 -07:00
ValarDragon
4b14e17d61 Add tests to demonstrate we follow the RFC 2018-08-30 21:38:25 -07:00
ValarDragon
582a4f9c6d wip: start refactor of merkle package 2018-08-30 20:57:31 -07:00
13 changed files with 241 additions and 97 deletions

View File

@@ -0,0 +1,99 @@
package merkle
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// These tests were taken from https://github.com/google/trillian/blob/master/merkle/rfc6962/rfc6962.go,
// and consequently fall under the above license.
import (
"bytes"
"encoding/hex"
"testing"
"github.com/tendermint/tendermint/crypto/tmhash"
)
func TestRFC6962Hasher(t *testing.T) {
_, leafHashTrail := trailsFromByteSlices([][]byte{[]byte("L123456")})
leafHash := leafHashTrail.Hash
_, leafHashTrail = trailsFromByteSlices([][]byte{[]byte{}})
emptyLeafHash := leafHashTrail.Hash
for _, tc := range []struct {
desc string
got []byte
want string
}{
// Check that the empty hash is not the same as the hash of an empty leaf.
// echo -n 00 | xxd -r -p | sha256sum
{
desc: "RFC6962 Empty Leaf",
want: "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"[:tmhash.Size*2],
got: emptyLeafHash,
},
// echo -n 004C313233343536 | xxd -r -p | sha256sum
{
desc: "RFC6962 Leaf",
want: "395aa064aa4c29f7010acfe3f25db9485bbd4b91897b6ad7ad547639252b4d56"[:tmhash.Size*2],
got: leafHash,
},
// echo -n 014E3132334E343536 | xxd -r -p | sha256sum
{
desc: "RFC6962 Node",
want: "aa217fe888e47007fa15edab33c2b492a722cb106c64667fc2b044444de66bbb"[:tmhash.Size*2],
got: SimpleHashFromTwoHashes([]byte("N123"), []byte("N456")),
},
} {
t.Run(tc.desc, func(t *testing.T) {
wantBytes, err := hex.DecodeString(tc.want)
if err != nil {
t.Fatalf("hex.DecodeString(%x): %v", tc.want, err)
}
if got, want := tc.got, wantBytes; !bytes.Equal(got, want) {
t.Errorf("got %x, want %x", got, want)
}
})
}
}
func TestRFC6962HasherCollisions(t *testing.T) {
// Check that different leaves have different hashes.
leaf1, leaf2 := []byte("Hello"), []byte("World")
_, leafHashTrail := trailsFromByteSlices([][]byte{leaf1})
hash1 := leafHashTrail.Hash
_, leafHashTrail = trailsFromByteSlices([][]byte{leaf2})
hash2 := leafHashTrail.Hash
if bytes.Equal(hash1, hash2) {
t.Errorf("Leaf hashes should differ, but both are %x", hash1)
}
// Compute an intermediate subtree hash.
_, subHash1Trail := trailsFromByteSlices([][]byte{hash1, hash2})
subHash1 := subHash1Trail.Hash
// Check that this is not the same as a leaf hash of their concatenation.
preimage := append(hash1, hash2...)
_, forgedHashTrail := trailsFromByteSlices([][]byte{preimage})
forgedHash := forgedHashTrail.Hash
if bytes.Equal(subHash1, forgedHash) {
t.Errorf("Hasher is not second-preimage resistant")
}
// Swap the order of nodes and check that the hash is different.
_, subHash2Trail := trailsFromByteSlices([][]byte{hash2, hash1})
subHash2 := subHash2Trail.Hash
if bytes.Equal(subHash1, subHash2) {
t.Errorf("Subtree hash does not depend on the order of leaves")
}
}

View File

@@ -1,7 +1,6 @@
package merkle
import (
"github.com/tendermint/tendermint/crypto/tmhash"
cmn "github.com/tendermint/tendermint/libs/common"
)
@@ -35,7 +34,7 @@ func (sm *simpleMap) Set(key string, value Hasher) {
})
}
// Hash Merkle root hash of items sorted by key
// Hash computes the Merkle root hash of items sorted by key
// (UNSTABLE: and by value too if duplicate key).
func (sm *simpleMap) Hash() []byte {
sm.Sort()
@@ -66,23 +65,17 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs {
// then hashed.
type KVPair cmn.KVPair
func (kv KVPair) Hash() []byte {
hasher := tmhash.New()
err := encodeByteSlice(hasher, kv.Key)
if err != nil {
panic(err)
}
err = encodeByteSlice(hasher, kv.Value)
if err != nil {
panic(err)
}
return hasher.Sum(nil)
func (kv KVPair) Bytes() []byte {
val := make([]byte, len(kv.Key)+len(kv.Value))
copy(val, kv.Key)
copy(val[len(kv.Key):], kv.Value)
return val
}
func hashKVPairs(kvs cmn.KVPairs) []byte {
kvsH := make([]Hasher, len(kvs))
kvsH := make([][]byte, len(kvs))
for i, kvp := range kvs {
kvsH[i] = KVPair(kvp)
kvsH[i] = KVPair(kvp).Bytes()
}
return SimpleHashFromHashers(kvsH)
return SimpleHashFromByteSlices(kvsH)
}

View File

@@ -18,37 +18,37 @@ func TestSimpleMap(t *testing.T) {
{
db := newSimpleMap()
db.Set("key1", strHasher("value1"))
assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
assert.Equal(t, "5e0c721faaa2e92bdb37e7d7297d6affb3bac87d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key1", strHasher("value2"))
assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
assert.Equal(t, "968fdd7e6e94448b0ab5d38cd9b3618b860e2443", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key1", strHasher("value1"))
db.Set("key2", strHasher("value2"))
assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
assert.Equal(t, "7dc11c8e1bc6fe0aaa0d691560c25c3f33940e14", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key2", strHasher("value2")) // NOTE: out of order
db.Set("key1", strHasher("value1"))
assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
assert.Equal(t, "7dc11c8e1bc6fe0aaa0d691560c25c3f33940e14", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key1", strHasher("value1"))
db.Set("key2", strHasher("value2"))
db.Set("key3", strHasher("value3"))
assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
assert.Equal(t, "2584d8b484db08cd807b931e0b95f64a9105d41d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key2", strHasher("value2")) // NOTE: out of order
db.Set("key1", strHasher("value1"))
db.Set("key3", strHasher("value3"))
assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
assert.Equal(t, "2584d8b484db08cd807b931e0b95f64a9105d41d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
}

View File

@@ -3,6 +3,8 @@ package merkle
import (
"bytes"
"fmt"
"github.com/tendermint/tendermint/crypto/tmhash"
)
// SimpleProof represents a simple merkle proof.
@@ -10,10 +12,10 @@ type SimpleProof struct {
Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child.
}
// SimpleProofsFromHashers computes inclusion proof for given items.
// SimpleProofsFromByteSlices computes inclusion proof for given items.
// proofs[0] is the proof for items[0].
func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) {
trails, rootSPN := trailsFromHashers(items)
func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*SimpleProof) {
trails, rootSPN := trailsFromByteSlices(items)
rootHash = rootSPN.Hash
proofs = make([]*SimpleProof, len(items))
for i, trail := range trails {
@@ -34,12 +36,12 @@ func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[strin
}
sm.Sort()
kvs := sm.kvs
kvsH := make([]Hasher, 0, len(kvs))
kvsB := make([][]byte, 0, len(kvs))
for _, kvp := range kvs {
kvsH = append(kvsH, KVPair(kvp))
kvsB = append(kvsB, KVPair(kvp).Bytes())
}
rootHash, proofList := SimpleProofsFromHashers(kvsH)
rootHash, proofList := SimpleProofsFromByteSlices(kvsB)
proofs = make(map[string]*SimpleProof)
keys = make([]string, len(proofList))
for i, kvp := range kvs {
@@ -49,9 +51,16 @@ func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[strin
return
}
// Verify that leafHash is a leaf hash of the simple-merkle-tree
// Verify that leaf is a leaf hash of the simple-merkle-tree
// which hashes to rootHash.
func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool {
func (sp *SimpleProof) Verify(index int, total int, leaf []byte, rootHash []byte) bool {
leafHash := tmhash.Sum(append([]byte{0}, leaf...))
return sp.VerifyHash(index, total, leafHash, rootHash)
}
// VerifyHash that leafHash is a leaf hash of the simple-merkle-tree
// which hashes to rootHash.
func (sp *SimpleProof) VerifyHash(index int, total int, leafHash []byte, rootHash []byte) bool {
computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts)
return computedHash != nil && bytes.Equal(computedHash, rootHash)
}
@@ -90,7 +99,7 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][
if len(innerHashes) == 0 {
return nil
}
numLeft := (total + 1) / 2
numLeft := getSplitPoint(total)
if index < numLeft {
leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1])
if leftHash == nil {
@@ -138,17 +147,19 @@ func (spn *SimpleProofNode) FlattenAunts() [][]byte {
// trails[0].Hash is the leaf hash for items[0].
// trails[i].Parent.Parent....Parent == root for all i.
func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) {
func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *SimpleProofNode) {
// Recursive impl.
switch len(items) {
case 0:
return nil, nil
case 1:
trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil}
hash0 := tmhash.Sum(append(LeafHashPrefix, items[0]...))
trail := &SimpleProofNode{hash0, nil, nil, nil}
return []*SimpleProofNode{trail}, trail
default:
lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2])
rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:])
k := getSplitPoint(len(items))
lefts, leftRoot := trailsFromByteSlices(items[:k])
rights, rightRoot := trailsFromByteSlices(items[k:])
rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash)
root := &SimpleProofNode{rootHash, nil, nil, nil}
leftRoot.Parent = root

View File

@@ -1,28 +1,34 @@
package merkle
import (
"math/bits"
"github.com/tendermint/tendermint/crypto/tmhash"
)
// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right).
func SimpleHashFromTwoHashes(left, right []byte) []byte {
var hasher = tmhash.New()
err := encodeByteSlice(hasher, left)
_, err := hasher.Write(InnerHashPrefix)
if err != nil {
panic(err)
}
err = encodeByteSlice(hasher, right)
_, err = hasher.Write(left)
if err != nil {
panic(err)
}
_, err = hasher.Write(right)
if err != nil {
panic(err)
}
return hasher.Sum(nil)
}
// SimpleHashFromHashers computes a Merkle tree from items that can be hashed.
func SimpleHashFromHashers(items []Hasher) []byte {
// SimpleHashFromByteSlices computes a Merkle tree from items that are byte slices.
func SimpleHashFromByteSlices(items [][]byte) []byte {
hashes := make([][]byte, len(items))
for i, item := range items {
hash := item.Hash()
hash := tmhash.Sum(append([]byte{0}, item...))
hashes[i] = hash
}
return simpleHashFromHashes(hashes)
@@ -51,8 +57,22 @@ func simpleHashFromHashes(hashes [][]byte) []byte {
case 1:
return hashes[0]
default:
left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2])
right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:])
k := getSplitPoint(len(hashes))
left := simpleHashFromHashes(hashes[:k])
right := simpleHashFromHashes(hashes[k:])
return SimpleHashFromTwoHashes(left, right)
}
}
func getSplitPoint(length int) int {
if length < 1 {
panic("Trying to split a tree with size < 1")
}
uLength := uint(length)
bitlen := bits.Len(uLength)
k := 1 << uint(bitlen-1)
if k == length {
k >>= 1
}
return k
}

View File

@@ -2,33 +2,29 @@ package merkle
import (
"bytes"
"crypto/rand"
"testing"
"github.com/stretchr/testify/require"
cmn "github.com/tendermint/tendermint/libs/common"
. "github.com/tendermint/tendermint/libs/test"
"testing"
"github.com/tendermint/tendermint/crypto/tmhash"
)
type testItem []byte
func (tI testItem) Hash() []byte {
return []byte(tI)
}
func TestSimpleProof(t *testing.T) {
total := 100
sliceSize := 100
items := make([]Hasher, total)
items := make([][]byte, total)
for i := 0; i < total; i++ {
items[i] = testItem(cmn.RandBytes(tmhash.Size))
items[i] = make([]byte, sliceSize)
rand.Read(items[i])
}
rootHash := SimpleHashFromHashers(items)
rootHash := SimpleHashFromByteSlices(items)
rootHash2, proofs := SimpleProofsFromHashers(items)
rootHash2, proofs := SimpleProofsFromByteSlices(items)
if !bytes.Equal(rootHash, rootHash2) {
t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2)
@@ -36,18 +32,17 @@ func TestSimpleProof(t *testing.T) {
// For each item, check the trail.
for i, item := range items {
itemHash := item.Hash()
proof := proofs[i]
// Verify success
ok := proof.Verify(i, total, itemHash, rootHash)
ok := proof.Verify(i, total, item, rootHash)
if !ok {
t.Errorf("Verification failed for index %v.", i)
}
// Wrong item index should make it fail
{
ok = proof.Verify((i+1)%total, total, itemHash, rootHash)
ok = proof.Verify((i+1)%total, total, item, rootHash)
if ok {
t.Errorf("Expected verification to fail for wrong index %v.", i)
}
@@ -57,7 +52,7 @@ func TestSimpleProof(t *testing.T) {
origAunts := proof.Aunts
proof.Aunts = append(proof.Aunts, cmn.RandBytes(32))
{
ok = proof.Verify(i, total, itemHash, rootHash)
ok = proof.Verify(i, total, item, rootHash)
if ok {
t.Errorf("Expected verification to fail for wrong trail length.")
}
@@ -67,7 +62,7 @@ func TestSimpleProof(t *testing.T) {
// Trail too short should make it fail
proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1]
{
ok = proof.Verify(i, total, itemHash, rootHash)
ok = proof.Verify(i, total, item, rootHash)
if ok {
t.Errorf("Expected verification to fail for wrong trail length.")
}
@@ -75,15 +70,38 @@ func TestSimpleProof(t *testing.T) {
proof.Aunts = origAunts
// Mutating the itemHash should make it fail.
ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash)
ok = proof.Verify(i, total, MutateByteSlice(item), rootHash)
if ok {
t.Errorf("Expected verification to fail for mutated leaf hash")
}
// Mutating the rootHash should make it fail.
ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash))
ok = proof.Verify(i, total, item, MutateByteSlice(rootHash))
if ok {
t.Errorf("Expected verification to fail for mutated root hash")
}
}
}
func Test_getSplitPoint(t *testing.T) {
tests := []struct {
length int
want int
}{
{1, 0},
{2, 1},
{3, 2},
{4, 2},
{5, 4},
{10, 8},
{20, 16},
{100, 64},
{255, 128},
{256, 128},
{257, 256},
}
for _, tt := range tests {
got := getSplitPoint(tt.length)
require.Equal(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want)
}
}

View File

@@ -1,9 +1,8 @@
package merkle
import (
"io"
amino "github.com/tendermint/go-amino"
var (
LeafHashPrefix = []byte{0}
InnerHashPrefix = []byte{1}
)
// Tree is a Merkle tree interface.
@@ -29,10 +28,3 @@ type Tree interface {
type Hasher interface {
Hash() []byte
}
//-----------------------------------------------------------------------
// Uvarint length prefixed byteslice
func encodeByteSlice(w io.Writer, bz []byte) (err error) {
return amino.EncodeByteSlice(w, bz)
}

View File

@@ -419,11 +419,11 @@ func (commit *Commit) Hash() cmn.HexBytes {
return nil
}
if commit.hash == nil {
bs := make([]merkle.Hasher, len(commit.Precommits))
bzs := make([][]byte, len(commit.Precommits))
for i, precommit := range commit.Precommits {
bs[i] = aminoHasher(precommit)
bzs[i] = cdc.MustMarshalBinaryBare(precommit)
}
commit.hash = merkle.SimpleHashFromHashers(bs)
commit.hash = merkle.SimpleHashFromByteSlices(bzs)
}
return commit.hash
}
@@ -638,11 +638,8 @@ type hasher struct {
func (h hasher) Hash() []byte {
hasher := tmhash.New()
if h.item != nil && !cmn.IsTypedNil(h.item) && !cmn.IsEmpty(h.item) {
bz, err := cdc.MarshalBinaryBare(h.item)
if err != nil {
panic(err)
}
_, err = hasher.Write(bz)
bz := cdc.MustMarshalBinaryBare(h.item)
_, err := hasher.Write(bz)
if err != nil {
panic(err)
}

View File

@@ -88,7 +88,7 @@ func NewPartSetFromData(data []byte, partSize int) *PartSet {
// divide data into 4kb parts.
total := (len(data) + partSize - 1) / partSize
parts := make([]*Part, total)
parts_ := make([]merkle.Hasher, total)
parts_ := make([][]byte, total)
partsBitArray := cmn.NewBitArray(total)
for i := 0; i < total; i++ {
part := &Part{
@@ -96,11 +96,11 @@ func NewPartSetFromData(data []byte, partSize int) *PartSet {
Bytes: data[i*partSize : cmn.MinInt(len(data), (i+1)*partSize)],
}
parts[i] = part
parts_[i] = part
parts_[i] = part.Bytes
partsBitArray.SetIndex(i, true)
}
// Compute merkle proofs
root, proofs := merkle.SimpleProofsFromHashers(parts_)
root, proofs := merkle.SimpleProofsFromByteSlices(parts_)
for i := 0; i < total; i++ {
parts[i].Proof = *proofs[i]
}

View File

@@ -54,20 +54,20 @@ func (a ABCIResults) Bytes() []byte {
func (a ABCIResults) Hash() []byte {
// NOTE: we copy the impl of the merkle tree for txs -
// we should be consistent and either do it for both or not.
return merkle.SimpleHashFromHashers(a.toHashers())
return merkle.SimpleHashFromByteSlices(a.toByteSlices())
}
// ProveResult returns a merkle proof of one result from the set
func (a ABCIResults) ProveResult(i int) merkle.SimpleProof {
_, proofs := merkle.SimpleProofsFromHashers(a.toHashers())
_, proofs := merkle.SimpleProofsFromByteSlices(a.toByteSlices())
return *proofs[i]
}
func (a ABCIResults) toHashers() []merkle.Hasher {
func (a ABCIResults) toByteSlices() [][]byte {
l := len(a)
hashers := make([]merkle.Hasher, l)
byteslices := make([][]byte, l)
for i := 0; i < l; i++ {
hashers[i] = a[i]
byteslices[i] = cdc.MustMarshalBinaryBare(a[i])
}
return hashers
return byteslices
}

View File

@@ -70,11 +70,11 @@ func (txs Txs) IndexByHash(hash []byte) int {
// TODO: optimize this!
func (txs Txs) Proof(i int) TxProof {
l := len(txs)
hashers := make([]merkle.Hasher, l)
bzs := make([][]byte, l)
for i := 0; i < l; i++ {
hashers[i] = txs[i]
bzs[i] = txs[i]
}
root, proofs := merkle.SimpleProofsFromHashers(hashers)
root, proofs := merkle.SimpleProofsFromByteSlices(bzs)
return TxProof{
Index: i,

View File

@@ -82,6 +82,20 @@ func (v *Validator) Hash() []byte {
})
}
// hashBytes computes the bytes to be hashed of a validator with a given voting power.
// It excludes the Accum value, which changes with every round.
func (v *Validator) hashBytes() []byte {
return cdc.MustMarshalBinaryBare(struct {
Address Address
PubKey crypto.PubKey
VotingPower int64
}{
v.Address,
v.PubKey,
v.VotingPower,
})
}
//----------------------------------------
// RandValidator

View File

@@ -176,11 +176,11 @@ func (vals *ValidatorSet) Hash() []byte {
if len(vals.Validators) == 0 {
return nil
}
hashers := make([]merkle.Hasher, len(vals.Validators))
bzs := make([][]byte, len(vals.Validators))
for i, val := range vals.Validators {
hashers[i] = val
bzs[i] = val.hashBytes()
}
return merkle.SimpleHashFromHashers(hashers)
return merkle.SimpleHashFromByteSlices(bzs)
}
// Add adds val to the validator set and returns true. It returns false if val