fluencelabs/tendermint
1
0
Fork 0
mirror of https://github.com/fluencelabs/tendermint synced 2025-04-24 22:32:15 +00:00
Code Issues Projects Releases Wiki Activity

crypto/merkle: Remove byter in favor of plain byte slices (#2595)

Browse Source 
* crypto/merkle: Remove byter in favor of plain byte slices

This PR is fully backwards compatible in terms of function output!
(The Go API differs though) The only test case changes was to refactor
it to be table driven.

* Update godocs per review comments
This commit is contained in:
Dev Ojha 2018-10-10 09:46:09 -07:00 committed by Ethan Buchman
parent 92343ef484
commit 12fa9d1cab
16 changed files with 134 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG_PENDING.md
View File

@ -23,6 +23,7 @@ BREAKING CHANGES:
* [rpc/client] \#2298 `ABCIQueryOptions.Trusted` -> `ABCIQueryOptions.Prove`
* [types] \#2298 Remove `Index` and `Total` fields from `TxProof`.
* [crypto/merkle & lite] \#2298 Various changes to accomodate General Merkle trees
* [crypto/merkle] \#2595 Remove all Hasher objects in favor of byte slices
* Blockchain Protocol
* [types] \#2459 `Vote`/`Proposal`/`Heartbeat` use amino encoding instead of JSON in `SignBytes`.

28
crypto/merkle/simple_map.go
View File

@ -1,6 +1,9 @@
package merkle
import (
"bytes"
amino "github.com/tendermint/go-amino"
"github.com/tendermint/tendermint/crypto/tmhash"
cmn "github.com/tendermint/tendermint/libs/common"
)
@ -20,14 +23,15 @@ func newSimpleMap() *simpleMap {
}
}
// Set hashes the key and value and appends it to the kv pairs.
func (sm *simpleMap) Set(key string, value Hasher) {
// Set creates a kv pair of the key and the hash of the value,
// and then appends it to simpleMap's kv pairs.
func (sm *simpleMap) Set(key string, value []byte) {
sm.sorted = false
// The value is hashed, so you can
// check for equality with a cached value (say)
// and make a determination to fetch or not.
vhash := value.Hash()
vhash := tmhash.Sum(value)
sm.kvs = append(sm.kvs, cmn.KVPair{
Key: []byte(key),
@ -66,23 +70,25 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs {
// then hashed.
type KVPair cmn.KVPair
func (kv KVPair) Hash() []byte {
hasher := tmhash.New()
err := encodeByteSlice(hasher, kv.Key)
// Bytes returns key || value, with both the
// key and value length prefixed.
func (kv KVPair) Bytes() []byte {
var b bytes.Buffer
err := amino.EncodeByteSlice(&b, kv.Key)
if err != nil {
panic(err)
}
err = encodeByteSlice(hasher, kv.Value)
err = amino.EncodeByteSlice(&b, kv.Value)
if err != nil {
panic(err)
}
return hasher.Sum(nil)
return b.Bytes()
}
func hashKVPairs(kvs cmn.KVPairs) []byte {
kvsH := make([]Hasher, len(kvs))
kvsH := make([][]byte, len(kvs))
for i, kvp := range kvs {
kvsH[i] = KVPair(kvp)
kvsH[i] = KVPair(kvp).Bytes()
}
return SimpleHashFromHashers(kvsH)
return SimpleHashFromByteSlices(kvsH)
}

59
crypto/merkle/simple_map_test.go
View File

@ -5,50 +5,29 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/tendermint/tendermint/crypto/tmhash"
)
type strHasher string
func (str strHasher) Hash() []byte {
return tmhash.Sum([]byte(str))
}
func TestSimpleMap(t *testing.T) {
{
db := newSimpleMap()
db.Set("key1", strHasher("value1"))
assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
tests := []struct {
keys []string
values []string // each string gets converted to []byte in test
want string
}{
{[]string{"key1"}, []string{"value1"}, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f"},
{[]string{"key1"}, []string{"value2"}, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8"},
// swap order with 2 keys
{[]string{"key1", "key2"}, []string{"value1", "value2"}, "eff12d1c703a1022ab509287c0f196130123d786"},
{[]string{"key2", "key1"}, []string{"value2", "value1"}, "eff12d1c703a1022ab509287c0f196130123d786"},
// swap order with 3 keys
{[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"},
{[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"},
}
{
for i, tc := range tests {
db := newSimpleMap()
db.Set("key1", strHasher("value2"))
assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key1", strHasher("value1"))
db.Set("key2", strHasher("value2"))
assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key2", strHasher("value2")) // NOTE: out of order
db.Set("key1", strHasher("value1"))
assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key1", strHasher("value1"))
db.Set("key2", strHasher("value2"))
db.Set("key3", strHasher("value3"))
assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
}
{
db := newSimpleMap()
db.Set("key2", strHasher("value2")) // NOTE: out of order
db.Set("key1", strHasher("value1"))
db.Set("key3", strHasher("value3"))
assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match")
for i := 0; i < len(tc.keys); i++ {
db.Set(tc.keys[i], []byte(tc.values[i]))
}
got := db.Hash()
assert.Equal(t, tc.want, fmt.Sprintf("%x", got), "Hash didn't match on tc %d", i)
}
}

25
crypto/merkle/simple_proof.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"github.com/tendermint/tendermint/crypto/tmhash"
cmn "github.com/tendermint/tendermint/libs/common"
)
@ -22,10 +23,10 @@ type SimpleProof struct {
Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child.
}
// SimpleProofsFromHashers computes inclusion proof for given items.
// SimpleProofsFromByteSlices computes inclusion proof for given items.
// proofs[0] is the proof for items[0].
func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) {
trails, rootSPN := trailsFromHashers(items)
func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*SimpleProof) {
trails, rootSPN := trailsFromByteSlices(items)
rootHash = rootSPN.Hash
proofs = make([]*SimpleProof, len(items))
for i, trail := range trails {
@ -42,19 +43,19 @@ func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleP
// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values
// in the underlying key-value pairs.
// The keys are sorted before the proofs are computed.
func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) {
func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) {
sm := newSimpleMap()
for k, v := range m {
sm.Set(k, v)
}
sm.Sort()
kvs := sm.kvs
kvsH := make([]Hasher, 0, len(kvs))
for _, kvp := range kvs {
kvsH = append(kvsH, KVPair(kvp))
kvsBytes := make([][]byte, len(kvs))
for i, kvp := range kvs {
kvsBytes[i] = KVPair(kvp).Bytes()
}
rootHash, proofList := SimpleProofsFromHashers(kvsH)
rootHash, proofList := SimpleProofsFromByteSlices(kvsBytes)
proofs = make(map[string]*SimpleProof)
keys = make([]string, len(proofList))
for i, kvp := range kvs {
@ -175,17 +176,17 @@ func (spn *SimpleProofNode) FlattenAunts() [][]byte {
// trails[0].Hash is the leaf hash for items[0].
// trails[i].Parent.Parent....Parent == root for all i.
func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) {
func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *SimpleProofNode) {
// Recursive impl.
switch len(items) {
case 0:
return nil, nil
case 1:
trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil}
trail := &SimpleProofNode{tmhash.Sum(items[0]), nil, nil, nil}
return []*SimpleProofNode{trail}, trail
default:
lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2])
rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:])
lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2])
rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:])
rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash)
root := &SimpleProofNode{rootHash, nil, nil, nil}
leftRoot.Parent = root

9
crypto/merkle/simple_tree.go
View File

@ -18,11 +18,12 @@ func SimpleHashFromTwoHashes(left, right []byte) []byte {
return hasher.Sum(nil)
}
// SimpleHashFromHashers computes a Merkle tree from items that can be hashed.
func SimpleHashFromHashers(items []Hasher) []byte {
// SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice,
// in the provided order.
func SimpleHashFromByteSlices(items [][]byte) []byte {
hashes := make([][]byte, len(items))
for i, item := range items {
hash := item.Hash()
hash := tmhash.Sum(item)
hashes[i] = hash
}
return simpleHashFromHashes(hashes)
@ -32,7 +33,7 @@ func SimpleHashFromHashers(items []Hasher) []byte {
// Like calling SimpleHashFromHashers with
// `item = []byte(Hash(key) | Hash(value))`,
// sorted by `item`.
func SimpleHashFromMap(m map[string]Hasher) []byte {
func SimpleHashFromMap(m map[string][]byte) []byte {
sm := newSimpleMap()
for k, v := range m {
sm.Set(k, v)

8
crypto/merkle/simple_tree_test.go
View File

@ -21,20 +21,20 @@ func TestSimpleProof(t *testing.T) {
total := 100
items := make([]Hasher, total)
items := make([][]byte, total)
for i := 0; i < total; i++ {
items[i] = testItem(cmn.RandBytes(tmhash.Size))
}
rootHash := SimpleHashFromHashers(items)
rootHash := SimpleHashFromByteSlices(items)
rootHash2, proofs := SimpleProofsFromHashers(items)
rootHash2, proofs := SimpleProofsFromByteSlices(items)
require.Equal(t, rootHash, rootHash2, "Unmatched root hashes: %X vs %X", rootHash, rootHash2)
// For each item, check the trail.
for i, item := range items {
itemHash := item.Hash()
itemHash := tmhash.Sum(item)
proof := proofs[i]
// Check total/index

5
crypto/merkle/types.go
View File

@ -25,11 +25,6 @@ type Tree interface {
IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool)
}
// Hasher represents a hashable piece of data which can be hashed in the Tree.
type Hasher interface {
Hash() []byte
}
//-----------------------------------------------------------------------
// Uvarint length prefixed byteslice

69
types/block.go
View File

@ -9,7 +9,6 @@ import (
"time"
"github.com/tendermint/tendermint/crypto/merkle"
"github.com/tendermint/tendermint/crypto/tmhash"
cmn "github.com/tendermint/tendermint/libs/common"
)
@ -290,22 +289,22 @@ func (h *Header) Hash() cmn.HexBytes {
if h == nil || len(h.ValidatorsHash) == 0 {
return nil
}
return merkle.SimpleHashFromMap(map[string]merkle.Hasher{
"ChainID": aminoHasher(h.ChainID),
"Height": aminoHasher(h.Height),
"Time": aminoHasher(h.Time),
"NumTxs": aminoHasher(h.NumTxs),
"TotalTxs": aminoHasher(h.TotalTxs),
"LastBlockID": aminoHasher(h.LastBlockID),
"LastCommit": aminoHasher(h.LastCommitHash),
"Data": aminoHasher(h.DataHash),
"Validators": aminoHasher(h.ValidatorsHash),
"NextValidators": aminoHasher(h.NextValidatorsHash),
"App": aminoHasher(h.AppHash),
"Consensus": aminoHasher(h.ConsensusHash),
"Results": aminoHasher(h.LastResultsHash),
"Evidence": aminoHasher(h.EvidenceHash),
"Proposer": aminoHasher(h.ProposerAddress),
return merkle.SimpleHashFromMap(map[string][]byte{
"ChainID": cdcEncode(h.ChainID),
"Height": cdcEncode(h.Height),
"Time": cdcEncode(h.Time),
"NumTxs": cdcEncode(h.NumTxs),
"TotalTxs": cdcEncode(h.TotalTxs),
"LastBlockID": cdcEncode(h.LastBlockID),
"LastCommit": cdcEncode(h.LastCommitHash),
"Data": cdcEncode(h.DataHash),
"Validators": cdcEncode(h.ValidatorsHash),
"NextValidators": cdcEncode(h.NextValidatorsHash),
"App": cdcEncode(h.AppHash),
"Consensus": cdcEncode(h.ConsensusHash),
"Results": cdcEncode(h.LastResultsHash),
"Evidence": cdcEncode(h.EvidenceHash),
"Proposer": cdcEncode(h.ProposerAddress),
})
}
@ -480,11 +479,11 @@ func (commit *Commit) Hash() cmn.HexBytes {
return nil
}
if commit.hash == nil {
bs := make([]merkle.Hasher, len(commit.Precommits))
bs := make([][]byte, len(commit.Precommits))
for i, precommit := range commit.Precommits {
bs[i] = aminoHasher(precommit)
bs[i] = cdcEncode(precommit)
}
commit.hash = merkle.SimpleHashFromHashers(bs)
commit.hash = merkle.SimpleHashFromByteSlices(bs)
}
return commit.hash
}
@ -689,33 +688,3 @@ func (blockID BlockID) Key() string {
func (blockID BlockID) String() string {
return fmt.Sprintf(`%v:%v`, blockID.Hash, blockID.PartsHeader)
}
//-------------------------------------------------------
type hasher struct {
item interface{}
}
func (h hasher) Hash() []byte {
hasher := tmhash.New()
if h.item != nil && !cmn.IsTypedNil(h.item) && !cmn.IsEmpty(h.item) {
bz, err := cdc.MarshalBinaryBare(h.item)
if err != nil {
panic(err)
}
_, err = hasher.Write(bz)
if err != nil {
panic(err)
}
}
return hasher.Sum(nil)
}
func aminoHash(item interface{}) []byte {
h := hasher{item}
return h.Hash()
}
func aminoHasher(item interface{}) merkle.Hasher {
return hasher{item}
}

14
types/encoding_helper.go Normal file
View File

@ -0,0 +1,14 @@
package types
import (
cmn "github.com/tendermint/tendermint/libs/common"
)
// cdcEncode returns nil if the input is nil, otherwise returns
// cdc.MustMarshalBinaryBare(item)
func cdcEncode(item interface{}) []byte {
if item != nil && !cmn.IsTypedNil(item) && !cmn.IsEmpty(item) {
return cdc.MustMarshalBinaryBare(item)
}
return nil
}

8
types/evidence.go
View File

@ -4,6 +4,8 @@ import (
"bytes"
"fmt"
"github.com/tendermint/tendermint/crypto/tmhash"
amino "github.com/tendermint/go-amino"
"github.com/tendermint/tendermint/crypto"
@ -104,7 +106,7 @@ func (dve *DuplicateVoteEvidence) Address() []byte {
// Hash returns the hash of the evidence.
func (dve *DuplicateVoteEvidence) Hash() []byte {
return aminoHasher(dve).Hash()
return tmhash.Sum(cdcEncode(dve))
}
// Verify returns an error if the two votes aren't conflicting.
@ -157,8 +159,8 @@ func (dve *DuplicateVoteEvidence) Equal(ev Evidence) bool {
}
// just check their hashes
dveHash := aminoHasher(dve).Hash()
evHash := aminoHasher(ev).Hash()
dveHash := tmhash.Sum(cdcEncode(dve))
evHash := tmhash.Sum(cdcEncode(ev))
return bytes.Equal(dveHash, evHash)
}

8
types/params.go
View File

@ -82,10 +82,10 @@ func (params *ConsensusParams) Validate() error {
// Hash returns a merkle hash of the parameters to store in the block header
func (params *ConsensusParams) Hash() []byte {
return merkle.SimpleHashFromMap(map[string]merkle.Hasher{
"block_size_max_bytes": aminoHasher(params.BlockSize.MaxBytes),
"block_size_max_gas": aminoHasher(params.BlockSize.MaxGas),
"evidence_params_max_age": aminoHasher(params.EvidenceParams.MaxAge),
return merkle.SimpleHashFromMap(map[string][]byte{
"block_size_max_bytes": cdcEncode(params.BlockSize.MaxBytes),
"block_size_max_gas": cdcEncode(params.BlockSize.MaxGas),
"evidence_params_max_age": cdcEncode(params.EvidenceParams.MaxAge),
})
}

6
types/part_set.go
View File

@ -88,7 +88,7 @@ func NewPartSetFromData(data []byte, partSize int) *PartSet {
// divide data into 4kb parts.
total := (len(data) + partSize - 1) / partSize
parts := make([]*Part, total)
parts_ := make([]merkle.Hasher, total)
partsBytes := make([][]byte, total)
partsBitArray := cmn.NewBitArray(total)
for i := 0; i < total; i++ {
part := &Part{
@ -96,11 +96,11 @@ func NewPartSetFromData(data []byte, partSize int) *PartSet {
Bytes: data[i*partSize : cmn.MinInt(len(data), (i+1)*partSize)],
}
parts[i] = part
parts_[i] = part
partsBytes[i] = part.Bytes
partsBitArray.SetIndex(i, true)
}
// Compute merkle proofs
root, proofs := merkle.SimpleProofsFromHashers(parts_)
root, proofs := merkle.SimpleProofsFromByteSlices(partsBytes)
for i := 0; i < total; i++ {
parts[i].Proof = *proofs[i]
}

19
types/results.go
View File

@ -3,6 +3,7 @@ package types
import (
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/crypto/merkle"
"github.com/tendermint/tendermint/crypto/tmhash"
cmn "github.com/tendermint/tendermint/libs/common"
)
@ -17,10 +18,14 @@ type ABCIResult struct {
// Hash returns the canonical hash of the ABCIResult
func (a ABCIResult) Hash() []byte {
bz := aminoHash(a)
bz := tmhash.Sum(cdcEncode(a))
return bz
}
func (a ABCIResult) Bytes() []byte {
return cdcEncode(a)
}
// ABCIResults wraps the deliver tx results to return a proof
type ABCIResults []ABCIResult
@ -54,20 +59,20 @@ func (a ABCIResults) Bytes() []byte {
func (a ABCIResults) Hash() []byte {
// NOTE: we copy the impl of the merkle tree for txs -
// we should be consistent and either do it for both or not.
return merkle.SimpleHashFromHashers(a.toHashers())
return merkle.SimpleHashFromByteSlices(a.toByteSlices())
}
// ProveResult returns a merkle proof of one result from the set
func (a ABCIResults) ProveResult(i int) merkle.SimpleProof {
_, proofs := merkle.SimpleProofsFromHashers(a.toHashers())
_, proofs := merkle.SimpleProofsFromByteSlices(a.toByteSlices())
return *proofs[i]
}
func (a ABCIResults) toHashers() []merkle.Hasher {
func (a ABCIResults) toByteSlices() [][]byte {
l := len(a)
hashers := make([]merkle.Hasher, l)
bzs := make([][]byte, l)
for i := 0; i < l; i++ {
hashers[i] = a[i]
bzs[i] = a[i].Bytes()
}
return hashers
return bzs
}

6
types/tx.go
View File

@ -70,11 +70,11 @@ func (txs Txs) IndexByHash(hash []byte) int {
// TODO: optimize this!
func (txs Txs) Proof(i int) TxProof {
l := len(txs)
hashers := make([]merkle.Hasher, l)
bzs := make([][]byte, l)
for i := 0; i < l; i++ {
hashers[i] = txs[i]
bzs[i] = txs[i]
}
root, proofs := merkle.SimpleProofsFromHashers(hashers)
root, proofs := merkle.SimpleProofsFromByteSlices(bzs)
return TxProof{
RootHash: root,

14
types/validator.go
View File

@ -4,6 +4,8 @@ import (
"bytes"
"fmt"
"github.com/tendermint/tendermint/crypto/tmhash"
"github.com/tendermint/tendermint/crypto"
cmn "github.com/tendermint/tendermint/libs/common"
)
@ -71,13 +73,21 @@ func (v *Validator) String() string {
// Hash computes the unique ID of a validator with a given voting power.
// It excludes the Accum value, which changes with every round.
func (v *Validator) Hash() []byte {
return aminoHash(struct {
return tmhash.Sum(v.Bytes())
}
// Bytes computes the unique encoding of a validator with a given voting power.
// These are the bytes that gets hashed in consensus. It excludes pubkey
// as its redundant with the address. This also excludes accum which changes
// every round.
func (v *Validator) Bytes() []byte {
return cdcEncode((struct {
Address Address
VotingPower int64
}{
v.Address,
v.VotingPower,
})
}))
}
//----------------------------------------

6
types/validator_set.go
View File

@ -176,11 +176,11 @@ func (vals *ValidatorSet) Hash() []byte {
if len(vals.Validators) == 0 {
return nil
}
hashers := make([]merkle.Hasher, len(vals.Validators))
bzs := make([][]byte, len(vals.Validators))
for i, val := range vals.Validators {
hashers[i] = val
bzs[i] = val.Bytes()
}
return merkle.SimpleHashFromHashers(hashers)
return merkle.SimpleHashFromByteSlices(bzs)
}
// Add adds val to the validator set and returns true. It returns false if val