From 7928659f70511deab1c89a0eaacf127f916a075f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 9 Jul 2017 14:10:00 -0400 Subject: [PATCH 01/35] track evidence, include in block --- consensus/state.go | 15 ++++++++----- state/execution.go | 2 ++ types/block.go | 50 +++++++++++++++++++++++++++++++++++++++++++- types/evidence.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 types/evidence.go diff --git a/consensus/state.go b/consensus/state.go index 67a1d821..0241cbd9 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -863,7 +863,9 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts // Mempool validated transactions txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) - return cs.state.MakeBlock(cs.Height, txs, commit) + block, parts := cs.state.MakeBlock(cs.Height, txs, commit) + block.AddEvidence(cs.Evidence) + return block, parts } // Enter: `timeoutPropose` after entering Propose. @@ -1231,6 +1233,10 @@ func (cs *ConsensusState) finalizeCommit(height int64) { fail.Fail() // XXX + // TODO: remove included evidence + // and persist remaining evidence + // ... is this the right spot? need to ensure we never lose evidence + // NewHeightStep! cs.updateToState(stateCopy) @@ -1327,15 +1333,14 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { // If it's otherwise invalid, punish peer. if err == ErrVoteHeightMismatch { return err - } else if _, ok := err.(*types.ErrVoteConflictingVotes); ok { + } else if voteErr, ok := err.(*types.ErrVoteConflictingVotes); ok { if bytes.Equal(vote.ValidatorAddress, cs.privValidator.GetAddress()) { cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return err } - cs.Logger.Error("Found conflicting vote. Publish evidence (TODO)", "height", vote.Height, "round", vote.Round, "type", vote.Type, "valAddr", vote.ValidatorAddress, "valIndex", vote.ValidatorIndex) - - // TODO: track evidence for inclusion in a block + cs.Logger.Error("Found conflicting vote. Recording evidence in the RoundState", "height", vote.Height, "round", vote.Round, "type", vote.Type, "valAddr", vote.ValidatorAddress, "valIndex", vote.ValidatorIndex) + cs.Evidence = append(cs.Evidence, &types.DuplicateVoteEvidence{voteErr.VoteA, voteErr.VoteB}) return err } else { // Probably an invalid signature / Bad peer. diff --git a/state/execution.go b/state/execution.go index dc38ac98..31f2662b 100644 --- a/state/execution.go +++ b/state/execution.go @@ -309,6 +309,8 @@ func (s *State) validateBlock(b *types.Block) error { } } + // TODO: Validate evidence + return nil } diff --git a/types/block.go b/types/block.go index 29775466..8d733d50 100644 --- a/types/block.go +++ b/types/block.go @@ -19,7 +19,8 @@ import ( type Block struct { *Header `json:"header"` *Data `json:"data"` - LastCommit *Commit `json:"last_commit"` + Evidence EvidenceData `json:"evidence"` + LastCommit *Commit `json:"last_commit"` } // MakeBlock returns a new block with an empty header, except what can be computed from itself. @@ -40,6 +41,11 @@ func MakeBlock(height int64, txs []Tx, commit *Commit) *Block { return block } +// AddEvidence appends the given evidence to the block +func (b *Block) AddEvidence(evidence []Evidence) { + b.Evidence.Evidence = append(b.Evidence.Evidence, evidence...) +} + // ValidateBasic performs basic validation that doesn't involve state data. // It checks the internal consistency of the block. func (b *Block) ValidateBasic() error { @@ -58,6 +64,9 @@ func (b *Block) ValidateBasic() error { if !bytes.Equal(b.DataHash, b.Data.Hash()) { return fmt.Errorf("Wrong Block.Header.DataHash. Expected %v, got %v", b.DataHash, b.Data.Hash()) } + if !bytes.Equal(b.EvidenceHash, b.Evidence.Hash()) { + return errors.New(cmn.Fmt("Wrong Block.Header.EvidenceHash. Expected %v, got %v", b.EvidenceHash, b.Evidence.Hash())) + } return nil } @@ -69,6 +78,9 @@ func (b *Block) FillHeader() { if b.DataHash == nil { b.DataHash = b.Data.Hash() } + if b.EvidenceHash == nil { + b.EvidenceHash = b.Evidence.Hash() + } } // Hash computes and returns the block hash. @@ -114,9 +126,11 @@ func (b *Block) StringIndented(indent string) string { %s %v %s %v %s %v +%s %v %s}#%v`, indent, b.Header.StringIndented(indent+" "), indent, b.Data.StringIndented(indent+" "), + indent, b.Evidence.StringIndented(indent+" "), indent, b.LastCommit.StringIndented(indent+" "), indent, b.Hash()) } @@ -134,6 +148,7 @@ func (b *Block) StringShort() string { // Header defines the structure of a Tendermint block header // TODO: limit header size +// NOTE: changes to the Header should be duplicated in the abci Header type Header struct { // basic block info ChainID string `json:"chain_id"` @@ -154,6 +169,9 @@ type Header struct { ConsensusHash data.Bytes `json:"consensus_hash"` // consensus params for current block AppHash data.Bytes `json:"app_hash"` // state after txs from the previous block LastResultsHash data.Bytes `json:"last_results_hash"` // root hash of all results from the txs from the previous block + + // consensus info + EvidenceHash data.Bytes `json:"evidence_hash"` // evidence included in the block } // Hash returns the hash of the header. @@ -175,6 +193,7 @@ func (h *Header) Hash() data.Bytes { "App": h.AppHash, "Consensus": h.ConsensusHash, "Results": h.LastResultsHash, + "Evidence": h.EvidenceHash, }) } @@ -196,6 +215,7 @@ func (h *Header) StringIndented(indent string) string { %s App: %v %s Conensus: %v %s Results: %v +%s Evidence: %v %s}#%v`, indent, h.ChainID, indent, h.Height, @@ -209,6 +229,7 @@ func (h *Header) StringIndented(indent string) string { indent, h.AppHash, indent, h.ConsensusHash, indent, h.LastResultsHash, + indent, h.EvidenceHash, indent, h.Hash()) } @@ -413,6 +434,33 @@ func (data *Data) StringIndented(indent string) string { indent, data.hash) } +//----------------------------------------------------------------------------- + +// EvidenceData contains any evidence of malicious wrong-doing by validators +type EvidenceData struct { + Evidence []Evidence `json:"evidence"` + + // Volatile + hash data.Bytes +} + +// Hash returns the hash of the data +func (data *EvidenceData) Hash() data.Bytes { + if data.hash == nil { + // TODO + } + return data.hash +} + +// StringIndented returns a string representation of the transactions +func (data *EvidenceData) StringIndented(indent string) string { + if data == nil { + return "nil-Data" + } + // TODO + return "" +} + //-------------------------------------------------------------------------------- // BlockID defines the unique ID of a block as its Hash and its PartSetHeader diff --git a/types/evidence.go b/types/evidence.go new file mode 100644 index 00000000..b3cf3317 --- /dev/null +++ b/types/evidence.go @@ -0,0 +1,52 @@ +package types + +import ( + "bytes" + "fmt" +) + +// Evidence represents any provable malicious activity by a validator +type Evidence interface { + Verify() error + Address() []byte +} + +//------------------------------------------- + +type DuplicateVoteEvidence struct { + VoteA *Vote + VoteB *Vote +} + +// Address returns the address of the validator +func (dve *DuplicateVoteEvidence) Address() []byte { + return dve.VoteA.ValidatorAddress +} + +// Verify returns an error if the two votes aren't from the same validator, for the same H/R/S, but for different blocks +func (dve *DuplicateVoteEvidence) Verify() error { + // H/R/S must be the same + if dve.VoteA.Height != dve.VoteB.Height || + dve.VoteA.Round != dve.VoteB.Round || + dve.VoteA.Type != dve.VoteB.Type { + return fmt.Errorf("DuplicateVoteEvidence Error: H/R/S does not match. Got %v and %v", dve.VoteA, dve.VoteB) + } + + // Address and Index must be the same + if !bytes.Equal(dve.VoteA.ValidatorAddress, dve.VoteB.ValidatorAddress) { + return fmt.Errorf("DuplicateVoteEvidence Error: Validator addresses do not match. Got %X and %X", dve.VoteA.ValidatorAddress, dve.VoteB.ValidatorAddress) + } + if dve.VoteA.ValidatorIndex != dve.VoteB.ValidatorIndex { + return fmt.Errorf("DuplicateVoteEvidence Error: Validator indices do not match. Got %d and %d", dve.VoteA.ValidatorIndex, dve.VoteB.ValidatorIndex) + } + + // BlockIDs must be different + if dve.VoteA.BlockID.Equals(dve.VoteB.BlockID) { + return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote!", dve.VoteA.BlockID) + } + + // Signatures must be valid + // TODO + + return nil +} From 4661c98c17c9510e598470ca1f27872eb1ef3d82 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 25 Jul 2017 12:10:48 -0400 Subject: [PATCH 02/35] add pubkey to conflicting vote evidence --- consensus/state.go | 2 +- types/evidence.go | 9 ++++++--- types/vote.go | 15 ++++++++++++--- types/vote_set.go | 5 +---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 0241cbd9..2323dfa0 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1340,7 +1340,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { } cs.Logger.Error("Found conflicting vote. Recording evidence in the RoundState", "height", vote.Height, "round", vote.Round, "type", vote.Type, "valAddr", vote.ValidatorAddress, "valIndex", vote.ValidatorIndex) - cs.Evidence = append(cs.Evidence, &types.DuplicateVoteEvidence{voteErr.VoteA, voteErr.VoteB}) + cs.Evidence = append(cs.Evidence, voteErr.DuplicateVoteEvidence) return err } else { // Probably an invalid signature / Bad peer. diff --git a/types/evidence.go b/types/evidence.go index b3cf3317..6eb74427 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -3,6 +3,8 @@ package types import ( "bytes" "fmt" + + "github.com/tendermint/go-crypto" ) // Evidence represents any provable malicious activity by a validator @@ -14,13 +16,14 @@ type Evidence interface { //------------------------------------------- type DuplicateVoteEvidence struct { - VoteA *Vote - VoteB *Vote + PubKey crypto.PubKey + VoteA *Vote + VoteB *Vote } // Address returns the address of the validator func (dve *DuplicateVoteEvidence) Address() []byte { - return dve.VoteA.ValidatorAddress + return dve.PubKey.Address() } // Verify returns an error if the two votes aren't from the same validator, for the same H/R/S, but for different blocks diff --git a/types/vote.go b/types/vote.go index aa993e33..a34bf9d4 100644 --- a/types/vote.go +++ b/types/vote.go @@ -22,12 +22,21 @@ var ( ) type ErrVoteConflictingVotes struct { - VoteA *Vote - VoteB *Vote + *DuplicateVoteEvidence } func (err *ErrVoteConflictingVotes) Error() string { - return "Conflicting votes" + return fmt.Sprintf("Conflicting votes from validator %v", err.PubKey.Address()) +} + +func NewConflictingVoteError(val *Validator, voteA, voteB *Vote) *ErrVoteConflictingVotes { + return &ErrVoteConflictingVotes{ + &DuplicateVoteEvidence{ + PubKey: val.PubKey, + VoteA: voteA, + VoteB: voteB, + }, + } } // Types of votes diff --git a/types/vote_set.go b/types/vote_set.go index 941852a8..5b802ceb 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -186,10 +186,7 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { // Add vote and get conflicting vote if any added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.VotingPower) if conflicting != nil { - return added, &ErrVoteConflictingVotes{ - VoteA: conflicting, - VoteB: vote, - } + return added, NewConflictingVoteError(val, conflicting, vote) } else { if !added { cmn.PanicSanity("Expected to add non-conflicting vote") From 35587658cd454e72b451f4e03cfc22f98a6d2cfb Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 25 Jul 2017 12:29:38 -0400 Subject: [PATCH 03/35] verify evidence in block --- state/execution.go | 9 ++++++++- types/evidence.go | 13 ++++++++++++- types/vote.go | 11 +++++++++++ types/vote_set.go | 5 ++--- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/state/execution.go b/state/execution.go index 31f2662b..d838531f 100644 --- a/state/execution.go +++ b/state/execution.go @@ -309,7 +309,14 @@ func (s *State) validateBlock(b *types.Block) error { } } - // TODO: Validate evidence + for _, ev := range block.Evidence.Evidence { + if err := ev.VoteA.Verify(s.ChainID, ev.PubKey); err != nil { + return types.ErrEvidenceInvalid(ev, err) + } + if err := ev.VoteB.Verify(s.ChainID, ev.PubKey); err != nil { + return types.ErrEvidenceInvalid(ev, err) + } + } return nil } diff --git a/types/evidence.go b/types/evidence.go index 6eb74427..5e2be9b5 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -7,6 +7,15 @@ import ( "github.com/tendermint/go-crypto" ) +type ErrEvidenceInvalid struct { + Evidence Evidence + Error error +} + +func (err *ErrEvidenceInvalid) Error() string { + return fmt.Sprintf("Invalid evidence: %v. Evidence: %v", err.Error, err.Evidence) +} + // Evidence represents any provable malicious activity by a validator type Evidence interface { Verify() error @@ -49,7 +58,9 @@ func (dve *DuplicateVoteEvidence) Verify() error { } // Signatures must be valid - // TODO + if !dve.PubKey.Verify(SignBytes(chainID, dve.VoteA), dve.VoteA.Signature) { + return ErrVoteInvalidSignature + } return nil } diff --git a/types/vote.go b/types/vote.go index a34bf9d4..82adf0d1 100644 --- a/types/vote.go +++ b/types/vote.go @@ -101,3 +101,14 @@ func (vote *Vote) String() string { cmn.Fingerprint(vote.BlockID.Hash), vote.Signature, CanonicalTime(vote.Timestamp)) } + +func (vote *Vote) Verify(chainID string, pubKey crypto.PubKey) error { + if !bytes.Equal(pubKey.Address(), v.ValidatorAddress) { + return ErrVoteInvalidValidatorAddress + } + + if !pubKey.VerifyBytes(SignBytes(chainID, vote), vote.Signature) { + return ErrVoteInvalidSignature + } + return nil +} diff --git a/types/vote_set.go b/types/vote_set.go index 5b802ceb..3f8a96e3 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -178,9 +178,8 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { } // Check signature. - if !val.PubKey.VerifyBytes(SignBytes(voteSet.chainID, vote), vote.Signature) { - // Bad signature. - return false, ErrVoteInvalidSignature + if err := vote.Verify(voteSet.chainID, val.PubKey); err != nil { + return false, err } // Add vote and get conflicting vote if any From 50850cf8a23a88428b0d49be82b48d731131c2e0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 21 Aug 2017 14:15:09 -0400 Subject: [PATCH 04/35] verify sigs on both votes; note about indices --- types/evidence.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/types/evidence.go b/types/evidence.go index 5e2be9b5..a9fdc42f 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -44,10 +44,11 @@ func (dve *DuplicateVoteEvidence) Verify() error { return fmt.Errorf("DuplicateVoteEvidence Error: H/R/S does not match. Got %v and %v", dve.VoteA, dve.VoteB) } - // Address and Index must be the same + // Address must be the same if !bytes.Equal(dve.VoteA.ValidatorAddress, dve.VoteB.ValidatorAddress) { return fmt.Errorf("DuplicateVoteEvidence Error: Validator addresses do not match. Got %X and %X", dve.VoteA.ValidatorAddress, dve.VoteB.ValidatorAddress) } + // XXX: Should we enforce index is the same ? if dve.VoteA.ValidatorIndex != dve.VoteB.ValidatorIndex { return fmt.Errorf("DuplicateVoteEvidence Error: Validator indices do not match. Got %d and %d", dve.VoteA.ValidatorIndex, dve.VoteB.ValidatorIndex) } @@ -61,6 +62,9 @@ func (dve *DuplicateVoteEvidence) Verify() error { if !dve.PubKey.Verify(SignBytes(chainID, dve.VoteA), dve.VoteA.Signature) { return ErrVoteInvalidSignature } + if !dve.PubKey.Verify(SignBytes(chainID, dve.VoteB), dve.VoteB.Signature) { + return ErrVoteInvalidSignature + } return nil } From 9cdcffbe4be2c629bc56d057c38d18f15dbb51ce Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 28 Aug 2017 19:46:38 -0400 Subject: [PATCH 05/35] types: comments; compiles; evidence test --- state/execution.go | 7 ++-- types/evidence.go | 26 +++++++++----- types/evidence_test.go | 82 ++++++++++++++++++++++++++++++++++++++++++ types/vote.go | 3 +- 4 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 types/evidence_test.go diff --git a/state/execution.go b/state/execution.go index d838531f..7e84ce2d 100644 --- a/state/execution.go +++ b/state/execution.go @@ -310,11 +310,8 @@ func (s *State) validateBlock(b *types.Block) error { } for _, ev := range block.Evidence.Evidence { - if err := ev.VoteA.Verify(s.ChainID, ev.PubKey); err != nil { - return types.ErrEvidenceInvalid(ev, err) - } - if err := ev.VoteB.Verify(s.ChainID, ev.PubKey); err != nil { - return types.ErrEvidenceInvalid(ev, err) + if err := ev.Verify(s.ChainID); err != nil { + return types.NewEvidenceInvalidErr(ev, err) } } diff --git a/types/evidence.go b/types/evidence.go index a9fdc42f..b61edcc2 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -7,36 +7,44 @@ import ( "github.com/tendermint/go-crypto" ) +// ErrEvidenceInvalid wraps a piece of evidence and the error denoting how or why it is invalid. type ErrEvidenceInvalid struct { - Evidence Evidence - Error error + Evidence Evidence + ErrorValue error } +func NewEvidenceInvalidErr(ev Evidence, err error) *ErrEvidenceInvalid { + return &ErrEvidenceInvalid{ev, err} +} + +// Error returns a string representation of the error. func (err *ErrEvidenceInvalid) Error() string { - return fmt.Sprintf("Invalid evidence: %v. Evidence: %v", err.Error, err.Evidence) + return fmt.Sprintf("Invalid evidence: %v. Evidence: %v", err.ErrorValue, err.Evidence) } // Evidence represents any provable malicious activity by a validator type Evidence interface { - Verify() error + Verify(chainID string) error Address() []byte } //------------------------------------------- +// DuplicateVoteEvidence contains evidence a validator signed two conflicting votes. type DuplicateVoteEvidence struct { PubKey crypto.PubKey VoteA *Vote VoteB *Vote } -// Address returns the address of the validator +// Address returns the address of the validator. func (dve *DuplicateVoteEvidence) Address() []byte { return dve.PubKey.Address() } -// Verify returns an error if the two votes aren't from the same validator, for the same H/R/S, but for different blocks -func (dve *DuplicateVoteEvidence) Verify() error { +// Verify returns an error if the two votes aren't conflicting. +// To be conflicting, they must be from the same validator, for the same H/R/S, but for different blocks. +func (dve *DuplicateVoteEvidence) Verify(chainID string) error { // H/R/S must be the same if dve.VoteA.Height != dve.VoteB.Height || dve.VoteA.Round != dve.VoteB.Round || @@ -59,10 +67,10 @@ func (dve *DuplicateVoteEvidence) Verify() error { } // Signatures must be valid - if !dve.PubKey.Verify(SignBytes(chainID, dve.VoteA), dve.VoteA.Signature) { + if !dve.PubKey.VerifyBytes(SignBytes(chainID, dve.VoteA), dve.VoteA.Signature) { return ErrVoteInvalidSignature } - if !dve.PubKey.Verify(SignBytes(chainID, dve.VoteB), dve.VoteB.Signature) { + if !dve.PubKey.VerifyBytes(SignBytes(chainID, dve.VoteB), dve.VoteB.Signature) { return ErrVoteInvalidSignature } diff --git a/types/evidence_test.go b/types/evidence_test.go new file mode 100644 index 00000000..a3f81154 --- /dev/null +++ b/types/evidence_test.go @@ -0,0 +1,82 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type voteData struct { + vote1 *Vote + vote2 *Vote + valid bool +} + +func makeVote(val *PrivValidator, chainID string, valIndex, height, round, step int, blockID BlockID) *Vote { + v := &Vote{ + ValidatorAddress: val.PubKey.Address(), + ValidatorIndex: valIndex, + Height: height, + Round: round, + Type: byte(step), + BlockID: blockID, + } + sig := val.PrivKey.Sign(SignBytes(chainID, v)) + v.Signature = sig + return v + +} + +func makeBlockID(hash string, partSetSize int, partSetHash string) BlockID { + return BlockID{ + Hash: []byte(hash), + PartsHeader: PartSetHeader{ + Total: partSetSize, + Hash: []byte(partSetHash), + }, + } + +} + +func TestEvidence(t *testing.T) { + val := GenPrivValidator() + val2 := GenPrivValidator() + blockID := makeBlockID("blockhash", 1000, "partshash") + blockID2 := makeBlockID("blockhash2", 1000, "partshash") + blockID3 := makeBlockID("blockhash", 10000, "partshash") + blockID4 := makeBlockID("blockhash", 10000, "partshash2") + + chainID := "mychain" + + vote1 := makeVote(val, chainID, 0, 10, 2, 1, blockID) + badVote := makeVote(val, chainID, 0, 10, 2, 1, blockID) + badVote.Signature = val2.PrivKey.Sign(SignBytes(chainID, badVote)) + + cases := []voteData{ + {vote1, makeVote(val, chainID, 0, 10, 2, 1, blockID2), true}, // different block ids + {vote1, makeVote(val, chainID, 0, 10, 2, 1, blockID3), true}, + {vote1, makeVote(val, chainID, 0, 10, 2, 1, blockID4), true}, + {vote1, makeVote(val, chainID, 0, 10, 2, 1, blockID), false}, // wrong block id + {vote1, makeVote(val, "mychain2", 0, 10, 2, 1, blockID2), false}, // wrong chain id + {vote1, makeVote(val, chainID, 1, 10, 2, 1, blockID2), false}, // wrong val index + {vote1, makeVote(val, chainID, 0, 11, 2, 1, blockID2), false}, // wrong height + {vote1, makeVote(val, chainID, 0, 10, 3, 1, blockID2), false}, // wrong round + {vote1, makeVote(val, chainID, 0, 10, 2, 2, blockID2), false}, // wrong step + {vote1, makeVote(val2, chainID, 0, 10, 2, 1, blockID), false}, // wrong validator + {vote1, badVote, false}, // signed by wrong key + } + + for _, c := range cases { + ev := &DuplicateVoteEvidence{ + PubKey: val.PubKey, + VoteA: c.vote1, + VoteB: c.vote2, + } + if c.valid { + assert.Nil(t, ev.Verify(chainID), "evidence should be valid") + } else { + assert.NotNil(t, ev.Verify(chainID), "evidence should be invalid") + } + } + +} diff --git a/types/vote.go b/types/vote.go index 82adf0d1..3136e4bb 100644 --- a/types/vote.go +++ b/types/vote.go @@ -1,6 +1,7 @@ package types import ( + "bytes" "errors" "fmt" "io" @@ -103,7 +104,7 @@ func (vote *Vote) String() string { } func (vote *Vote) Verify(chainID string, pubKey crypto.PubKey) error { - if !bytes.Equal(pubKey.Address(), v.ValidatorAddress) { + if !bytes.Equal(pubKey.Address(), vote.ValidatorAddress) { return ErrVoteInvalidValidatorAddress } From 77e45756f20f07f390f853ba8e1a9e11a94f1e9a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 28 Aug 2017 20:01:34 -0400 Subject: [PATCH 06/35] types: Evidences for merkle hashing; Evidence.String() --- state/execution.go | 2 +- types/block.go | 26 +++++++++++++++++++------- types/evidence.go | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/state/execution.go b/state/execution.go index 7e84ce2d..adc6c139 100644 --- a/state/execution.go +++ b/state/execution.go @@ -309,7 +309,7 @@ func (s *State) validateBlock(b *types.Block) error { } } - for _, ev := range block.Evidence.Evidence { + for _, ev := range block.Evidence.Evidences { if err := ev.Verify(s.ChainID); err != nil { return types.NewEvidenceInvalidErr(ev, err) } diff --git a/types/block.go b/types/block.go index 8d733d50..fa59c842 100644 --- a/types/block.go +++ b/types/block.go @@ -43,7 +43,7 @@ func MakeBlock(height int64, txs []Tx, commit *Commit) *Block { // AddEvidence appends the given evidence to the block func (b *Block) AddEvidence(evidence []Evidence) { - b.Evidence.Evidence = append(b.Evidence.Evidence, evidence...) + b.Evidence.Evidences = append(b.Evidence.Evidences, evidence...) } // ValidateBasic performs basic validation that doesn't involve state data. @@ -438,26 +438,38 @@ func (data *Data) StringIndented(indent string) string { // EvidenceData contains any evidence of malicious wrong-doing by validators type EvidenceData struct { - Evidence []Evidence `json:"evidence"` + Evidences Evidences `json:"evidence"` // Volatile hash data.Bytes } -// Hash returns the hash of the data +// Hash returns the hash of the data. func (data *EvidenceData) Hash() data.Bytes { if data.hash == nil { - // TODO + data.hash = data.Evidences.Hash() } return data.hash } -// StringIndented returns a string representation of the transactions +// StringIndented returns a string representation of the evidence. func (data *EvidenceData) StringIndented(indent string) string { if data == nil { - return "nil-Data" + return "nil-Evidence" } - // TODO + evStrings := make([]string, cmn.MinInt(len(data.Evidences), 21)) + for i, ev := range data.Evidences { + if i == 20 { + evStrings[i] = fmt.Sprintf("... (%v total)", len(data.Evidences)) + break + } + evStrings[i] = fmt.Sprintf("Evidence:%v", ev) + } + return fmt.Sprintf(`Data{ +%s %v +%s}#%v`, + indent, strings.Join(evStrings, "\n"+indent+" "), + indent, data.hash) return "" } diff --git a/types/evidence.go b/types/evidence.go index b61edcc2..e248eeb8 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/tendermint/go-crypto" + "github.com/tendermint/tmlibs/merkle" ) // ErrEvidenceInvalid wraps a piece of evidence and the error denoting how or why it is invalid. @@ -22,10 +23,34 @@ func (err *ErrEvidenceInvalid) Error() string { return fmt.Sprintf("Invalid evidence: %v. Evidence: %v", err.ErrorValue, err.Evidence) } +//------------------------------------------- + // Evidence represents any provable malicious activity by a validator type Evidence interface { - Verify(chainID string) error Address() []byte + Hash() []byte + Verify(chainID string) error + + String() string +} + +//------------------------------------------- + +type Evidences []Evidence + +func (evs Evidences) Hash() []byte { + // Recursive impl. + // Copied from tmlibs/merkle to avoid allocations + switch len(evs) { + case 0: + return nil + case 1: + return evs[0].Hash() + default: + left := Evidences(evs[:(len(evs)+1)/2]).Hash() + right := Evidences(evs[(len(evs)+1)/2:]).Hash() + return merkle.SimpleHashFromTwoHashes(left, right) + } } //------------------------------------------- @@ -37,11 +62,22 @@ type DuplicateVoteEvidence struct { VoteB *Vote } +// String returns a string representation of the evidence. +func (dve *DuplicateVoteEvidence) String() string { + return fmt.Sprintf("VoteA: %v; VoteB: %v", dve.VoteA, dve.VoteB) + +} + // Address returns the address of the validator. func (dve *DuplicateVoteEvidence) Address() []byte { return dve.PubKey.Address() } +// Hash returns the hash of the evidence. +func (dve *DuplicateVoteEvidence) Hash() []byte { + return merkle.SimpleHashFromBinary(dve) +} + // Verify returns an error if the two votes aren't conflicting. // To be conflicting, they must be from the same validator, for the same H/R/S, but for different blocks. func (dve *DuplicateVoteEvidence) Verify(chainID string) error { From eeab0efa569f2356148d9be1704ca00a1dd0d9af Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 28 Aug 2017 20:09:18 -0400 Subject: [PATCH 07/35] types: tx.go comments --- types/tx.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/types/tx.go b/types/tx.go index 5761b83e..4cf5843a 100644 --- a/types/tx.go +++ b/types/tx.go @@ -10,18 +10,18 @@ import ( "github.com/tendermint/tmlibs/merkle" ) -// Tx represents a transaction, which may contain arbitrary bytes. +// Tx is an arbitrary byte array. +// NOTE: Tx has no types at this level, so when go-wire encoded it's just length-prefixed. +// Alternatively, it may make sense to add types here and let +// []byte be type 0x1 so we can have versioned txs if need be in the future. type Tx []byte -// Hash returns the hash of the go-wire encoded Tx. -// Tx has no types at this level, so go-wire encoding only adds length-prefix. -// NOTE: It may make sense to add types here one day and let []byte be type 0x1 -// so we can have versioned txs if need be in the future. +// Hash computes the RIPEMD160 hash of the go-wire encoded transaction. func (tx Tx) Hash() []byte { return merkle.SimpleHashFromBinary(tx) } -// String returns a string representation of the Tx. +// String returns the hex-encoded transaction as a string. func (tx Tx) String() string { return fmt.Sprintf("Tx{%X}", []byte(tx)) } @@ -29,7 +29,7 @@ func (tx Tx) String() string { // Txs is a slice of Tx. type Txs []Tx -// Hash returns the simple Merkle root hash of the Txs. +// Hash returns the simple Merkle root hash of the transactions. func (txs Txs) Hash() []byte { // Recursive impl. // Copied from tmlibs/merkle to avoid allocations @@ -87,6 +87,7 @@ func (txs Txs) Proof(i int) TxProof { } } +// TxProof represents a Merkle proof of the presence of a transaction in the Merkle tree. type TxProof struct { Index, Total int RootHash data.Bytes @@ -94,12 +95,13 @@ type TxProof struct { Proof merkle.SimpleProof } +// LeadHash returns the hash of the transaction this proof refers to. func (tp TxProof) LeafHash() []byte { return tp.Data.Hash() } -// Validate returns nil if it matches the dataHash, and is internally consistent -// otherwise, returns a sensible error +// Validate verifies the proof. It returns nil if the RootHash matches the dataHash argument, +// and if the proof is internally consistent. Otherwise, it returns a sensible error. func (tp TxProof) Validate(dataHash []byte) error { if !bytes.Equal(dataHash, tp.RootHash) { return errors.New("Proof matches different data hash") From 39299e5cc107e331e1e88e6ad2f9a7b13987e8af Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 28 Aug 2017 21:08:26 -0400 Subject: [PATCH 08/35] consensus: note about duplicate evidence --- consensus/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/state.go b/consensus/state.go index 2323dfa0..2f9f07ec 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1340,6 +1340,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { } cs.Logger.Error("Found conflicting vote. Recording evidence in the RoundState", "height", vote.Height, "round", vote.Round, "type", vote.Type, "valAddr", vote.ValidatorAddress, "valIndex", vote.ValidatorIndex) + // TODO: ensure we haven't seen this evidence already ! cs.Evidence = append(cs.Evidence, voteErr.DuplicateVoteEvidence) return err } else { From 6e9433c7a828186c46e88d5e6bea30bc3f9a5915 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 1 Nov 2017 02:05:27 -0400 Subject: [PATCH 09/35] post rebase fix --- consensus/types/state.go | 7 +++++-- types/evidence.go | 8 ++++++++ types/evidence_test.go | 8 +++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/consensus/types/state.go b/consensus/types/state.go index da4df6a4..06194711 100644 --- a/consensus/types/state.go +++ b/consensus/types/state.go @@ -74,6 +74,7 @@ type RoundState struct { CommitRound int // LastCommit *types.VoteSet // Last precommits at Height-1 LastValidators *types.ValidatorSet + Evidence types.Evidences } // RoundStateEvent returns the H/R/S of the RoundState as an event. @@ -107,8 +108,9 @@ func (rs *RoundState) StringIndented(indent string) string { %s LockedRound: %v %s LockedBlock: %v %v %s Votes: %v -%s LastCommit: %v -%s LastValidators: %v +%s LastCommit: %v +%s LastValidators:%v +%s Evidence: %v %s}`, indent, rs.Height, rs.Round, rs.Step, indent, rs.StartTime, @@ -121,6 +123,7 @@ func (rs *RoundState) StringIndented(indent string) string { indent, rs.Votes.StringIndented(indent+" "), indent, rs.LastCommit.StringShort(), indent, rs.LastValidators.StringIndented(indent+" "), + indent, rs.Evidence.String(), indent) } diff --git a/types/evidence.go b/types/evidence.go index e248eeb8..2756faf3 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -53,6 +53,14 @@ func (evs Evidences) Hash() []byte { } } +func (evs Evidences) String() string { + s := "" + for _, e := range evs { + s += fmt.Sprintf("%s\t\t", e) + } + return s +} + //------------------------------------------- // DuplicateVoteEvidence contains evidence a validator signed two conflicting votes. diff --git a/types/evidence_test.go b/types/evidence_test.go index a3f81154..a69040d4 100644 --- a/types/evidence_test.go +++ b/types/evidence_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + cmn "github.com/tendermint/tmlibs/common" ) type voteData struct { @@ -12,7 +13,7 @@ type voteData struct { valid bool } -func makeVote(val *PrivValidator, chainID string, valIndex, height, round, step int, blockID BlockID) *Vote { +func makeVote(val *PrivValidatorFS, chainID string, valIndex, height, round, step int, blockID BlockID) *Vote { v := &Vote{ ValidatorAddress: val.PubKey.Address(), ValidatorIndex: valIndex, @@ -39,8 +40,9 @@ func makeBlockID(hash string, partSetSize int, partSetHash string) BlockID { } func TestEvidence(t *testing.T) { - val := GenPrivValidator() - val2 := GenPrivValidator() + _, tmpFilePath := cmn.Tempfile("priv_validator_") + val := GenPrivValidatorFS(tmpFilePath) + val2 := GenPrivValidatorFS(tmpFilePath) blockID := makeBlockID("blockhash", 1000, "partshash") blockID2 := makeBlockID("blockhash2", 1000, "partshash") blockID3 := makeBlockID("blockhash", 10000, "partshash") From 7d086e9524c3c25431ac6783726a15f0c323ba23 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 1 Nov 2017 13:57:22 -0600 Subject: [PATCH 10/35] check if we already have evidence --- consensus/state.go | 15 +++++++++++---- types/block.go | 1 - types/evidence.go | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 2f9f07ec..4785b362 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1338,10 +1338,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return err } - cs.Logger.Error("Found conflicting vote. Recording evidence in the RoundState", "height", vote.Height, "round", vote.Round, "type", vote.Type, "valAddr", vote.ValidatorAddress, "valIndex", vote.ValidatorIndex) - - // TODO: ensure we haven't seen this evidence already ! - cs.Evidence = append(cs.Evidence, voteErr.DuplicateVoteEvidence) + cs.addEvidence(voteErr.DuplicateVoteEvidence) return err } else { // Probably an invalid signature / Bad peer. @@ -1353,6 +1350,16 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { return nil } +func (cs *ConsensusState) addEvidence(ev types.Evidence) { + if cs.Evidence.Has(ev) { + return + } + + cs.Logger.Error("Found conflicting vote. Recording evidence in the RoundState", "evidence", ev) + + cs.Evidence = append(cs.Evidence, ev) +} + //----------------------------------------------------------------------------- func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, err error) { diff --git a/types/block.go b/types/block.go index fa59c842..b3e75442 100644 --- a/types/block.go +++ b/types/block.go @@ -86,7 +86,6 @@ func (b *Block) FillHeader() { // Hash computes and returns the block hash. // If the block is incomplete, block hash is nil for safety. func (b *Block) Hash() data.Bytes { - // fmt.Println(">>", b.Data) if b == nil || b.Header == nil || b.Data == nil || b.LastCommit == nil { return nil } diff --git a/types/evidence.go b/types/evidence.go index 2756faf3..d40c63ea 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -30,6 +30,7 @@ type Evidence interface { Address() []byte Hash() []byte Verify(chainID string) error + Equal(Evidence) bool String() string } @@ -61,6 +62,15 @@ func (evs Evidences) String() string { return s } +func (evs Evidences) Has(evidence Evidence) bool { + for _, ev := range evs { + if ev.Equal(evidence) { + return true + } + } + return false +} + //------------------------------------------- // DuplicateVoteEvidence contains evidence a validator signed two conflicting votes. @@ -120,3 +130,13 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string) error { return nil } + +// Equal checks if two pieces of evidence are equal. +func (dve *DuplicateVoteEvidence) Equal(ev Evidence) bool { + if _, ok := ev.(*DuplicateVoteEvidence); !ok { + return false + } + + // just check their hashes + return bytes.Equal(merkle.SimpleHashFromBinary(dve), merkle.SimpleHashFromBinary(ev)) +} From 48d778c4b39f9a65c24b1173ea0258bbed4324d3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 1 Nov 2017 13:57:38 -0600 Subject: [PATCH 11/35] types/params: introduce EvidenceParams --- types/params.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/types/params.go b/types/params.go index 19e86d44..15d8dbe2 100644 --- a/types/params.go +++ b/types/params.go @@ -14,9 +14,10 @@ const ( // ConsensusParams contains consensus critical parameters // that determine the validity of blocks. type ConsensusParams struct { - BlockSize `json:"block_size_params"` - TxSize `json:"tx_size_params"` - BlockGossip `json:"block_gossip_params"` + BlockSize `json:"block_size_params"` + TxSize `json:"tx_size_params"` + BlockGossip `json:"block_gossip_params"` + EvidenceParams `json:"evidence_params"` } // BlockSize contain limits on the block size. @@ -37,12 +38,18 @@ type BlockGossip struct { BlockPartSizeBytes int `json:"block_part_size_bytes"` // NOTE: must not be 0 } +// EvidenceParams determine how we handle evidence of malfeasance +type EvidenceParams struct { + MaxHeightDiff int `json:"max_height_diff"` // only accept new evidence more recent than this +} + // DefaultConsensusParams returns a default ConsensusParams. func DefaultConsensusParams() *ConsensusParams { return &ConsensusParams{ DefaultBlockSize(), DefaultTxSize(), DefaultBlockGossip(), + DefaultEvidenceParams(), } } @@ -70,6 +77,13 @@ func DefaultBlockGossip() BlockGossip { } } +// DefaultEvidence Params returns a default EvidenceParams. +func DefaultEvidenceParams() EvidenceParams { + return EvidenceParams{ + MaxHeightDiff: 100000, // 27.8 hrs at 1block/s + } +} + // Validate validates the ConsensusParams to ensure all values // are within their allowed limits, and returns an error if they are not. func (params *ConsensusParams) Validate() error { From 5b1f987ed105f7486ac175ad60e063dfcf436b04 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 2 Nov 2017 12:06:11 -0600 Subject: [PATCH 12/35] mempool: remove Peer interface. use p2p.Peer --- mempool/reactor.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/mempool/reactor.go b/mempool/reactor.go index 9aed416f..4523f824 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -100,17 +100,10 @@ type PeerState interface { GetHeight() int64 } -// Peer describes a peer. -type Peer interface { - IsRunning() bool - Send(byte, interface{}) bool - Get(string) interface{} -} - // Send new mempool txs to peer. // TODO: Handle mempool or reactor shutdown? // As is this routine may block forever if no new txs come in. -func (memR *MempoolReactor) broadcastTxRoutine(peer Peer) { +func (memR *MempoolReactor) broadcastTxRoutine(peer p2p.Peer) { if !memR.config.Broadcast { return } From fe4b53a4637283f97d8104230e6a58d9dfde540a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 2 Nov 2017 12:06:48 -0600 Subject: [PATCH 13/35] EvidencePool --- consensus/state.go | 19 +--- consensus/types/state.go | 3 - evidence/evidence_pool.go | 202 +++++++++++++++++++++++++++++++++ evidence/evidence_pool_test.go | 202 +++++++++++++++++++++++++++++++++ evidence/reactor.go | 138 ++++++++++++++++++++++ evidence/reactor_test.go | 108 ++++++++++++++++++ 6 files changed, 656 insertions(+), 16 deletions(-) create mode 100644 evidence/evidence_pool.go create mode 100644 evidence/evidence_pool_test.go create mode 100644 evidence/reactor.go create mode 100644 evidence/reactor_test.go diff --git a/consensus/state.go b/consensus/state.go index 4785b362..b26095de 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -78,6 +78,7 @@ type ConsensusState struct { proxyAppConn proxy.AppConnConsensus blockStore types.BlockStore mempool types.Mempool + evpool types.EvidencePool // internal state mtx sync.Mutex @@ -113,7 +114,7 @@ type ConsensusState struct { } // NewConsensusState returns a new ConsensusState. -func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { +func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool, evpool types.Mempool) *ConsensusState { cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, @@ -125,6 +126,7 @@ func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppCon done: make(chan struct{}), doWALCatchup: true, wal: nilWAL{}, + evpool: evpool, } // set function defaults (may be overwritten before calling Start) cs.decideProposal = cs.defaultDecideProposal @@ -864,7 +866,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts // Mempool validated transactions txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) block, parts := cs.state.MakeBlock(cs.Height, txs, commit) - block.AddEvidence(cs.Evidence) + evidence := cs.evpool.Evidence() + block.AddEvidence(evidence) return block, parts } @@ -1338,7 +1341,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return err } - cs.addEvidence(voteErr.DuplicateVoteEvidence) + cs.evpool.AddEvidence(voteErr.DuplicateVoteEvidence) return err } else { // Probably an invalid signature / Bad peer. @@ -1350,16 +1353,6 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { return nil } -func (cs *ConsensusState) addEvidence(ev types.Evidence) { - if cs.Evidence.Has(ev) { - return - } - - cs.Logger.Error("Found conflicting vote. Recording evidence in the RoundState", "evidence", ev) - - cs.Evidence = append(cs.Evidence, ev) -} - //----------------------------------------------------------------------------- func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, err error) { diff --git a/consensus/types/state.go b/consensus/types/state.go index 06194711..b95131f4 100644 --- a/consensus/types/state.go +++ b/consensus/types/state.go @@ -74,7 +74,6 @@ type RoundState struct { CommitRound int // LastCommit *types.VoteSet // Last precommits at Height-1 LastValidators *types.ValidatorSet - Evidence types.Evidences } // RoundStateEvent returns the H/R/S of the RoundState as an event. @@ -110,7 +109,6 @@ func (rs *RoundState) StringIndented(indent string) string { %s Votes: %v %s LastCommit: %v %s LastValidators:%v -%s Evidence: %v %s}`, indent, rs.Height, rs.Round, rs.Step, indent, rs.StartTime, @@ -123,7 +121,6 @@ func (rs *RoundState) StringIndented(indent string) string { indent, rs.Votes.StringIndented(indent+" "), indent, rs.LastCommit.StringShort(), indent, rs.LastValidators.StringIndented(indent+" "), - indent, rs.Evidence.String(), indent) } diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go new file mode 100644 index 00000000..d84eb13d --- /dev/null +++ b/evidence/evidence_pool.go @@ -0,0 +1,202 @@ +package evpool + +import ( + "container/list" + "fmt" + "sync" + "sync/atomic" + + "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/types" +) + +const cacheSize = 100000 + +// EvidencePool maintains a set of valid uncommitted evidence. +type EvidencePool struct { + config *cfg.EvidencePoolConfig + + mtx sync.Mutex + height int // the last block Update()'d to + evidence types.Evidences + // TODO: evidenceCache + + // TODO: need to persist evidence so we never lose it + + logger log.Logger +} + +func NewEvidencePool(config *cfg.EvidencePoolConfig, height int) *EvidencePool { + evpool := &EvidencePool{ + config: config, + height: height, + logger: log.NewNopLogger(), + } + evpool.initWAL() + return evpool +} + +// SetLogger sets the Logger. +func (evpool *EvidencePool) SetLogger(l log.Logger) { + evpool.logger = l +} + +// Evidence returns a copy of the pool's evidence. +func (evpool *EvidencePool) Evidence() types.Evidences { + evpool.mtx.Lock() + defer evpool.mtx.Unlock() + + evCopy := make(types.Evidences, len(evpool.evidence)) + for i, ev := range evpool.evidence { + evCopy[i] = ev + } + return evCopy +} + +// Size returns the number of pieces of evidence in the evpool. +func (evpool *EvidencePool) Size() int { + evpool.mtx.Lock() + defer evpool.mtx.Unlock() + return len(evpool.evidence) +} + +// Flush removes all evidence from the evpool +func (evpool *EvidencePool) Flush() { + evpool.mtx.Lock() + defer evpool.mtx.Unlock() + evpool.evidence = make(types.Evidence) +} + +// AddEvidence checks the evidence is valid and adds it to the pool. +func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { + evpool.mtx.Lock() + defer evpool.mtx.Unlock() + + if evpool.evidence.Has(evidence) { + return fmt.Errorf("Evidence already exists", "evidence", evidence) + } + cs.Logger.Info("Found conflicting vote. Recording evidence", "evidence", ev) + evpool.evidence = append(evpool.evidence, ev) + // TODO: write to disk ? WAL ? + return nil +} + +// Update informs the evpool that the given evidence was committed and can be discarded. +// NOTE: this should be called *after* block is committed by consensus. +func (evpool *EvidencePool) Update(height int, evidence types.Evidences) { + + // First, create a lookup map of txns in new txs. + evMap := make(map[string]struct{}) + for _, ev := range evidence { + evMap[string(evidence.Hash())] = struct{}{} + } + + // Set height + evpool.height = height + + // Remove evidence that is already committed . + goodEvidence := evpool.filterEvidence(evMap) + _ = goodEvidence + +} + +// TODO: +func (evpool *EvidencePool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx { + goodTxs := make([]types.Tx, 0, evpool.txs.Len()) + for e := evpool.txs.Front(); e != nil; e = e.Next() { + memTx := e.Value.(*evpoolTx) + // Remove the tx if it's alredy in a block. + if _, ok := blockTxsMap[string(memTx.tx)]; ok { + // remove from clist + evpool.txs.Remove(e) + e.DetachPrev() + + // NOTE: we don't remove committed txs from the cache. + continue + } + // Good tx! + goodTxs = append(goodTxs, memTx.tx) + } + return goodTxs +} + +//-------------------------------------------------------------------------------- + +// evpoolTx is a transaction that successfully ran +type evpoolEvidence struct { + counter int64 // a simple incrementing counter + height int64 // height that this tx had been validated in + evidence types.Evidence // +} + +// Height returns the height for this transaction +func (memTx *evpoolTx) Height() int { + return int(atomic.LoadInt64(&memTx.height)) +} + +//-------------------------------------------------------------------------------- +// TODO: + +// txCache maintains a cache of evidence +type txCache struct { + mtx sync.Mutex + size int + map_ map[string]struct{} + list *list.List // to remove oldest tx when cache gets too big +} + +// newTxCache returns a new txCache. +func newTxCache(cacheSize int) *txCache { + return &txCache{ + size: cacheSize, + map_: make(map[string]struct{}, cacheSize), + list: list.New(), + } +} + +// Reset resets the txCache to empty. +func (cache *txCache) Reset() { + cache.mtx.Lock() + cache.map_ = make(map[string]struct{}, cacheSize) + cache.list.Init() + cache.mtx.Unlock() +} + +// Exists returns true if the given tx is cached. +func (cache *txCache) Exists(tx types.Tx) bool { + cache.mtx.Lock() + _, exists := cache.map_[string(tx)] + cache.mtx.Unlock() + return exists +} + +// Push adds the given tx to the txCache. It returns false if tx is already in the cache. +func (cache *txCache) Push(tx types.Tx) bool { + cache.mtx.Lock() + defer cache.mtx.Unlock() + + if _, exists := cache.map_[string(tx)]; exists { + return false + } + + if cache.list.Len() >= cache.size { + popped := cache.list.Front() + poppedTx := popped.Value.(types.Tx) + // NOTE: the tx may have already been removed from the map + // but deleting a non-existent element is fine + delete(cache.map_, string(poppedTx)) + cache.list.Remove(popped) + } + cache.map_[string(tx)] = struct{}{} + cache.list.PushBack(tx) + return true +} + +// Remove removes the given tx from the cache. +func (cache *txCache) Remove(tx types.Tx) { + cache.mtx.Lock() + delete(cache.map_, string(tx)) + cache.mtx.Unlock() +} diff --git a/evidence/evidence_pool_test.go b/evidence/evidence_pool_test.go new file mode 100644 index 00000000..46401e88 --- /dev/null +++ b/evidence/evidence_pool_test.go @@ -0,0 +1,202 @@ +package mempool + +import ( + "crypto/rand" + "encoding/binary" + "testing" + "time" + + "github.com/tendermint/abci/example/counter" + "github.com/tendermint/abci/example/dummy" + "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/types" +) + +func newMempoolWithApp(cc proxy.ClientCreator) *Mempool { + config := cfg.ResetTestRoot("mempool_test") + + appConnMem, _ := cc.NewABCIClient() + appConnMem.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "mempool")) + appConnMem.Start() + mempool := NewMempool(config.Mempool, appConnMem, 0) + mempool.SetLogger(log.TestingLogger()) + return mempool +} + +func ensureNoFire(t *testing.T, ch <-chan int, timeoutMS int) { + timer := time.NewTimer(time.Duration(timeoutMS) * time.Millisecond) + select { + case <-ch: + t.Fatal("Expected not to fire") + case <-timer.C: + } +} + +func ensureFire(t *testing.T, ch <-chan int, timeoutMS int) { + timer := time.NewTimer(time.Duration(timeoutMS) * time.Millisecond) + select { + case <-ch: + case <-timer.C: + t.Fatal("Expected to fire") + } +} + +func checkTxs(t *testing.T, mempool *Mempool, count int) types.Txs { + txs := make(types.Txs, count) + for i := 0; i < count; i++ { + txBytes := make([]byte, 20) + txs[i] = txBytes + rand.Read(txBytes) + err := mempool.CheckTx(txBytes, nil) + if err != nil { + t.Fatal("Error after CheckTx: %v", err) + } + } + return txs +} + +func TestTxsAvailable(t *testing.T) { + app := dummy.NewDummyApplication() + cc := proxy.NewLocalClientCreator(app) + mempool := newMempoolWithApp(cc) + mempool.EnableTxsAvailable() + + timeoutMS := 500 + + // with no txs, it shouldnt fire + ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) + + // send a bunch of txs, it should only fire once + txs := checkTxs(t, mempool, 100) + ensureFire(t, mempool.TxsAvailable(), timeoutMS) + ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) + + // call update with half the txs. + // it should fire once now for the new height + // since there are still txs left + committedTxs, txs := txs[:50], txs[50:] + mempool.Update(1, committedTxs) + ensureFire(t, mempool.TxsAvailable(), timeoutMS) + ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) + + // send a bunch more txs. we already fired for this height so it shouldnt fire again + moreTxs := checkTxs(t, mempool, 50) + ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) + + // now call update with all the txs. it should not fire as there are no txs left + committedTxs = append(txs, moreTxs...) + mempool.Update(2, committedTxs) + ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) + + // send a bunch more txs, it should only fire once + checkTxs(t, mempool, 100) + ensureFire(t, mempool.TxsAvailable(), timeoutMS) + ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) +} + +func TestSerialReap(t *testing.T) { + app := counter.NewCounterApplication(true) + app.SetOption("serial", "on") + cc := proxy.NewLocalClientCreator(app) + + mempool := newMempoolWithApp(cc) + appConnCon, _ := cc.NewABCIClient() + appConnCon.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "consensus")) + if _, err := appConnCon.Start(); err != nil { + t.Fatalf("Error starting ABCI client: %v", err.Error()) + } + + deliverTxsRange := func(start, end int) { + // Deliver some txs. + for i := start; i < end; i++ { + + // This will succeed + txBytes := make([]byte, 8) + binary.BigEndian.PutUint64(txBytes, uint64(i)) + err := mempool.CheckTx(txBytes, nil) + if err != nil { + t.Fatal("Error after CheckTx: %v", err) + } + + // This will fail because not serial (incrementing) + // However, error should still be nil. + // It just won't show up on Reap(). + err = mempool.CheckTx(txBytes, nil) + if err != nil { + t.Fatal("Error after CheckTx: %v", err) + } + + } + } + + reapCheck := func(exp int) { + txs := mempool.Reap(-1) + if len(txs) != exp { + t.Fatalf("Expected to reap %v txs but got %v", exp, len(txs)) + } + } + + updateRange := func(start, end int) { + txs := make([]types.Tx, 0) + for i := start; i < end; i++ { + txBytes := make([]byte, 8) + binary.BigEndian.PutUint64(txBytes, uint64(i)) + txs = append(txs, txBytes) + } + mempool.Update(0, txs) + } + + commitRange := func(start, end int) { + // Deliver some txs. + for i := start; i < end; i++ { + txBytes := make([]byte, 8) + binary.BigEndian.PutUint64(txBytes, uint64(i)) + res := appConnCon.DeliverTxSync(txBytes) + if !res.IsOK() { + t.Errorf("Error committing tx. Code:%v result:%X log:%v", + res.Code, res.Data, res.Log) + } + } + res := appConnCon.CommitSync() + if len(res.Data) != 8 { + t.Errorf("Error committing. Hash:%X log:%v", res.Data, res.Log) + } + } + + //---------------------------------------- + + // Deliver some txs. + deliverTxsRange(0, 100) + + // Reap the txs. + reapCheck(100) + + // Reap again. We should get the same amount + reapCheck(100) + + // Deliver 0 to 999, we should reap 900 new txs + // because 100 were already counted. + deliverTxsRange(0, 1000) + + // Reap the txs. + reapCheck(1000) + + // Reap again. We should get the same amount + reapCheck(1000) + + // Commit from the conensus AppConn + commitRange(0, 500) + updateRange(0, 500) + + // We should have 500 left. + reapCheck(500) + + // Deliver 100 invalid txs and 100 valid txs + deliverTxsRange(900, 1100) + + // We should have 600 now. + reapCheck(600) +} diff --git a/evidence/reactor.go b/evidence/reactor.go new file mode 100644 index 00000000..7554bb4c --- /dev/null +++ b/evidence/reactor.go @@ -0,0 +1,138 @@ +package evpool + +import ( + "bytes" + "fmt" + "reflect" + + wire "github.com/tendermint/go-wire" + "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/types" +) + +const ( + EvidencePoolChannel = byte(0x38) + + maxEvidencePoolMessageSize = 1048576 // 1MB TODO make it configurable + peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount +) + +// EvidencePoolReactor handles evpool evidence broadcasting amongst peers. +type EvidencePoolReactor struct { + p2p.BaseReactor + config *cfg.EvidencePoolConfig + EvidencePool *EvidencePool + evsw types.EventSwitch +} + +// NewEvidencePoolReactor returns a new EvidencePoolReactor with the given config and evpool. +func NewEvidencePoolReactor(config *cfg.EvidencePoolConfig, evpool *EvidencePool) *EvidencePoolReactor { + evR := &EvidencePoolReactor{ + config: config, + EvidencePool: evpool, + } + evR.BaseReactor = *p2p.NewBaseReactor("EvidencePoolReactor", evR) + return evR +} + +// SetLogger sets the Logger on the reactor and the underlying EvidencePool. +func (evR *EvidencePoolReactor) SetLogger(l log.Logger) { + evR.Logger = l + evR.EvidencePool.SetLogger(l) +} + +// GetChannels implements Reactor. +// It returns the list of channels for this reactor. +func (evR *EvidencePoolReactor) GetChannels() []*p2p.ChannelDescriptor { + return []*p2p.ChannelDescriptor{ + &p2p.ChannelDescriptor{ + ID: EvidencePoolChannel, + Priority: 5, + }, + } +} + +// AddPeer implements Reactor. +func (evR *EvidencePoolReactor) AddPeer(peer p2p.Peer) { + // send the new peer all current evidence + evidence := evR.evpool.Evidence() + msg := EvidenceMessage{evidence} + success := peer.Send(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) + if !success { + // TODO: remove peer ? + } +} + +// RemovePeer implements Reactor. +func (evR *EvidencePoolReactor) RemovePeer(peer p2p.Peer, reason interface{}) { +} + +// Receive implements Reactor. +// It adds any received evidence to the evpool. +func (evR *EvidencePoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { + _, msg, err := DecodeMessage(msgBytes) + if err != nil { + evR.Logger.Error("Error decoding message", "err", err) + return + } + evR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) + + switch msg := msg.(type) { + case *EvidenceMessage: + for _, ev := range msg.Evidence { + err := evR.EvidencePool.AddEvidence(msg.Evidence, nil) + if err != nil { + evR.Logger.Info("Evidence is not valid", "evidence", msg.Evidence, "err", err) + // TODO: punish peer + } else { + // TODO: broadcast good evidence to all peers (except sender? ) + } + } + default: + evR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) + } +} + +// SetEventSwitch implements events.Eventable. +func (evR *EvidencePoolReactor) SetEventSwitch(evsw types.EventSwitch) { + evR.evsw = evsw +} + +//----------------------------------------------------------------------------- +// Messages + +const ( + msgTypeEvidence = byte(0x01) +) + +// EvidencePoolMessage is a message sent or received by the EvidencePoolReactor. +type EvidencePoolMessage interface{} + +var _ = wire.RegisterInterface( + struct{ EvidencePoolMessage }{}, + wire.ConcreteType{&EvidenceMessage{}, msgTypeEvidence}, +) + +// DecodeMessage decodes a byte-array into a EvidencePoolMessage. +func DecodeMessage(bz []byte) (msgType byte, msg EvidencePoolMessage, err error) { + msgType = bz[0] + n := new(int) + r := bytes.NewReader(bz) + msg = wire.ReadBinary(struct{ EvidencePoolMessage }{}, r, maxEvidencePoolMessageSize, n, &err).(struct{ EvidencePoolMessage }).EvidencePoolMessage + return +} + +//------------------------------------- + +// EvidenceMessage contains a list of evidence. +type EvidenceMessage struct { + Evidence types.Evidences +} + +// String returns a string representation of the EvidenceMessage. +func (m *EvidenceMessage) String() string { + return fmt.Sprintf("[EvidenceMessage %v]", m.Evidence) +} diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go new file mode 100644 index 00000000..e488311b --- /dev/null +++ b/evidence/reactor_test.go @@ -0,0 +1,108 @@ +package evpool + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/go-kit/kit/log/term" + + "github.com/tendermint/abci/example/dummy" + "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/types" +) + +// evpoolLogger is a TestingLogger which uses a different +// color for each validator ("validator" key must exist). +func evpoolLogger() log.Logger { + return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor { + for i := 0; i < len(keyvals)-1; i += 2 { + if keyvals[i] == "validator" { + return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))} + } + } + return term.FgBgColor{} + }) +} + +// connect N evpool reactors through N switches +func makeAndConnectEvidencePoolReactors(config *cfg.Config, N int) []*EvidencePoolReactor { + reactors := make([]*EvidencePoolReactor, N) + logger := evpoolLogger() + for i := 0; i < N; i++ { + app := dummy.NewDummyApplication() + cc := proxy.NewLocalClientCreator(app) + evpool := newEvidencePoolWithApp(cc) + + reactors[i] = NewEvidencePoolReactor(config.EvidencePool, evpool) // so we dont start the consensus states + reactors[i].SetLogger(logger.With("validator", i)) + } + + p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("MEMPOOL", reactors[i]) + return s + + }, p2p.Connect2Switches) + return reactors +} + +// wait for all evidences on all reactors +func waitForTxs(t *testing.T, evidences types.Txs, reactors []*EvidencePoolReactor) { + // wait for the evidences in all evpools + wg := new(sync.WaitGroup) + for i := 0; i < len(reactors); i++ { + wg.Add(1) + go _waitForTxs(t, wg, evidences, i, reactors) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + timer := time.After(TIMEOUT) + select { + case <-timer: + t.Fatal("Timed out waiting for evidences") + case <-done: + } +} + +// wait for all evidences on a single evpool +func _waitForTxs(t *testing.T, wg *sync.WaitGroup, evidences types.Txs, reactorIdx int, reactors []*EvidencePoolReactor) { + + evpool := reactors[reactorIdx].EvidencePool + for evpool.Size() != len(evidences) { + time.Sleep(time.Second) + } + + reapedTxs := evpool.Reap(len(evidences)) + for i, evidence := range evidences { + assert.Equal(t, evidence, reapedTxs[i], fmt.Sprintf("evidences at index %d on reactor %d don't match: %v vs %v", i, reactorIdx, evidence, reapedTxs[i])) + } + wg.Done() +} + +var ( + NUM_TXS = 1000 + TIMEOUT = 120 * time.Second // ridiculously high because CircleCI is slow +) + +func TestReactorBroadcastTxMessage(t *testing.T) { + config := cfg.TestConfig() + N := 4 + reactors := makeAndConnectEvidencePoolReactors(config, N) + + // send a bunch of evidences to the first reactor's evpool + // and wait for them all to be received in the others + evidences := checkTxs(t, reactors[0].EvidencePool, NUM_TXS) + waitForTxs(t, evidences, reactors) +} From 10c43c9edc96669cea97a1261bcf67d421e2b3ad Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 2 Nov 2017 18:26:07 -0600 Subject: [PATCH 14/35] introduce evidence store --- consensus/state.go | 9 +- evidence/evidence_pool.go | 193 +++++++-------------------------- evidence/evidence_pool_test.go | 2 +- evidence/reactor.go | 63 ++++++++--- evidence/store.go | 100 +++++++++++++++++ types/block.go | 3 +- types/evidence.go | 82 +++++++++++++- 7 files changed, 275 insertions(+), 177 deletions(-) create mode 100644 evidence/store.go diff --git a/consensus/state.go b/consensus/state.go index b26095de..7b7c04c4 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -17,6 +17,7 @@ import ( cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" + evpool "github.com/tendermint/tendermint/evidence" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -78,7 +79,7 @@ type ConsensusState struct { proxyAppConn proxy.AppConnConsensus blockStore types.BlockStore mempool types.Mempool - evpool types.EvidencePool + evpool evpool.EvidencePool // internal state mtx sync.Mutex @@ -114,7 +115,7 @@ type ConsensusState struct { } // NewConsensusState returns a new ConsensusState. -func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool, evpool types.Mempool) *ConsensusState { +func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, @@ -126,7 +127,7 @@ func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppCon done: make(chan struct{}), doWALCatchup: true, wal: nilWAL{}, - evpool: evpool, + // evpool: evpool, } // set function defaults (may be overwritten before calling Start) cs.decideProposal = cs.defaultDecideProposal @@ -866,7 +867,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts // Mempool validated transactions txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) block, parts := cs.state.MakeBlock(cs.Height, txs, commit) - evidence := cs.evpool.Evidence() + evidence := cs.evpool.PendingEvidence() block.AddEvidence(evidence) return block, parts } diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go index d84eb13d..3a31c40c 100644 --- a/evidence/evidence_pool.go +++ b/evidence/evidence_pool.go @@ -1,14 +1,8 @@ package evpool import ( - "container/list" - "fmt" - "sync" - "sync/atomic" - "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" ) @@ -16,25 +10,23 @@ const cacheSize = 100000 // EvidencePool maintains a set of valid uncommitted evidence. type EvidencePool struct { - config *cfg.EvidencePoolConfig - - mtx sync.Mutex - height int // the last block Update()'d to - evidence types.Evidences - // TODO: evidenceCache - - // TODO: need to persist evidence so we never lose it - + config *EvidencePoolConfig logger log.Logger + + evidenceStore *EvidenceStore + newEvidenceChan chan types.Evidence } -func NewEvidencePool(config *cfg.EvidencePoolConfig, height int) *EvidencePool { +type EvidencePoolConfig struct { +} + +func NewEvidencePool(config *EvidencePoolConfig, evidenceStore *EvidenceStore) *EvidencePool { evpool := &EvidencePool{ - config: config, - height: height, - logger: log.NewNopLogger(), + config: config, + logger: log.NewNopLogger(), + evidenceStore: evidenceStore, + newEvidenceChan: make(chan types.Evidence), } - evpool.initWAL() return evpool } @@ -43,160 +35,59 @@ func (evpool *EvidencePool) SetLogger(l log.Logger) { evpool.logger = l } -// Evidence returns a copy of the pool's evidence. -func (evpool *EvidencePool) Evidence() types.Evidences { - evpool.mtx.Lock() - defer evpool.mtx.Unlock() - - evCopy := make(types.Evidences, len(evpool.evidence)) - for i, ev := range evpool.evidence { - evCopy[i] = ev - } - return evCopy +// NewEvidenceChan returns a channel on which new evidence is sent. +func (evpool *EvidencePool) NewEvidenceChan() chan types.Evidence { + return evpool.newEvidenceChan } -// Size returns the number of pieces of evidence in the evpool. -func (evpool *EvidencePool) Size() int { - evpool.mtx.Lock() - defer evpool.mtx.Unlock() - return len(evpool.evidence) +// PriorityEvidence returns the priority evidence. +func (evpool *EvidencePool) PriorityEvidence() []types.Evidence { + // TODO + return nil } -// Flush removes all evidence from the evpool -func (evpool *EvidencePool) Flush() { - evpool.mtx.Lock() - defer evpool.mtx.Unlock() - evpool.evidence = make(types.Evidence) +// PendingEvidence returns all uncommitted evidence. +func (evpool *EvidencePool) PendingEvidence() []types.Evidence { + // TODO + return nil } // AddEvidence checks the evidence is valid and adds it to the pool. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { - evpool.mtx.Lock() - defer evpool.mtx.Unlock() - - if evpool.evidence.Has(evidence) { - return fmt.Errorf("Evidence already exists", "evidence", evidence) + idx := 1 // TODO + added, err := evpool.evidenceStore.AddNewEvidence(idx, evidence) + if err != nil { + return err + } else if !added { + // evidence already known, just ignore + return } - cs.Logger.Info("Found conflicting vote. Recording evidence", "evidence", ev) - evpool.evidence = append(evpool.evidence, ev) - // TODO: write to disk ? WAL ? + + evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence) + + evpool.newEvidenceChan <- evidence return nil } // Update informs the evpool that the given evidence was committed and can be discarded. // NOTE: this should be called *after* block is committed by consensus. -func (evpool *EvidencePool) Update(height int, evidence types.Evidences) { +func (evpool *EvidencePool) Update(height int, evidence []types.Evidence) { + + // First, create a lookup map of new committed evidence - // First, create a lookup map of txns in new txs. evMap := make(map[string]struct{}) for _, ev := range evidence { - evMap[string(evidence.Hash())] = struct{}{} + evpool.evidenceStore.MarkEvidenceAsCommitted(ev) + evMap[string(ev.Hash())] = struct{}{} } - // Set height - evpool.height = height - // Remove evidence that is already committed . goodEvidence := evpool.filterEvidence(evMap) _ = goodEvidence } -// TODO: -func (evpool *EvidencePool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx { - goodTxs := make([]types.Tx, 0, evpool.txs.Len()) - for e := evpool.txs.Front(); e != nil; e = e.Next() { - memTx := e.Value.(*evpoolTx) - // Remove the tx if it's alredy in a block. - if _, ok := blockTxsMap[string(memTx.tx)]; ok { - // remove from clist - evpool.txs.Remove(e) - e.DetachPrev() - - // NOTE: we don't remove committed txs from the cache. - continue - } - // Good tx! - goodTxs = append(goodTxs, memTx.tx) - } - return goodTxs -} - -//-------------------------------------------------------------------------------- - -// evpoolTx is a transaction that successfully ran -type evpoolEvidence struct { - counter int64 // a simple incrementing counter - height int64 // height that this tx had been validated in - evidence types.Evidence // -} - -// Height returns the height for this transaction -func (memTx *evpoolTx) Height() int { - return int(atomic.LoadInt64(&memTx.height)) -} - -//-------------------------------------------------------------------------------- -// TODO: - -// txCache maintains a cache of evidence -type txCache struct { - mtx sync.Mutex - size int - map_ map[string]struct{} - list *list.List // to remove oldest tx when cache gets too big -} - -// newTxCache returns a new txCache. -func newTxCache(cacheSize int) *txCache { - return &txCache{ - size: cacheSize, - map_: make(map[string]struct{}, cacheSize), - list: list.New(), - } -} - -// Reset resets the txCache to empty. -func (cache *txCache) Reset() { - cache.mtx.Lock() - cache.map_ = make(map[string]struct{}, cacheSize) - cache.list.Init() - cache.mtx.Unlock() -} - -// Exists returns true if the given tx is cached. -func (cache *txCache) Exists(tx types.Tx) bool { - cache.mtx.Lock() - _, exists := cache.map_[string(tx)] - cache.mtx.Unlock() - return exists -} - -// Push adds the given tx to the txCache. It returns false if tx is already in the cache. -func (cache *txCache) Push(tx types.Tx) bool { - cache.mtx.Lock() - defer cache.mtx.Unlock() - - if _, exists := cache.map_[string(tx)]; exists { - return false - } - - if cache.list.Len() >= cache.size { - popped := cache.list.Front() - poppedTx := popped.Value.(types.Tx) - // NOTE: the tx may have already been removed from the map - // but deleting a non-existent element is fine - delete(cache.map_, string(poppedTx)) - cache.list.Remove(popped) - } - cache.map_[string(tx)] = struct{}{} - cache.list.PushBack(tx) - return true -} - -// Remove removes the given tx from the cache. -func (cache *txCache) Remove(tx types.Tx) { - cache.mtx.Lock() - delete(cache.map_, string(tx)) - cache.mtx.Unlock() +func (evpool *EvidencePool) filterEvidence(blockEvidenceMap map[string]struct{}) []types.Evidence { + // TODO: + return nil } diff --git a/evidence/evidence_pool_test.go b/evidence/evidence_pool_test.go index 46401e88..fba5941c 100644 --- a/evidence/evidence_pool_test.go +++ b/evidence/evidence_pool_test.go @@ -1,4 +1,4 @@ -package mempool +package evpool import ( "crypto/rand" diff --git a/evidence/reactor.go b/evidence/reactor.go index 7554bb4c..d62e88b3 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -4,11 +4,11 @@ import ( "bytes" "fmt" "reflect" + "time" wire "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -18,21 +18,22 @@ const ( maxEvidencePoolMessageSize = 1048576 // 1MB TODO make it configurable peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount + broadcastEvidenceIntervalS = 60 // broadcast uncommitted evidence this often ) // EvidencePoolReactor handles evpool evidence broadcasting amongst peers. type EvidencePoolReactor struct { p2p.BaseReactor - config *cfg.EvidencePoolConfig - EvidencePool *EvidencePool - evsw types.EventSwitch + config *EvidencePoolConfig + evpool *EvidencePool + evsw types.EventSwitch } // NewEvidencePoolReactor returns a new EvidencePoolReactor with the given config and evpool. -func NewEvidencePoolReactor(config *cfg.EvidencePoolConfig, evpool *EvidencePool) *EvidencePoolReactor { +func NewEvidencePoolReactor(config *EvidencePoolConfig, evpool *EvidencePool) *EvidencePoolReactor { evR := &EvidencePoolReactor{ - config: config, - EvidencePool: evpool, + config: config, + evpool: evpool, } evR.BaseReactor = *p2p.NewBaseReactor("EvidencePoolReactor", evR) return evR @@ -41,7 +42,16 @@ func NewEvidencePoolReactor(config *cfg.EvidencePoolConfig, evpool *EvidencePool // SetLogger sets the Logger on the reactor and the underlying EvidencePool. func (evR *EvidencePoolReactor) SetLogger(l log.Logger) { evR.Logger = l - evR.EvidencePool.SetLogger(l) + evR.evpool.SetLogger(l) +} + +// OnStart implements cmn.Service +func (evR *EvidencePoolReactor) OnStart() error { + if err := evR.BaseReactor.OnStart(); err != nil { + return err + } + go evR.broadcastRoutine() + return nil } // GetChannels implements Reactor. @@ -57,13 +67,16 @@ func (evR *EvidencePoolReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor. func (evR *EvidencePoolReactor) AddPeer(peer p2p.Peer) { - // send the new peer all current evidence - evidence := evR.evpool.Evidence() + // first send the peer high-priority evidence + evidence := evR.evpool.PriorityEvidence() msg := EvidenceMessage{evidence} success := peer.Send(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) if !success { // TODO: remove peer ? } + + // TODO: send the remaining pending evidence + // or just let the broadcastRoutine do it ? } // RemovePeer implements Reactor. @@ -83,12 +96,10 @@ func (evR *EvidencePoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte switch msg := msg.(type) { case *EvidenceMessage: for _, ev := range msg.Evidence { - err := evR.EvidencePool.AddEvidence(msg.Evidence, nil) + err := evR.evpool.AddEvidence(ev) if err != nil { evR.Logger.Info("Evidence is not valid", "evidence", msg.Evidence, "err", err) // TODO: punish peer - } else { - // TODO: broadcast good evidence to all peers (except sender? ) } } default: @@ -101,6 +112,30 @@ func (evR *EvidencePoolReactor) SetEventSwitch(evsw types.EventSwitch) { evR.evsw = evsw } +// broadcast new evidence to all peers +func (evR *EvidencePoolReactor) broadcastRoutine() { + ticker := time.NewTicker(time.Second * broadcastEvidenceIntervalS) + for { + select { + case evidence := <-evR.evpool.NewEvidenceChan(): + // broadcast some new evidence + msg := EvidenceMessage{[]types.Evidence{evidence}} + evR.Switch.Broadcast(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) + + // NOTE: Broadcast runs asynchronously, so this should wait on the successChan + // in another routine before marking to be proper. + idx := 1 // TODO + evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(idx, evidence) + case <-ticker.C: + // broadcast all pending evidence + msg := EvidenceMessage{evR.evpool.PendingEvidence()} + evR.Switch.Broadcast(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) + case <-evR.Quit: + return + } + } +} + //----------------------------------------------------------------------------- // Messages @@ -129,7 +164,7 @@ func DecodeMessage(bz []byte) (msgType byte, msg EvidencePoolMessage, err error) // EvidenceMessage contains a list of evidence. type EvidenceMessage struct { - Evidence types.Evidences + Evidence []types.Evidence } // String returns a string representation of the EvidenceMessage. diff --git a/evidence/store.go b/evidence/store.go new file mode 100644 index 00000000..82268391 --- /dev/null +++ b/evidence/store.go @@ -0,0 +1,100 @@ +package evpool + +import ( + "fmt" + + wire "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/types" + dbm "github.com/tendermint/tmlibs/db" +) + +/* +"evidence-lookup"// -> evidence struct +"evidence-outqueue"/// -> nil +"evidence-pending"//evidence-hash> -> nil +*/ + +var nullValue = []byte{0} + +type evidenceInfo struct { + Committed bool + Priority int + Evidence types.Evidence +} + +func keyLookup(evidence types.Evidence) []byte { + return []byte(fmt.Sprintf("evidence-lookup/%d/%X", evidence.Height(), evidence.Hash())) +} + +func keyOutqueue(idx int, evidence types.Evidence) []byte { + return []byte(fmt.Sprintf("evidence-outqueue/%d/%d/%X", idx, evidence.Height(), evidence.Hash())) +} + +func keyPending(evidence types.Evidence) []byte { + return []byte(fmt.Sprintf("evidence-pending/%d/%X", evidence.Height(), evidence.Hash())) +} + +// EvidenceStore stores all the evidence we've seen, including +// evidence that has been committed, evidence that has been seen but not broadcast, +// and evidence that has been broadcast but not yet committed. +type EvidenceStore struct { + chainID string + db dbm.DB +} + +func NewEvidenceStore(chainID string, db dbm.DB) *EvidenceStore { + return &EvidenceStore{ + chainID: chainID, + db: db, + } +} + +// AddNewEvidence adds the given evidence to the database. +func (store *EvidenceStore) AddNewEvidence(idx int, evidence types.Evidence) (bool, error) { + // check if we already have seen it + key := keyLookup(evidence) + v := store.db.Get(key) + if len(v) == 0 { + return false, nil + } + + // verify the evidence + if err := evidence.Verify(store.chainID); err != nil { + return false, err + } + + // add it to the store + ei := evidenceInfo{ + Committed: false, + Priority: idx, + Evidence: evidence, + } + store.db.Set(key, wire.BinaryBytes(ei)) + + key = keyOutqueue(idx, evidence) + store.db.Set(key, nullValue) + + key = keyPending(evidence) + store.db.Set(key, nullValue) + + return true, nil +} + +// MarkEvidenceAsBroadcasted removes evidence from the outqueue. +func (store *EvidenceStore) MarkEvidenceAsBroadcasted(idx int, evidence types.Evidence) { + key := keyOutqueue(idx, evidence) + store.db.Delete(key) +} + +// MarkEvidenceAsPending removes evidence from pending and sets the state to committed. +func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { + key := keyPending(evidence) + store.db.Delete(key) + + key = keyLookup(evidence) + var ei evidenceInfo + b := store.db.Get(key) + wire.ReadBinaryBytes(b, &ei) + ei.Committed = true + store.db.Set(key, wire.BinaryBytes(ei)) +} diff --git a/types/block.go b/types/block.go index b3e75442..96bfbf9d 100644 --- a/types/block.go +++ b/types/block.go @@ -437,7 +437,8 @@ func (data *Data) StringIndented(indent string) string { // EvidenceData contains any evidence of malicious wrong-doing by validators type EvidenceData struct { - Evidences Evidences `json:"evidence"` + // TODO: FIXME + Evidences evidences `json:"evidence"` // Volatile hash data.Bytes diff --git a/types/evidence.go b/types/evidence.go index d40c63ea..65f34e90 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -3,6 +3,7 @@ package types import ( "bytes" "fmt" + "sync" "github.com/tendermint/go-crypto" "github.com/tendermint/tmlibs/merkle" @@ -27,6 +28,7 @@ func (err *ErrEvidenceInvalid) Error() string { // Evidence represents any provable malicious activity by a validator type Evidence interface { + Height() int Address() []byte Hash() []byte Verify(chainID string) error @@ -37,9 +39,72 @@ type Evidence interface { //------------------------------------------- -type Evidences []Evidence +//EvidenceSet is a thread-safe set of evidence. +type EvidenceSet struct { + sync.RWMutex + evidences evidences +} -func (evs Evidences) Hash() []byte { +//Evidence returns a copy of all the evidence. +func (evset EvidenceSet) Evidence() []Evidence { + evset.RLock() + defer evset.RUnlock() + evCopy := make([]Evidence, len(evset.evidences)) + for i, ev := range evset.evidences { + evCopy[i] = ev + } + return evCopy +} + +// Size returns the number of pieces of evidence in the set. +func (evset EvidenceSet) Size() int { + evset.RLock() + defer evset.RUnlock() + return len(evset.evidences) +} + +// Hash returns a merkle hash of the evidence. +func (evset EvidenceSet) Hash() []byte { + evset.RLock() + defer evset.RUnlock() + return evset.evidences.Hash() +} + +// Has returns true if the given evidence is in the set. +func (evset EvidenceSet) Has(evidence Evidence) bool { + evset.RLock() + defer evset.RUnlock() + return evset.evidences.Has(evidence) +} + +// String returns a string representation of the evidence. +func (evset EvidenceSet) String() string { + evset.RLock() + defer evset.RUnlock() + return evset.evidences.String() +} + +// Add adds the given evidence to the set. +// TODO: and persists it to disk. +func (evset EvidenceSet) Add(evidence Evidence) { + evset.Lock() + defer evset.Unlock() + evset.evidences = append(evset.evidences, evidence) +} + +// Reset empties the evidence set. +func (evset EvidenceSet) Reset() { + evset.Lock() + defer evset.Unlock() + evset.evidences = make(evidences, 0) + +} + +//------------------------------------------- + +type evidences []Evidence + +func (evs evidences) Hash() []byte { // Recursive impl. // Copied from tmlibs/merkle to avoid allocations switch len(evs) { @@ -48,13 +113,13 @@ func (evs Evidences) Hash() []byte { case 1: return evs[0].Hash() default: - left := Evidences(evs[:(len(evs)+1)/2]).Hash() - right := Evidences(evs[(len(evs)+1)/2:]).Hash() + left := evidences(evs[:(len(evs)+1)/2]).Hash() + right := evidences(evs[(len(evs)+1)/2:]).Hash() return merkle.SimpleHashFromTwoHashes(left, right) } } -func (evs Evidences) String() string { +func (evs evidences) String() string { s := "" for _, e := range evs { s += fmt.Sprintf("%s\t\t", e) @@ -62,7 +127,7 @@ func (evs Evidences) String() string { return s } -func (evs Evidences) Has(evidence Evidence) bool { +func (evs evidences) Has(evidence Evidence) bool { for _, ev := range evs { if ev.Equal(evidence) { return true @@ -86,6 +151,11 @@ func (dve *DuplicateVoteEvidence) String() string { } +// Height returns the height this evidence refers to. +func (dve *DuplicateVoteEvidence) Height() int { + return dve.VoteA.Height +} + // Address returns the address of the validator. func (dve *DuplicateVoteEvidence) Address() []byte { return dve.PubKey.Address() From df3f4de7c3eaa9c4e5697b8d1025fc6e5ab9d5f2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 3 Nov 2017 01:50:05 -0600 Subject: [PATCH 15/35] check evidence is from validator; some cleanup --- evidence/evidence_pool.go | 9 ++--- evidence/reactor.go | 12 +++---- evidence/store.go | 74 ++++++++++++++++++++++++++++++--------- state/execution.go | 3 +- types/evidence.go | 37 ++++++++++++++++---- 5 files changed, 99 insertions(+), 36 deletions(-) diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go index 3a31c40c..1068b997 100644 --- a/evidence/evidence_pool.go +++ b/evidence/evidence_pool.go @@ -42,20 +42,17 @@ func (evpool *EvidencePool) NewEvidenceChan() chan types.Evidence { // PriorityEvidence returns the priority evidence. func (evpool *EvidencePool) PriorityEvidence() []types.Evidence { - // TODO - return nil + return evpool.evidenceStore.PriorityEvidence() } // PendingEvidence returns all uncommitted evidence. func (evpool *EvidencePool) PendingEvidence() []types.Evidence { - // TODO - return nil + return evpool.evidenceStore.PendingEvidence() } // AddEvidence checks the evidence is valid and adds it to the pool. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { - idx := 1 // TODO - added, err := evpool.evidenceStore.AddNewEvidence(idx, evidence) + added, err := evpool.evidenceStore.AddNewEvidence(evidence) if err != nil { return err } else if !added { diff --git a/evidence/reactor.go b/evidence/reactor.go index d62e88b3..d52949aa 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -67,20 +67,19 @@ func (evR *EvidencePoolReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor. func (evR *EvidencePoolReactor) AddPeer(peer p2p.Peer) { - // first send the peer high-priority evidence + // send the peer our high-priority evidence. + // the rest will be sent by the broadcastRoutine evidence := evR.evpool.PriorityEvidence() msg := EvidenceMessage{evidence} success := peer.Send(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) if !success { // TODO: remove peer ? } - - // TODO: send the remaining pending evidence - // or just let the broadcastRoutine do it ? } // RemovePeer implements Reactor. func (evR *EvidencePoolReactor) RemovePeer(peer p2p.Peer, reason interface{}) { + // nothing to do } // Receive implements Reactor. @@ -122,10 +121,9 @@ func (evR *EvidencePoolReactor) broadcastRoutine() { msg := EvidenceMessage{[]types.Evidence{evidence}} evR.Switch.Broadcast(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) - // NOTE: Broadcast runs asynchronously, so this should wait on the successChan + // TODO: Broadcast runs asynchronously, so this should wait on the successChan // in another routine before marking to be proper. - idx := 1 // TODO - evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(idx, evidence) + evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(evidence) case <-ticker.C: // broadcast all pending evidence msg := EvidenceMessage{evR.evpool.PendingEvidence()} diff --git a/evidence/store.go b/evidence/store.go index 82268391..e4a20665 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -22,16 +22,22 @@ type evidenceInfo struct { Evidence types.Evidence } +const ( + baseKeyLookup = "evidence-lookup" + baseKeyOutqueue = "evidence-outqueue" + baseKeyPending = "evidence-pending" +) + func keyLookup(evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("evidence-lookup/%d/%X", evidence.Height(), evidence.Hash())) + return []byte(fmt.Sprintf("%s/%d/%X", baseKeyLookup, evidence.Height(), evidence.Hash())) } -func keyOutqueue(idx int, evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("evidence-outqueue/%d/%d/%X", idx, evidence.Height(), evidence.Hash())) +func keyOutqueue(evidence types.Evidence) []byte { + return []byte(fmt.Sprintf("%s/%d/%X", baseKeyOutqueue, evidence.Height(), evidence.Hash())) } func keyPending(evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("evidence-pending/%d/%X", evidence.Height(), evidence.Hash())) + return []byte(fmt.Sprintf("%s/%d/%X", baseKeyPending, evidence.Height(), evidence.Hash())) } // EvidenceStore stores all the evidence we've seen, including @@ -40,49 +46,85 @@ func keyPending(evidence types.Evidence) []byte { type EvidenceStore struct { chainID string db dbm.DB + + historicalValidators types.HistoricalValidators } func NewEvidenceStore(chainID string, db dbm.DB) *EvidenceStore { return &EvidenceStore{ chainID: chainID, db: db, + // TODO historicalValidators } } +// PriorityEvidence returns the evidence from the outqueue, sorted by highest priority. +func (store *EvidenceStore) PriorityEvidence() (evidence []types.Evidence) { + iter := store.db.IteratorPrefix([]byte(baseKeyOutqueue)) + for iter.Next() { + val := iter.Value() + + var ei evidenceInfo + wire.ReadBinaryBytes(val, &ei) + evidence = append(evidence, ei.Evidence) + } + // TODO: sort + return evidence +} + +func (store *EvidenceStore) PendingEvidence() (evidence []types.Evidence) { + iter := store.db.IteratorPrefix([]byte(baseKeyPending)) + for iter.Next() { + val := iter.Value() + + var ei evidenceInfo + wire.ReadBinaryBytes(val, &ei) + evidence = append(evidence, ei.Evidence) + } + return evidence +} + // AddNewEvidence adds the given evidence to the database. -func (store *EvidenceStore) AddNewEvidence(idx int, evidence types.Evidence) (bool, error) { +func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence) (bool, error) { // check if we already have seen it key := keyLookup(evidence) v := store.db.Get(key) - if len(v) == 0 { + if len(v) != 0 { return false, nil } - // verify the evidence - if err := evidence.Verify(store.chainID); err != nil { + // verify evidence consistency + if err := evidence.Verify(store.chainID, store.historicalValidators); err != nil { return false, err } - // add it to the store + // TODO: or we let Verify return the val to avoid running this again? + valSet := store.historicalValidators.LoadValidators(evidence.Height()) + _, val := valSet.GetByAddress(evidence.Address()) + priority := int(val.VotingPower) + ei := evidenceInfo{ Committed: false, - Priority: idx, + Priority: priority, Evidence: evidence, } - store.db.Set(key, wire.BinaryBytes(ei)) + eiBytes := wire.BinaryBytes(ei) - key = keyOutqueue(idx, evidence) - store.db.Set(key, nullValue) + // add it to the store + store.db.Set(key, eiBytes) + + key = keyOutqueue(evidence) + store.db.Set(key, eiBytes) key = keyPending(evidence) - store.db.Set(key, nullValue) + store.db.Set(key, eiBytes) return true, nil } // MarkEvidenceAsBroadcasted removes evidence from the outqueue. -func (store *EvidenceStore) MarkEvidenceAsBroadcasted(idx int, evidence types.Evidence) { - key := keyOutqueue(idx, evidence) +func (store *EvidenceStore) MarkEvidenceAsBroadcasted(evidence types.Evidence) { + key := keyOutqueue(evidence) store.db.Delete(key) } diff --git a/state/execution.go b/state/execution.go index adc6c139..bdc4bd52 100644 --- a/state/execution.go +++ b/state/execution.go @@ -310,9 +310,10 @@ func (s *State) validateBlock(b *types.Block) error { } for _, ev := range block.Evidence.Evidences { - if err := ev.Verify(s.ChainID); err != nil { + if err := ev.Verify(s.ChainID, s); err != nil { return types.NewEvidenceInvalidErr(ev, err) } + // TODO: mark evidence as committed } return nil diff --git a/types/evidence.go b/types/evidence.go index 65f34e90..2021269b 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -26,13 +26,18 @@ func (err *ErrEvidenceInvalid) Error() string { //------------------------------------------- +type HistoricalValidators interface { + LoadValidators(height int) *ValidatorSet +} + // Evidence represents any provable malicious activity by a validator type Evidence interface { - Height() int - Address() []byte - Hash() []byte - Verify(chainID string) error - Equal(Evidence) bool + Height() int // height of the equivocation + Address() []byte // address of the equivocating validator + Index() int // index of the validator in the validator set + Hash() []byte // hash of the evidence + Verify(chainID string, vals HistoricalValidators) error // verify the evidence + Equal(Evidence) bool // check equality of evidence String() string } @@ -161,6 +166,11 @@ func (dve *DuplicateVoteEvidence) Address() []byte { return dve.PubKey.Address() } +// Index returns the index of the validator. +func (dve *DuplicateVoteEvidence) Index() int { + return dve.VoteA.ValidatorIndex +} + // Hash returns the hash of the evidence. func (dve *DuplicateVoteEvidence) Hash() []byte { return merkle.SimpleHashFromBinary(dve) @@ -168,7 +178,10 @@ func (dve *DuplicateVoteEvidence) Hash() []byte { // Verify returns an error if the two votes aren't conflicting. // To be conflicting, they must be from the same validator, for the same H/R/S, but for different blocks. -func (dve *DuplicateVoteEvidence) Verify(chainID string) error { +func (dve *DuplicateVoteEvidence) Verify(chainID string, vals HistoricalValidators) error { + + // TODO: verify (cs.Height - dve.Height) < MaxHeightDiff + // H/R/S must be the same if dve.VoteA.Height != dve.VoteB.Height || dve.VoteA.Round != dve.VoteB.Round || @@ -198,6 +211,18 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string) error { return ErrVoteInvalidSignature } + // The address must have been an active validator at the height + height := dve.Height() + addr := dve.Address() + idx := dve.Index() + valset := vals.LoadValidators(height) + valIdx, val := valset.GetByAddress(addr) + if val == nil { + return fmt.Errorf("Address %X was not a validator at height %d", addr, height) + } else if idx != valIdx { + return fmt.Errorf("Address %X was validator %d at height %d, not %d", addr, valIdx, height, idx) + } + return nil } From f7731d38f64023a70790f6009929ba156c848765 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 00:57:55 +0000 Subject: [PATCH 16/35] some comments and cleanup --- consensus/state.go | 6 +----- evidence/evidence_pool.go | 26 +++++++++++++------------- evidence/reactor.go | 5 +++-- evidence/store.go | 34 +++++++++++++++++++++++----------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 7b7c04c4..99bc4809 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1237,10 +1237,6 @@ func (cs *ConsensusState) finalizeCommit(height int64) { fail.Fail() // XXX - // TODO: remove included evidence - // and persist remaining evidence - // ... is this the right spot? need to ensure we never lose evidence - // NewHeightStep! cs.updateToState(stateCopy) @@ -1333,7 +1329,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { _, err := cs.addVote(vote, peerKey) if err != nil { // If the vote height is off, we'll just ignore it, - // But if it's a conflicting sig, broadcast evidence tx for slashing. + // But if it's a conflicting sig, add it to the cs.evpool. // If it's otherwise invalid, punish peer. if err == ErrVoteHeightMismatch { return err diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go index 1068b997..cc9e01e7 100644 --- a/evidence/evidence_pool.go +++ b/evidence/evidence_pool.go @@ -6,15 +6,14 @@ import ( "github.com/tendermint/tendermint/types" ) -const cacheSize = 100000 - -// EvidencePool maintains a set of valid uncommitted evidence. +// EvidencePool maintains a pool of valid evidence +// in an EvidenceStore. type EvidencePool struct { config *EvidencePoolConfig logger log.Logger - evidenceStore *EvidenceStore - newEvidenceChan chan types.Evidence + evidenceStore *EvidenceStore + evidenceChan chan types.Evidence } type EvidencePoolConfig struct { @@ -22,10 +21,10 @@ type EvidencePoolConfig struct { func NewEvidencePool(config *EvidencePoolConfig, evidenceStore *EvidenceStore) *EvidencePool { evpool := &EvidencePool{ - config: config, - logger: log.NewNopLogger(), - evidenceStore: evidenceStore, - newEvidenceChan: make(chan types.Evidence), + config: config, + logger: log.NewNopLogger(), + evidenceStore: evidenceStore, + evidenceChan: make(chan types.Evidence), } return evpool } @@ -35,9 +34,9 @@ func (evpool *EvidencePool) SetLogger(l log.Logger) { evpool.logger = l } -// NewEvidenceChan returns a channel on which new evidence is sent. -func (evpool *EvidencePool) NewEvidenceChan() chan types.Evidence { - return evpool.newEvidenceChan +// EvidenceChan returns an unbuffered channel on which new evidence can be received. +func (evpool *EvidencePool) EvidenceChan() chan types.Evidence { + return evpool.evidenceChan } // PriorityEvidence returns the priority evidence. @@ -51,6 +50,7 @@ func (evpool *EvidencePool) PendingEvidence() []types.Evidence { } // AddEvidence checks the evidence is valid and adds it to the pool. +// Blocks on the EvidenceChan. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { added, err := evpool.evidenceStore.AddNewEvidence(evidence) if err != nil { @@ -62,7 +62,7 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence) - evpool.newEvidenceChan <- evidence + evpool.evidenceChan <- evidence return nil } diff --git a/evidence/reactor.go b/evidence/reactor.go index d52949aa..7e086079 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -111,12 +111,13 @@ func (evR *EvidencePoolReactor) SetEventSwitch(evsw types.EventSwitch) { evR.evsw = evsw } -// broadcast new evidence to all peers +// broadcast new evidence to all peers. +// broadcasts must be non-blocking so routine is always available to read off EvidenceChan. func (evR *EvidencePoolReactor) broadcastRoutine() { ticker := time.NewTicker(time.Second * broadcastEvidenceIntervalS) for { select { - case evidence := <-evR.evpool.NewEvidenceChan(): + case evidence := <-evR.evpool.EvidenceChan(): // broadcast some new evidence msg := EvidenceMessage{[]types.Evidence{evidence}} evR.Switch.Broadcast(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) diff --git a/evidence/store.go b/evidence/store.go index e4a20665..f4587dd7 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -9,9 +9,12 @@ import ( ) /* +Schema for indexing evidence: + "evidence-lookup"// -> evidence struct "evidence-outqueue"/// -> nil "evidence-pending"//evidence-hash> -> nil + */ var nullValue = []byte{0} @@ -23,21 +26,25 @@ type evidenceInfo struct { } const ( - baseKeyLookup = "evidence-lookup" - baseKeyOutqueue = "evidence-outqueue" - baseKeyPending = "evidence-pending" + baseKeyLookup = "evidence-lookup" // all evidence + baseKeyOutqueue = "evidence-outqueue" // not-yet broadcast + baseKeyPending = "evidence-pending" // broadcast but not committed ) func keyLookup(evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("%s/%d/%X", baseKeyLookup, evidence.Height(), evidence.Hash())) + return _key(baseKeyLookup, evidence) } func keyOutqueue(evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("%s/%d/%X", baseKeyOutqueue, evidence.Height(), evidence.Hash())) + return _key(baseKeyOutqueue, evidence) } func keyPending(evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("%s/%d/%X", baseKeyPending, evidence.Height(), evidence.Hash())) + return _key(baseKeyPending, evidence) +} + +func _key(key string, evidence types.Evidence) []byte { + return []byte(fmt.Sprintf("%s/%d/%X", key, evidence.Height(), evidence.Hash())) } // EvidenceStore stores all the evidence we've seen, including @@ -47,14 +54,15 @@ type EvidenceStore struct { chainID string db dbm.DB + // so we can verify evidence was from a real validator historicalValidators types.HistoricalValidators } -func NewEvidenceStore(chainID string, db dbm.DB) *EvidenceStore { +func NewEvidenceStore(chainID string, db dbm.DB, vals types.HistoricalValidators) *EvidenceStore { return &EvidenceStore{ - chainID: chainID, - db: db, - // TODO historicalValidators + chainID: chainID, + db: db, + historicalValidators: vals, } } @@ -111,6 +119,7 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence) (bool, error eiBytes := wire.BinaryBytes(ei) // add it to the store + key = keyLookup(evidence) store.db.Set(key, eiBytes) key = keyOutqueue(evidence) @@ -128,8 +137,11 @@ func (store *EvidenceStore) MarkEvidenceAsBroadcasted(evidence types.Evidence) { store.db.Delete(key) } -// MarkEvidenceAsPending removes evidence from pending and sets the state to committed. +// MarkEvidenceAsPending removes evidence from pending and outqueue and sets the state to committed. func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { + // if its committed, its been broadcast + store.MarkEvidenceAsBroadcasted(evidence) + key := keyPending(evidence) store.db.Delete(key) From 6c4a0f936397cb4e2d1f0e6c3f7faf7904d37d1c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 01:34:11 +0000 Subject: [PATCH 17/35] cleanup evidence pkg. state.VerifyEvidence --- evidence/reactor.go | 81 +++++++++++++++++++++++---------------------- evidence/store.go | 19 ++++------- state/execution.go | 2 +- state/state.go | 31 +++++++++++++++++ types/evidence.go | 28 +++++----------- types/services.go | 29 ++++++++++++++-- 6 files changed, 115 insertions(+), 75 deletions(-) diff --git a/evidence/reactor.go b/evidence/reactor.go index 7e086079..f6e7501a 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -9,44 +9,45 @@ import ( wire "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/log" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) const ( - EvidencePoolChannel = byte(0x38) + EvidenceChannel = byte(0x38) - maxEvidencePoolMessageSize = 1048576 // 1MB TODO make it configurable + maxEvidenceMessageSize = 1048576 // 1MB TODO make it configurable peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount broadcastEvidenceIntervalS = 60 // broadcast uncommitted evidence this often ) -// EvidencePoolReactor handles evpool evidence broadcasting amongst peers. -type EvidencePoolReactor struct { +// EvidenceReactor handles evpool evidence broadcasting amongst peers. +type EvidenceReactor struct { p2p.BaseReactor - config *EvidencePoolConfig - evpool *EvidencePool - evsw types.EventSwitch + config *cfg.EvidenceConfig + evpool *EvidencePool + eventBus *types.EventBus } -// NewEvidencePoolReactor returns a new EvidencePoolReactor with the given config and evpool. -func NewEvidencePoolReactor(config *EvidencePoolConfig, evpool *EvidencePool) *EvidencePoolReactor { - evR := &EvidencePoolReactor{ +// NewEvidenceReactor returns a new EvidenceReactor with the given config and evpool. +func NewEvidenceReactor(config *cfg.EvidenceConfig, evpool *EvidencePool) *EvidenceReactor { + evR := &EvidenceReactor{ config: config, evpool: evpool, } - evR.BaseReactor = *p2p.NewBaseReactor("EvidencePoolReactor", evR) + evR.BaseReactor = *p2p.NewBaseReactor("EvidenceReactor", evR) return evR } -// SetLogger sets the Logger on the reactor and the underlying EvidencePool. -func (evR *EvidencePoolReactor) SetLogger(l log.Logger) { +// SetLogger sets the Logger on the reactor and the underlying Evidence. +func (evR *EvidenceReactor) SetLogger(l log.Logger) { evR.Logger = l evR.evpool.SetLogger(l) } // OnStart implements cmn.Service -func (evR *EvidencePoolReactor) OnStart() error { +func (evR *EvidenceReactor) OnStart() error { if err := evR.BaseReactor.OnStart(); err != nil { return err } @@ -56,35 +57,35 @@ func (evR *EvidencePoolReactor) OnStart() error { // GetChannels implements Reactor. // It returns the list of channels for this reactor. -func (evR *EvidencePoolReactor) GetChannels() []*p2p.ChannelDescriptor { +func (evR *EvidenceReactor) GetChannels() []*p2p.ChannelDescriptor { return []*p2p.ChannelDescriptor{ &p2p.ChannelDescriptor{ - ID: EvidencePoolChannel, + ID: EvidenceChannel, Priority: 5, }, } } // AddPeer implements Reactor. -func (evR *EvidencePoolReactor) AddPeer(peer p2p.Peer) { +func (evR *EvidenceReactor) AddPeer(peer p2p.Peer) { // send the peer our high-priority evidence. // the rest will be sent by the broadcastRoutine evidence := evR.evpool.PriorityEvidence() - msg := EvidenceMessage{evidence} - success := peer.Send(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) + msg := EvidenceListMessage{evidence} + success := peer.Send(EvidenceChannel, struct{ EvidenceMessage }{msg}) if !success { // TODO: remove peer ? } } // RemovePeer implements Reactor. -func (evR *EvidencePoolReactor) RemovePeer(peer p2p.Peer, reason interface{}) { +func (evR *EvidenceReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // nothing to do } // Receive implements Reactor. // It adds any received evidence to the evpool. -func (evR *EvidencePoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { +func (evR *EvidenceReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { _, msg, err := DecodeMessage(msgBytes) if err != nil { evR.Logger.Error("Error decoding message", "err", err) @@ -93,7 +94,7 @@ func (evR *EvidencePoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte evR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) switch msg := msg.(type) { - case *EvidenceMessage: + case *EvidenceListMessage: for _, ev := range msg.Evidence { err := evR.evpool.AddEvidence(ev) if err != nil { @@ -107,28 +108,28 @@ func (evR *EvidencePoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte } // SetEventSwitch implements events.Eventable. -func (evR *EvidencePoolReactor) SetEventSwitch(evsw types.EventSwitch) { - evR.evsw = evsw +func (evR *EvidenceReactor) SetEventBus(b *types.EventBus) { + evR.eventBus = b } // broadcast new evidence to all peers. // broadcasts must be non-blocking so routine is always available to read off EvidenceChan. -func (evR *EvidencePoolReactor) broadcastRoutine() { +func (evR *EvidenceReactor) broadcastRoutine() { ticker := time.NewTicker(time.Second * broadcastEvidenceIntervalS) for { select { case evidence := <-evR.evpool.EvidenceChan(): // broadcast some new evidence - msg := EvidenceMessage{[]types.Evidence{evidence}} - evR.Switch.Broadcast(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) + msg := EvidenceListMessage{[]types.Evidence{evidence}} + evR.Switch.Broadcast(EvidenceChannel, struct{ EvidenceMessage }{msg}) // TODO: Broadcast runs asynchronously, so this should wait on the successChan // in another routine before marking to be proper. evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(evidence) case <-ticker.C: // broadcast all pending evidence - msg := EvidenceMessage{evR.evpool.PendingEvidence()} - evR.Switch.Broadcast(EvidencePoolChannel, struct{ EvidencePoolMessage }{msg}) + msg := EvidenceListMessage{evR.evpool.PendingEvidence()} + evR.Switch.Broadcast(EvidenceChannel, struct{ EvidenceMessage }{msg}) case <-evR.Quit: return } @@ -142,31 +143,31 @@ const ( msgTypeEvidence = byte(0x01) ) -// EvidencePoolMessage is a message sent or received by the EvidencePoolReactor. -type EvidencePoolMessage interface{} +// EvidenceMessage is a message sent or received by the EvidenceReactor. +type EvidenceMessage interface{} var _ = wire.RegisterInterface( - struct{ EvidencePoolMessage }{}, - wire.ConcreteType{&EvidenceMessage{}, msgTypeEvidence}, + struct{ EvidenceMessage }{}, + wire.ConcreteType{&EvidenceListMessage{}, msgTypeEvidence}, ) -// DecodeMessage decodes a byte-array into a EvidencePoolMessage. -func DecodeMessage(bz []byte) (msgType byte, msg EvidencePoolMessage, err error) { +// DecodeMessage decodes a byte-array into a EvidenceMessage. +func DecodeMessage(bz []byte) (msgType byte, msg EvidenceMessage, err error) { msgType = bz[0] n := new(int) r := bytes.NewReader(bz) - msg = wire.ReadBinary(struct{ EvidencePoolMessage }{}, r, maxEvidencePoolMessageSize, n, &err).(struct{ EvidencePoolMessage }).EvidencePoolMessage + msg = wire.ReadBinary(struct{ EvidenceMessage }{}, r, maxEvidenceMessageSize, n, &err).(struct{ EvidenceMessage }).EvidenceMessage return } //------------------------------------- // EvidenceMessage contains a list of evidence. -type EvidenceMessage struct { +type EvidenceListMessage struct { Evidence []types.Evidence } -// String returns a string representation of the EvidenceMessage. -func (m *EvidenceMessage) String() string { - return fmt.Sprintf("[EvidenceMessage %v]", m.Evidence) +// String returns a string representation of the EvidenceListMessage. +func (m *EvidenceListMessage) String() string { + return fmt.Sprintf("[EvidenceListMessage %v]", m.Evidence) } diff --git a/evidence/store.go b/evidence/store.go index f4587dd7..0748a721 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -55,14 +55,14 @@ type EvidenceStore struct { db dbm.DB // so we can verify evidence was from a real validator - historicalValidators types.HistoricalValidators + state types.State } -func NewEvidenceStore(chainID string, db dbm.DB, vals types.HistoricalValidators) *EvidenceStore { +func NewEvidenceStore(chainID string, db dbm.DB, state types.State) *EvidenceStore { return &EvidenceStore{ - chainID: chainID, - db: db, - historicalValidators: vals, + chainID: chainID, + db: db, + state: state, } } @@ -101,16 +101,11 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence) (bool, error return false, nil } - // verify evidence consistency - if err := evidence.Verify(store.chainID, store.historicalValidators); err != nil { + priority, err := store.state.VerifyEvidence(evidence) + if err != nil { return false, err } - // TODO: or we let Verify return the val to avoid running this again? - valSet := store.historicalValidators.LoadValidators(evidence.Height()) - _, val := valSet.GetByAddress(evidence.Address()) - priority := int(val.VotingPower) - ei := evidenceInfo{ Committed: false, Priority: priority, diff --git a/state/execution.go b/state/execution.go index bdc4bd52..860c0caa 100644 --- a/state/execution.go +++ b/state/execution.go @@ -310,7 +310,7 @@ func (s *State) validateBlock(b *types.Block) error { } for _, ev := range block.Evidence.Evidences { - if err := ev.Verify(s.ChainID, s); err != nil { + if _, err := s.VerifyEvidence(ev); err != nil { return types.NewEvidenceInvalidErr(ev, err) } // TODO: mark evidence as committed diff --git a/state/state.go b/state/state.go index 0dd105cb..78888152 100644 --- a/state/state.go +++ b/state/state.go @@ -195,6 +195,7 @@ func (s *State) LoadABCIResponses(height int64) (*ABCIResponses, error) { } // LoadValidators loads the ValidatorSet for a given height. +// Returns ErrNoValSetForHeight if the validator set can't be found for this height. func (s *State) LoadValidators(height int64) (*types.ValidatorSet, error) { valInfo := s.loadValidatorsInfo(height) if valInfo == nil { @@ -382,6 +383,36 @@ func (s *State) GetValidators() (last *types.ValidatorSet, current *types.Valida return s.LastValidators, s.Validators } +// VerifyEvidence verifies the evidence fully by checking it is internally +// consistent and corresponds to an existing or previous validator. +// It returns the priority of this evidence, or an error. +// NOTE: return error may be ErrLoadValidators, in which case the validator set +// for the evidence height could not be loaded. +func (s *State) VerifyEvidence(evidence types.Evidence) (priority int, err error) { + if err := evidence.Verify(s.ChainID); err != nil { + return priority, err + } + + // The address must have been an active validator at the height + ev := evidence + height, addr, idx := ev.Height(), ev.Address(), ev.Index() + valset, err := s.LoadValidators(height) + if err != nil { + // XXX/TODO: what do we do if we can't load the valset? + // eg. if we have pruned the state or height is too high? + return priority, err + } + valIdx, val := valset.GetByAddress(addr) + if val == nil { + return priority, fmt.Errorf("Address %X was not a validator at height %d", addr, height) + } else if idx != valIdx { + return priority, fmt.Errorf("Address %X was validator %d at height %d, not %d", addr, valIdx, height, idx) + } + + priority = int(val.VotingPower) + return priority, nil +} + //------------------------------------------------------------------------ // ABCIResponses retains the responses diff --git a/types/evidence.go b/types/evidence.go index 2021269b..42047faf 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -27,17 +27,17 @@ func (err *ErrEvidenceInvalid) Error() string { //------------------------------------------- type HistoricalValidators interface { - LoadValidators(height int) *ValidatorSet + LoadValidators(height int) (*ValidatorSet, error) } // Evidence represents any provable malicious activity by a validator type Evidence interface { - Height() int // height of the equivocation - Address() []byte // address of the equivocating validator - Index() int // index of the validator in the validator set - Hash() []byte // hash of the evidence - Verify(chainID string, vals HistoricalValidators) error // verify the evidence - Equal(Evidence) bool // check equality of evidence + Height() int // height of the equivocation + Address() []byte // address of the equivocating validator + Index() int // index of the validator in the validator set + Hash() []byte // hash of the evidence + Verify(chainID string) error // verify the evidence + Equal(Evidence) bool // check equality of evidence String() string } @@ -178,7 +178,7 @@ func (dve *DuplicateVoteEvidence) Hash() []byte { // Verify returns an error if the two votes aren't conflicting. // To be conflicting, they must be from the same validator, for the same H/R/S, but for different blocks. -func (dve *DuplicateVoteEvidence) Verify(chainID string, vals HistoricalValidators) error { +func (dve *DuplicateVoteEvidence) Verify(chainID string) error { // TODO: verify (cs.Height - dve.Height) < MaxHeightDiff @@ -211,18 +211,6 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string, vals HistoricalValidato return ErrVoteInvalidSignature } - // The address must have been an active validator at the height - height := dve.Height() - addr := dve.Address() - idx := dve.Index() - valset := vals.LoadValidators(height) - valIdx, val := valset.GetByAddress(addr) - if val == nil { - return fmt.Errorf("Address %X was not a validator at height %d", addr, height) - } else if idx != valIdx { - return fmt.Errorf("Address %X was validator %d at height %d, not %d", addr, valIdx, height, idx) - } - return nil } diff --git a/types/services.go b/types/services.go index 0e007554..f1d8ddd5 100644 --- a/types/services.go +++ b/types/services.go @@ -14,7 +14,7 @@ import ( //------------------------------------------------------ // mempool -// Mempool defines the mempool interface. +// Mempool defines the mempool interface as used by the ConsensusState. // Updates to the mempool need to be synchronized with committing a block // so apps can reset their transient state on Commit // UNSTABLE @@ -63,9 +63,34 @@ type BlockStoreRPC interface { LoadSeenCommit(height int64) *Commit } -// BlockStore defines the BlockStore interface. +// BlockStore defines the BlockStore interface used by the ConsensusState. // UNSTABLE type BlockStore interface { BlockStoreRPC SaveBlock(block *Block, blockParts *PartSet, seenCommit *Commit) } + +//------------------------------------------------------ +// state + +type State interface { + VerifyEvidence(Evidence) (priority int, err error) +} + +//------------------------------------------------------ +// evidence pool + +// EvidencePool defines the EvidencePool interface used by the ConsensusState. +// UNSTABLE +type EvidencePool interface { + PendingEvidence() []Evidence + AddEvidence(Evidence) +} + +// MockMempool is an empty implementation of a Mempool, useful for testing. +// UNSTABLE +type MockEvidencePool struct { +} + +func (m MockEvidencePool) PendingEvidence() []Evidence { return nil } +func (m MockEvidencePool) AddEvidence(Evidence) {} From 7a18fa887d4bc984141357994a113ec11e686f1e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 02:02:58 +0000 Subject: [PATCH 18/35] evidence linked with consensus/node. compiles --- consensus/common_test.go | 5 ++++- consensus/replay_file.go | 6 ++++-- consensus/state.go | 9 +++++---- evidence/evidence_pool.go | 21 ++++++++++++++------- evidence/store.go | 19 ++++--------------- node/node.go | 31 +++++++++++++++++++++++++++---- rpc/core/pipe.go | 5 +++++ types/services.go | 4 ++-- 8 files changed, 65 insertions(+), 35 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index dd6fce66..6598c15e 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -260,8 +260,11 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state *sm. mempool.EnableTxsAvailable() } + // mock the evidence pool + evpool := types.MockEvidencePool{} + // Make ConsensusReactor - cs := NewConsensusState(thisConfig.Consensus, state, proxyAppConnCon, blockStore, mempool) + cs := NewConsensusState(thisConfig.Consensus, state, proxyAppConnCon, blockStore, mempool, evpool) cs.SetLogger(log.TestingLogger()) cs.SetPrivValidator(pv) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index d291e87c..4db58ada 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -123,7 +123,8 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { pb.cs.Stop() pb.cs.Wait() - newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool) + newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, + pb.cs.blockStore, pb.cs.mempool, pb.cs.evpool) newCS.SetEventBus(pb.cs.eventBus) newCS.startForReplay() @@ -302,7 +303,8 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo cmn.Exit(cmn.Fmt("Failed to start event bus: %v", err)) } - consensusState := NewConsensusState(csConfig, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{}) + consensusState := NewConsensusState(csConfig, state.Copy(), proxyApp.Consensus(), + blockStore, types.MockMempool{}, types.MockEvidencePool{}) consensusState.SetEventBus(eventBus) return consensusState diff --git a/consensus/state.go b/consensus/state.go index 99bc4809..7531f197 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -17,7 +17,6 @@ import ( cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" - evpool "github.com/tendermint/tendermint/evidence" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -79,7 +78,7 @@ type ConsensusState struct { proxyAppConn proxy.AppConnConsensus blockStore types.BlockStore mempool types.Mempool - evpool evpool.EvidencePool + evpool types.EvidencePool // internal state mtx sync.Mutex @@ -115,7 +114,7 @@ type ConsensusState struct { } // NewConsensusState returns a new ConsensusState. -func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { +func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool, evpool types.EvidencePool) *ConsensusState { cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, @@ -127,7 +126,7 @@ func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppCon done: make(chan struct{}), doWALCatchup: true, wal: nilWAL{}, - // evpool: evpool, + evpool: evpool, } // set function defaults (may be overwritten before calling Start) cs.decideProposal = cs.defaultDecideProposal @@ -1237,6 +1236,8 @@ func (cs *ConsensusState) finalizeCommit(height int64) { fail.Fail() // XXX + // TODO: cs.evpool.Update() + // NewHeightStep! cs.updateToState(stateCopy) diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go index cc9e01e7..41f51ee9 100644 --- a/evidence/evidence_pool.go +++ b/evidence/evidence_pool.go @@ -3,27 +3,28 @@ package evpool import ( "github.com/tendermint/tmlibs/log" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" ) // EvidencePool maintains a pool of valid evidence // in an EvidenceStore. type EvidencePool struct { - config *EvidencePoolConfig + config *cfg.EvidenceConfig logger log.Logger + state types.State evidenceStore *EvidenceStore - evidenceChan chan types.Evidence + + evidenceChan chan types.Evidence } -type EvidencePoolConfig struct { -} - -func NewEvidencePool(config *EvidencePoolConfig, evidenceStore *EvidenceStore) *EvidencePool { +func NewEvidencePool(config *cfg.EvidenceConfig, evidenceStore *EvidenceStore, state types.State) *EvidencePool { evpool := &EvidencePool{ config: config, logger: log.NewNopLogger(), evidenceStore: evidenceStore, + state: state, evidenceChan: make(chan types.Evidence), } return evpool @@ -52,7 +53,13 @@ func (evpool *EvidencePool) PendingEvidence() []types.Evidence { // AddEvidence checks the evidence is valid and adds it to the pool. // Blocks on the EvidenceChan. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { - added, err := evpool.evidenceStore.AddNewEvidence(evidence) + + priority, err := evpool.state.VerifyEvidence(evidence) + if err != nil { + return err + } + + added, err := evpool.evidenceStore.AddNewEvidence(evidence, priority) if err != nil { return err } else if !added { diff --git a/evidence/store.go b/evidence/store.go index 0748a721..e186aea5 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -51,18 +51,12 @@ func _key(key string, evidence types.Evidence) []byte { // evidence that has been committed, evidence that has been seen but not broadcast, // and evidence that has been broadcast but not yet committed. type EvidenceStore struct { - chainID string - db dbm.DB - - // so we can verify evidence was from a real validator - state types.State + db dbm.DB } -func NewEvidenceStore(chainID string, db dbm.DB, state types.State) *EvidenceStore { +func NewEvidenceStore(db dbm.DB) *EvidenceStore { return &EvidenceStore{ - chainID: chainID, - db: db, - state: state, + db: db, } } @@ -93,7 +87,7 @@ func (store *EvidenceStore) PendingEvidence() (evidence []types.Evidence) { } // AddNewEvidence adds the given evidence to the database. -func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence) (bool, error) { +func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) (bool, error) { // check if we already have seen it key := keyLookup(evidence) v := store.db.Get(key) @@ -101,11 +95,6 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence) (bool, error return false, nil } - priority, err := store.state.VerifyEvidence(evidence) - if err != nil { - return false, err - } - ei := evidenceInfo{ Committed: false, Priority: priority, diff --git a/node/node.go b/node/node.go index 352c13cc..0d20df66 100644 --- a/node/node.go +++ b/node/node.go @@ -19,6 +19,7 @@ import ( bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/consensus" + evidence "github.com/tendermint/tendermint/evidence" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/p2p/trust" @@ -107,6 +108,7 @@ type Node struct { mempoolReactor *mempl.MempoolReactor // for gossipping transactions consensusState *consensus.ConsensusState // latest consensus state consensusReactor *consensus.ConsensusReactor // for participating in the consensus + evidencePool *evidence.EvidencePool // tracking evidence proxyApp proxy.AppConns // connection to the application rpcListeners []net.Listener // rpc servers txIndexer txindex.TxIndexer @@ -128,9 +130,6 @@ func NewNode(config *cfg.Config, } blockStore := bc.NewBlockStore(blockStoreDB) - consensusLogger := logger.With("module", "consensus") - stateLogger := logger.With("module", "state") - // Get State stateDB, err := dbProvider(&DBContext{"state", config}) if err != nil { @@ -149,6 +148,7 @@ func NewNode(config *cfg.Config, saveGenesisDoc(stateDB, genDoc) } + stateLogger := logger.With("module", "state") state := sm.LoadState(stateDB) if state == nil { state, err = sm.MakeGenesisState(stateDB, genDoc) @@ -161,6 +161,7 @@ func NewNode(config *cfg.Config, // Create the proxyApp, which manages connections (consensus, mempool, query) // and sync tendermint and the app by replaying any necessary blocks + consensusLogger := logger.With("module", "consensus") handshaker := consensus.NewHandshaker(state, blockStore) handshaker.SetLogger(consensusLogger) proxyApp := proxy.NewAppConns(clientCreator, handshaker) @@ -208,8 +209,22 @@ func NewNode(config *cfg.Config, mempool.EnableTxsAvailable() } + // Make Evidence Reactor + evidenceConfig := &cfg.EvidenceConfig{} // TODO + evidenceDB, err := dbProvider(&DBContext{"evidence", config}) + if err != nil { + return nil, err + } + evidenceLogger := logger.With("module", "evidence") + evidenceStore := evidence.NewEvidenceStore(evidenceDB) + evidencePool := evidence.NewEvidencePool(evidenceConfig, evidenceStore, state) + evidencePool.SetLogger(evidenceLogger) + evidenceReactor := evidence.NewEvidenceReactor(evidenceConfig, evidencePool) + evidenceReactor.SetLogger(evidenceLogger) + // Make ConsensusReactor - consensusState := consensus.NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool) + consensusState := consensus.NewConsensusState(config.Consensus, state.Copy(), + proxyApp.Consensus(), blockStore, mempool, evidencePool) consensusState.SetLogger(consensusLogger) if privValidator != nil { consensusState.SetPrivValidator(privValidator) @@ -224,6 +239,7 @@ func NewNode(config *cfg.Config, sw.AddReactor("MEMPOOL", mempoolReactor) sw.AddReactor("BLOCKCHAIN", bcReactor) sw.AddReactor("CONSENSUS", consensusReactor) + sw.AddReactor("EVIDENCE", evidenceReactor) // Optionally, start the pex reactor var addrBook *p2p.AddrBook @@ -323,6 +339,7 @@ func NewNode(config *cfg.Config, mempoolReactor: mempoolReactor, consensusState: consensusState, consensusReactor: consensusReactor, + evidencePool: evidencePool, proxyApp: proxyApp, txIndexer: txIndexer, indexerService: indexerService, @@ -416,6 +433,7 @@ func (n *Node) ConfigureRPC() { rpccore.SetBlockStore(n.blockStore) rpccore.SetConsensusState(n.consensusState) rpccore.SetMempool(n.mempoolReactor.Mempool) + rpccore.SetEvidencePool(n.evidencePool) rpccore.SetSwitch(n.sw) rpccore.SetPubKey(n.privValidator.GetPubKey()) rpccore.SetGenesisDoc(n.genesisDoc) @@ -489,6 +507,11 @@ func (n *Node) MempoolReactor() *mempl.MempoolReactor { return n.mempoolReactor } +// EvidencePool returns the Node's EvidencePool. +func (n *Node) EvidencePool() *evidence.EvidencePool { + return n.evidencePool +} + // EventBus returns the Node's EventBus. func (n *Node) EventBus() *types.EventBus { return n.eventBus diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index d0b0f87d..325625c7 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -45,6 +45,7 @@ var ( // interfaces defined in types and above blockStore types.BlockStore mempool types.Mempool + evidencePool types.EvidencePool consensusState Consensus p2pSwitch P2P @@ -67,6 +68,10 @@ func SetMempool(mem types.Mempool) { mempool = mem } +func SetEvidencePool(evpool types.EvidencePool) { + evidencePool = evpool +} + func SetConsensusState(cs Consensus) { consensusState = cs } diff --git a/types/services.go b/types/services.go index f1d8ddd5..25f77405 100644 --- a/types/services.go +++ b/types/services.go @@ -84,7 +84,7 @@ type State interface { // UNSTABLE type EvidencePool interface { PendingEvidence() []Evidence - AddEvidence(Evidence) + AddEvidence(Evidence) error } // MockMempool is an empty implementation of a Mempool, useful for testing. @@ -93,4 +93,4 @@ type MockEvidencePool struct { } func (m MockEvidencePool) PendingEvidence() []Evidence { return nil } -func (m MockEvidencePool) AddEvidence(Evidence) {} +func (m MockEvidencePool) AddEvidence(Evidence) error { return nil } From 4854c231e108a92c7c3566159c79d900ec2dd939 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 22:06:01 +0000 Subject: [PATCH 19/35] evidence store comments and cleanup --- evidence/store.go | 66 +++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/evidence/store.go b/evidence/store.go index e186aea5..b7c11d51 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -9,16 +9,27 @@ import ( ) /* -Schema for indexing evidence: +Requirements: + - Valid new evidence must be persisted immediately and never forgotten + - Uncommitted evidence must be continuously broadcast + - Uncommitted evidence has a partial order, the evidence's priority -"evidence-lookup"// -> evidence struct -"evidence-outqueue"/// -> nil -"evidence-pending"//evidence-hash> -> nil +Impl: + - First commit atomically in outqueue, pending, lookup. + - Once broadcast, remove from outqueue. No need to sync + - Once committed, atomically remove from pending and update lookup. + - TODO: If we crash after committed but before removing/updating, + we'll be stuck broadcasting evidence we never know we committed. + so either share the state db and atomically MarkCommitted + with ApplyBlock, or check all outqueue/pending on Start to see if its committed +Schema for indexing evidence (note you need both height and hash to find a piece of evidence): + +"evidence-lookup"// -> evidenceInfo +"evidence-outqueue"/// -> evidenceInfo +"evidence-pending"// -> evidenceInfo */ -var nullValue = []byte{0} - type evidenceInfo struct { Committed bool Priority int @@ -32,19 +43,19 @@ const ( ) func keyLookup(evidence types.Evidence) []byte { - return _key(baseKeyLookup, evidence) + return _key("%s/%d/%X", baseKeyLookup, evidence.Height(), evidence.Hash()) } -func keyOutqueue(evidence types.Evidence) []byte { - return _key(baseKeyOutqueue, evidence) +func keyOutqueue(evidence types.Evidence, priority int) []byte { + return _key("%s/%d/%d/%X", baseKeyOutqueue, priority, evidence.Height(), evidence.Hash()) } func keyPending(evidence types.Evidence) []byte { - return _key(baseKeyPending, evidence) + return _key("%s/%d/%X", baseKeyPending, evidence.Height(), evidence.Hash()) } -func _key(key string, evidence types.Evidence) []byte { - return []byte(fmt.Sprintf("%s/%d/%X", key, evidence.Height(), evidence.Hash())) +func _key(fmt_ string, o ...interface{}) []byte { + return []byte(fmt.Sprintf(fmt_, o...)) } // EvidenceStore stores all the evidence we've seen, including @@ -70,10 +81,11 @@ func (store *EvidenceStore) PriorityEvidence() (evidence []types.Evidence) { wire.ReadBinaryBytes(val, &ei) evidence = append(evidence, ei.Evidence) } - // TODO: sort + // TODO: revert order for highest first return evidence } +// PendingEvidence returns all known uncommitted evidence. func (store *EvidenceStore) PendingEvidence() (evidence []types.Evidence) { iter := store.db.IteratorPrefix([]byte(baseKeyPending)) for iter.Next() { @@ -103,21 +115,22 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int eiBytes := wire.BinaryBytes(ei) // add it to the store - key = keyLookup(evidence) - store.db.Set(key, eiBytes) - - key = keyOutqueue(evidence) + key = keyOutqueue(evidence, priority) store.db.Set(key, eiBytes) key = keyPending(evidence) store.db.Set(key, eiBytes) + key = keyLookup(evidence) + store.db.SetSync(key, eiBytes) + return true, nil } -// MarkEvidenceAsBroadcasted removes evidence from the outqueue. +// MarkEvidenceAsBroadcasted removes evidence from Outqueue. func (store *EvidenceStore) MarkEvidenceAsBroadcasted(evidence types.Evidence) { - key := keyOutqueue(evidence) + ei := store.getEvidenceInfo(evidence) + key := keyOutqueue(evidence, ei.Priority) store.db.Delete(key) } @@ -129,10 +142,19 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { key := keyPending(evidence) store.db.Delete(key) - key = keyLookup(evidence) + ei := store.getEvidenceInfo(evidence) + ei.Committed = true + + // TODO: we should use the state db and db.Sync in state.Save instead. + // Else, if we call this before state.Save, we may never mark committed evidence as committed. + // Else, if we call this after state.Save, we may get stuck broadcasting evidence we never know we committed. + store.db.SetSync(key, wire.BinaryBytes(ei)) +} + +func (store *EvidenceStore) getEvidenceInfo(evidence types.Evidence) evidenceInfo { + key := keyLookup(evidence) var ei evidenceInfo b := store.db.Get(key) wire.ReadBinaryBytes(b, &ei) - ei.Committed = true - store.db.Set(key, wire.BinaryBytes(ei)) + return ei } From 3271634e7a1833c161d17c1c9c996f7743de27a5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 22:18:43 +0000 Subject: [PATCH 20/35] types: evidence cleanup --- state/execution.go | 2 +- state/state.go | 2 +- types/block.go | 13 +++---- types/evidence.go | 91 +++++++--------------------------------------- types/services.go | 4 +- 5 files changed, 24 insertions(+), 88 deletions(-) diff --git a/state/execution.go b/state/execution.go index 860c0caa..539d1cd3 100644 --- a/state/execution.go +++ b/state/execution.go @@ -309,7 +309,7 @@ func (s *State) validateBlock(b *types.Block) error { } } - for _, ev := range block.Evidence.Evidences { + for _, ev := range block.Evidence.Evidence { if _, err := s.VerifyEvidence(ev); err != nil { return types.NewEvidenceInvalidErr(ev, err) } diff --git a/state/state.go b/state/state.go index 78888152..871e2f70 100644 --- a/state/state.go +++ b/state/state.go @@ -386,7 +386,7 @@ func (s *State) GetValidators() (last *types.ValidatorSet, current *types.Valida // VerifyEvidence verifies the evidence fully by checking it is internally // consistent and corresponds to an existing or previous validator. // It returns the priority of this evidence, or an error. -// NOTE: return error may be ErrLoadValidators, in which case the validator set +// NOTE: return error may be ErrNoValSetForHeight, in which case the validator set // for the evidence height could not be loaded. func (s *State) VerifyEvidence(evidence types.Evidence) (priority int, err error) { if err := evidence.Verify(s.ChainID); err != nil { diff --git a/types/block.go b/types/block.go index 96bfbf9d..6aa97c2d 100644 --- a/types/block.go +++ b/types/block.go @@ -43,7 +43,7 @@ func MakeBlock(height int64, txs []Tx, commit *Commit) *Block { // AddEvidence appends the given evidence to the block func (b *Block) AddEvidence(evidence []Evidence) { - b.Evidence.Evidences = append(b.Evidence.Evidences, evidence...) + b.Evidence.Evidence = append(b.Evidence.Evidence, evidence...) } // ValidateBasic performs basic validation that doesn't involve state data. @@ -437,8 +437,7 @@ func (data *Data) StringIndented(indent string) string { // EvidenceData contains any evidence of malicious wrong-doing by validators type EvidenceData struct { - // TODO: FIXME - Evidences evidences `json:"evidence"` + Evidence EvidenceList `json:"evidence"` // Volatile hash data.Bytes @@ -447,7 +446,7 @@ type EvidenceData struct { // Hash returns the hash of the data. func (data *EvidenceData) Hash() data.Bytes { if data.hash == nil { - data.hash = data.Evidences.Hash() + data.hash = data.Evidence.Hash() } return data.hash } @@ -457,10 +456,10 @@ func (data *EvidenceData) StringIndented(indent string) string { if data == nil { return "nil-Evidence" } - evStrings := make([]string, cmn.MinInt(len(data.Evidences), 21)) - for i, ev := range data.Evidences { + evStrings := make([]string, cmn.MinInt(len(data.Evidence), 21)) + for i, ev := range data.Evidence { if i == 20 { - evStrings[i] = fmt.Sprintf("... (%v total)", len(data.Evidences)) + evStrings[i] = fmt.Sprintf("... (%v total)", len(data.Evidence)) break } evStrings[i] = fmt.Sprintf("Evidence:%v", ev) diff --git a/types/evidence.go b/types/evidence.go index 42047faf..cb433ced 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -3,7 +3,6 @@ package types import ( "bytes" "fmt" - "sync" "github.com/tendermint/go-crypto" "github.com/tendermint/tmlibs/merkle" @@ -26,10 +25,6 @@ func (err *ErrEvidenceInvalid) Error() string { //------------------------------------------- -type HistoricalValidators interface { - LoadValidators(height int) (*ValidatorSet, error) -} - // Evidence represents any provable malicious activity by a validator type Evidence interface { Height() int // height of the equivocation @@ -44,96 +39,36 @@ type Evidence interface { //------------------------------------------- -//EvidenceSet is a thread-safe set of evidence. -type EvidenceSet struct { - sync.RWMutex - evidences evidences -} +// EvidenceList is a list of Evidence. Evidences is not a word. +type EvidenceList []Evidence -//Evidence returns a copy of all the evidence. -func (evset EvidenceSet) Evidence() []Evidence { - evset.RLock() - defer evset.RUnlock() - evCopy := make([]Evidence, len(evset.evidences)) - for i, ev := range evset.evidences { - evCopy[i] = ev - } - return evCopy -} - -// Size returns the number of pieces of evidence in the set. -func (evset EvidenceSet) Size() int { - evset.RLock() - defer evset.RUnlock() - return len(evset.evidences) -} - -// Hash returns a merkle hash of the evidence. -func (evset EvidenceSet) Hash() []byte { - evset.RLock() - defer evset.RUnlock() - return evset.evidences.Hash() -} - -// Has returns true if the given evidence is in the set. -func (evset EvidenceSet) Has(evidence Evidence) bool { - evset.RLock() - defer evset.RUnlock() - return evset.evidences.Has(evidence) -} - -// String returns a string representation of the evidence. -func (evset EvidenceSet) String() string { - evset.RLock() - defer evset.RUnlock() - return evset.evidences.String() -} - -// Add adds the given evidence to the set. -// TODO: and persists it to disk. -func (evset EvidenceSet) Add(evidence Evidence) { - evset.Lock() - defer evset.Unlock() - evset.evidences = append(evset.evidences, evidence) -} - -// Reset empties the evidence set. -func (evset EvidenceSet) Reset() { - evset.Lock() - defer evset.Unlock() - evset.evidences = make(evidences, 0) - -} - -//------------------------------------------- - -type evidences []Evidence - -func (evs evidences) Hash() []byte { +// Hash returns the simple merkle root hash of the EvidenceList. +func (evl EvidenceList) Hash() []byte { // Recursive impl. // Copied from tmlibs/merkle to avoid allocations - switch len(evs) { + switch len(evl) { case 0: return nil case 1: - return evs[0].Hash() + return evl[0].Hash() default: - left := evidences(evs[:(len(evs)+1)/2]).Hash() - right := evidences(evs[(len(evs)+1)/2:]).Hash() + left := EvidenceList(evl[:(len(evl)+1)/2]).Hash() + right := EvidenceList(evl[(len(evl)+1)/2:]).Hash() return merkle.SimpleHashFromTwoHashes(left, right) } } -func (evs evidences) String() string { +func (evl EvidenceList) String() string { s := "" - for _, e := range evs { + for _, e := range evl { s += fmt.Sprintf("%s\t\t", e) } return s } -func (evs evidences) Has(evidence Evidence) bool { - for _, ev := range evs { +// Has returns true if the evidence is in the EvidenceList. +func (evl EvidenceList) Has(evidence Evidence) bool { + for _, ev := range evl { if ev.Equal(evidence) { return true } diff --git a/types/services.go b/types/services.go index 25f77405..10014e66 100644 --- a/types/services.go +++ b/types/services.go @@ -4,7 +4,7 @@ import ( abci "github.com/tendermint/abci/types" ) -// NOTE: all types in this file are considered UNSTABLE +// NOTE/XXX: all type definitions in this file are considered UNSTABLE //------------------------------------------------------ // blockchain services types @@ -73,6 +73,8 @@ type BlockStore interface { //------------------------------------------------------ // state +// State defines the stateful interface used to verify evidence. +// UNSTABLE type State interface { VerifyEvidence(Evidence) (priority int, err error) } From 869d873d5cc06f0a6ec8dc0af1a1814fa3af1cd3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 22:44:46 +0000 Subject: [PATCH 21/35] state.ApplyBlock takes evpool and calls MarkEvidenceAsCommitted --- blockchain/reactor.go | 4 +++- consensus/replay.go | 4 +++- consensus/replay_test.go | 6 ++++-- consensus/state.go | 6 +++--- evidence/evidence_pool.go | 20 ++------------------ state/execution.go | 5 ++++- state/execution_test.go | 5 ++++- types/services.go | 6 ++++-- 8 files changed, 27 insertions(+), 29 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index e41300d5..f985e284 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -272,7 +272,9 @@ FOR_LOOP: // NOTE: we could improve performance if we // didn't make the app commit to disk every block // ... but we would need a way to get the hash without it persisting - err := bcR.state.ApplyBlock(bcR.eventBus, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{}) + err := bcR.state.ApplyBlock(bcR.eventBus, bcR.proxyAppConn, + first, firstPartsHeader, + types.MockMempool{}, types.MockEvidencePool{}) // TODO unmock! if err != nil { // TODO This is bad, are we zombie? cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) diff --git a/consensus/replay.go b/consensus/replay.go index 57a9dd4c..56f09d12 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -354,11 +354,13 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store // ApplyBlock on the proxyApp with the last block. func (h *Handshaker) replayBlock(height int64, proxyApp proxy.AppConnConsensus) ([]byte, error) { mempool := types.MockMempool{} + evpool := types.MockEvidencePool{} block := h.store.LoadBlock(height) meta := h.store.LoadBlockMeta(height) - if err := h.state.ApplyBlock(types.NopEventBus{}, proxyApp, block, meta.BlockID.PartsHeader, mempool); err != nil { + if err := h.state.ApplyBlock(types.NopEventBus{}, proxyApp, + block, meta.BlockID.PartsHeader, mempool, evpool); err != nil { return nil, err } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 43c2a469..495fc42c 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -261,7 +261,9 @@ const ( ) var ( - mempool = types.MockMempool{} + NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal + mempool = types.MockMempool{} + evpool = types.MockEvidencePool{} ) //--------------------------------------- @@ -394,7 +396,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) { testPartSize := st.ConsensusParams.BlockPartSizeBytes - err := st.ApplyBlock(types.NopEventBus{}, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool) + err := st.ApplyBlock(types.NopEventBus{}, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool, evpool) if err != nil { panic(err) } diff --git a/consensus/state.go b/consensus/state.go index 7531f197..5e83e6a5 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1208,7 +1208,9 @@ func (cs *ConsensusState) finalizeCommit(height int64) { // Execute and commit the block, update and save the state, and update the mempool. // All calls to the proxyAppConn come here. // NOTE: the block.AppHash wont reflect these txs until the next block - err := stateCopy.ApplyBlock(txEventBuffer, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) + err := stateCopy.ApplyBlock(txEventBuffer, cs.proxyAppConn, + block, blockParts.Header(), + cs.mempool, cs.evpool) if err != nil { cs.Logger.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "err", err) err := cmn.Kill() @@ -1236,8 +1238,6 @@ func (cs *ConsensusState) finalizeCommit(height int64) { fail.Fail() // XXX - // TODO: cs.evpool.Update() - // NewHeightStep! cs.updateToState(stateCopy) diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go index 41f51ee9..daa0f9be 100644 --- a/evidence/evidence_pool.go +++ b/evidence/evidence_pool.go @@ -73,25 +73,9 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { return nil } -// Update informs the evpool that the given evidence was committed and can be discarded. -// NOTE: this should be called *after* block is committed by consensus. -func (evpool *EvidencePool) Update(height int, evidence []types.Evidence) { - - // First, create a lookup map of new committed evidence - - evMap := make(map[string]struct{}) +// MarkEvidenceAsCommitted marks all the evidence as committed. +func (evpool *EvidencePool) MarkEvidenceAsCommitted(evidence []types.Evidence) { for _, ev := range evidence { evpool.evidenceStore.MarkEvidenceAsCommitted(ev) - evMap[string(ev.Hash())] = struct{}{} } - - // Remove evidence that is already committed . - goodEvidence := evpool.filterEvidence(evMap) - _ = goodEvidence - -} - -func (evpool *EvidencePool) filterEvidence(blockEvidenceMap map[string]struct{}) []types.Evidence { - // TODO: - return nil } diff --git a/state/execution.go b/state/execution.go index 539d1cd3..f6f6a314 100644 --- a/state/execution.go +++ b/state/execution.go @@ -327,7 +327,8 @@ func (s *State) validateBlock(b *types.Block) error { // commits it, and saves the block and state. It's the only function that needs to be called // from outside this package to process and commit an entire block. func (s *State) ApplyBlock(txEventPublisher types.TxEventPublisher, proxyAppConn proxy.AppConnConsensus, - block *types.Block, partsHeader types.PartSetHeader, mempool types.Mempool) error { + block *types.Block, partsHeader types.PartSetHeader, + mempool types.Mempool, evpool types.EvidencePool) error { abciResponses, err := s.ValExecBlock(txEventPublisher, proxyAppConn, block) if err != nil { @@ -355,6 +356,8 @@ func (s *State) ApplyBlock(txEventPublisher types.TxEventPublisher, proxyAppConn fail.Fail() // XXX + evpool.MarkEvidenceAsCommitted(block.Evidence.Evidence) + // save the state and the validators s.Save() diff --git a/state/execution_test.go b/state/execution_test.go index 25d32cbb..7cda5c1d 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -93,7 +93,10 @@ func TestApplyBlock(t *testing.T) { block := makeBlock(state, 1) - err = state.ApplyBlock(types.NopEventBus{}, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), types.MockMempool{}) + err = state.ApplyBlock(types.NopEventBus{}, proxyApp.Consensus(), + block, block.MakePartSet(testPartSize).Header(), + types.MockMempool{}, types.MockEvidencePool{}) + require.Nil(t, err) // TODO check state and mempool diff --git a/types/services.go b/types/services.go index 10014e66..b23ccff4 100644 --- a/types/services.go +++ b/types/services.go @@ -87,6 +87,7 @@ type State interface { type EvidencePool interface { PendingEvidence() []Evidence AddEvidence(Evidence) error + MarkEvidenceAsCommitted([]Evidence) } // MockMempool is an empty implementation of a Mempool, useful for testing. @@ -94,5 +95,6 @@ type EvidencePool interface { type MockEvidencePool struct { } -func (m MockEvidencePool) PendingEvidence() []Evidence { return nil } -func (m MockEvidencePool) AddEvidence(Evidence) error { return nil } +func (m MockEvidencePool) PendingEvidence() []Evidence { return nil } +func (m MockEvidencePool) AddEvidence(Evidence) error { return nil } +func (m MockEvidencePool) MarkEvidenceAsCommitted([]Evidence) {} From c7acdfadf26c8f89be2b985254e418a788dbc464 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 23:43:36 +0000 Subject: [PATCH 22/35] evidence: more funcs in store.go --- evidence/evidence_pool.go | 1 + evidence/store.go | 50 ++++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/evidence/evidence_pool.go b/evidence/evidence_pool.go index daa0f9be..0f296f46 100644 --- a/evidence/evidence_pool.go +++ b/evidence/evidence_pool.go @@ -54,6 +54,7 @@ func (evpool *EvidencePool) PendingEvidence() []types.Evidence { // Blocks on the EvidenceChan. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { + // XXX: is this thread safe ? priority, err := evpool.state.VerifyEvidence(evidence) if err != nil { return err diff --git a/evidence/store.go b/evidence/store.go index b7c11d51..4a6eb024 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -43,7 +43,11 @@ const ( ) func keyLookup(evidence types.Evidence) []byte { - return _key("%s/%d/%X", baseKeyLookup, evidence.Height(), evidence.Hash()) + return keyLookupFromHeightAndHash(evidence.Height(), evidence.Hash()) +} + +func keyLookupFromHeightAndHash(height int, hash []byte) []byte { + return _key("%s/%d/%X", baseKeyLookup, height, hash) } func keyOutqueue(evidence types.Evidence, priority int) []byte { @@ -58,8 +62,8 @@ func _key(fmt_ string, o ...interface{}) []byte { return []byte(fmt.Sprintf(fmt_, o...)) } -// EvidenceStore stores all the evidence we've seen, including -// evidence that has been committed, evidence that has been seen but not broadcast, +// EvidenceStore is a store of all the evidence we've seen, including +// evidence that has been committed, evidence that has been verified but not broadcast, // and evidence that has been broadcast but not yet committed. type EvidenceStore struct { db dbm.DB @@ -73,21 +77,19 @@ func NewEvidenceStore(db dbm.DB) *EvidenceStore { // PriorityEvidence returns the evidence from the outqueue, sorted by highest priority. func (store *EvidenceStore) PriorityEvidence() (evidence []types.Evidence) { - iter := store.db.IteratorPrefix([]byte(baseKeyOutqueue)) - for iter.Next() { - val := iter.Value() - - var ei evidenceInfo - wire.ReadBinaryBytes(val, &ei) - evidence = append(evidence, ei.Evidence) - } // TODO: revert order for highest first - return evidence + return store.ListEvidence(baseKeyOutqueue) } // PendingEvidence returns all known uncommitted evidence. func (store *EvidenceStore) PendingEvidence() (evidence []types.Evidence) { - iter := store.db.IteratorPrefix([]byte(baseKeyPending)) + return store.ListEvidence(baseKeyPending) +} + +// ListEvidence lists the evidence for the given prefix key. +// It is wrapped by PriorityEvidence and PendingEvidence for convenience. +func (store *EvidenceStore) ListEvidence(prefixKey string) (evidence []types.Evidence) { + iter := store.db.IteratorPrefix([]byte(prefixKey)) for iter.Next() { val := iter.Value() @@ -98,12 +100,23 @@ func (store *EvidenceStore) PendingEvidence() (evidence []types.Evidence) { return evidence } +// GetEvidence fetches the evidence with the given height and hash. +func (store *EvidenceStore) GetEvidence(height int, hash []byte) types.Evidence { + key := keyLookupFromHeightAndHash(height, hash) + val := store.db.Get(key) + if len(val) == 0 { + return nil + } + var ei evidenceInfo + wire.ReadBinaryBytes(val, &ei) + return ei.Evidence +} + // AddNewEvidence adds the given evidence to the database. func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) (bool, error) { // check if we already have seen it - key := keyLookup(evidence) - v := store.db.Get(key) - if len(v) != 0 { + ev := store.GetEvidence(evidence.Height(), evidence.Hash()) + if ev != nil { return false, nil } @@ -115,7 +128,7 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int eiBytes := wire.BinaryBytes(ei) // add it to the store - key = keyOutqueue(evidence, priority) + key := keyOutqueue(evidence, priority) store.db.Set(key, eiBytes) key = keyPending(evidence) @@ -151,6 +164,9 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { store.db.SetSync(key, wire.BinaryBytes(ei)) } +//--------------------------------------------------- +// utils + func (store *EvidenceStore) getEvidenceInfo(evidence types.Evidence) evidenceInfo { key := keyLookup(evidence) var ei evidenceInfo From cc418e5dab3e65e9a742c0810b1f8bbd62a22097 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 19 Nov 2017 23:51:51 +0000 Subject: [PATCH 23/35] state.VerifyEvidence enforces EvidenceParams.MaxAge --- state/state.go | 7 +++++++ types/params.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/state/state.go b/state/state.go index 871e2f70..54ad0b10 100644 --- a/state/state.go +++ b/state/state.go @@ -389,6 +389,13 @@ func (s *State) GetValidators() (last *types.ValidatorSet, current *types.Valida // NOTE: return error may be ErrNoValSetForHeight, in which case the validator set // for the evidence height could not be loaded. func (s *State) VerifyEvidence(evidence types.Evidence) (priority int, err error) { + evidenceAge := s.LastBlockHeight - evidence.Height() + maxAge := s.Params.EvidenceParams.MaxAge + if evidenceAge > maxAge { + return priority, fmt.Errorf("Evidence from height %d is too old. Min height is %d", + evidence.Height(), s.LastBlockHeight-maxAge) + } + if err := evidence.Verify(s.ChainID); err != nil { return priority, err } diff --git a/types/params.go b/types/params.go index 15d8dbe2..216f1390 100644 --- a/types/params.go +++ b/types/params.go @@ -40,7 +40,7 @@ type BlockGossip struct { // EvidenceParams determine how we handle evidence of malfeasance type EvidenceParams struct { - MaxHeightDiff int `json:"max_height_diff"` // only accept new evidence more recent than this + MaxAge int `json:"max_age"` // only accept new evidence more recent than this } // DefaultConsensusParams returns a default ConsensusParams. @@ -80,7 +80,7 @@ func DefaultBlockGossip() BlockGossip { // DefaultEvidence Params returns a default EvidenceParams. func DefaultEvidenceParams() EvidenceParams { return EvidenceParams{ - MaxHeightDiff: 100000, // 27.8 hrs at 1block/s + MaxAge: 100000, // 27.8 hrs at 1block/s } } From 1d021c27903a70179fdc5b3ff9a5a1bc9be20155 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 00:50:02 +0000 Subject: [PATCH 24/35] update consensus/test_data/many_blocks.cswal --- consensus/test_data/many_blocks.cswal | Bin 0 -> 11137 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 consensus/test_data/many_blocks.cswal diff --git a/consensus/test_data/many_blocks.cswal b/consensus/test_data/many_blocks.cswal new file mode 100644 index 0000000000000000000000000000000000000000..d443fff78eab4ee086c5c5b3783eba7df00522f6 GIT binary patch literal 11137 zcmcgy2{cvf`?ilE6$d#P%5aNJnT~VJZAoP)WGF+0lsQ8oGK37Jfyk60rHrY_kdm1s zGM9)9B~<1}r24=6oO|mI)mqkXt-p2Gx;=Zp=lh=b-QRhh@BQ|9c&jvnhK6PpLilt& zbuBSgl@1W)Pu`y)F#L!%Gl13K@6!Hx6Z;<)5ZdK73jgGw2B>xh(0+rz&{)AQ2ttEk z5l#ZzjUj@c8ew5ZgJ5@94P1^nTN%1rAJKDlIpX4GjfFakY+VbX&fi0EY?NRGA=CxF zY#@rhBxD!deY>Tyee)yv2EHi-f{{x+Uo2Pc{c^7M#*Xc=7d>ZPEiYx0(!;F;2gJ?V zYYU{~=8GGO7kBFO?R2we>mN0esuJewgJ@kE%iJMM5SKC0i^@(Lv)OhRvz)I;NL9p1Yd zl|h)2bOndr_~6-Xgi#X0#8f)X#~P{re&{?(MwnP_P2MW;wVAc8M`e*^T0%4TPb*0w zOkOFtjp<583c^~bJi_#a(TIzjTVkdA8C5`59KNWAiaxhZKot?@`r!fb}flkG~ZH#Oi< z8F_>`m@Lnx6s7v4ts9k50H~~H80|YoyKo1U0WPY(t8S^q9eRyxM@hg%J9bPW%;v$P z=0cPNT!xOt_izr!@DIL0Nx-F#_oQE;S4T_Q0hB~SnA0A52S4(8Q2(kCC6N*4YcJpB zyk_B@_$G~#fK_+Ow{x$8O)KBjpd?^rGPi5Zg$LI>`yEjdu=+grA)htGc|r|`l7Q9S zK;xZ@T77PBbWjqolD{WOYg6V-?>C2%ft9XjfB6*Jmtb9wl7SViM_lB)Se?_leJB}N z;SRqYa5Fy>Eq@6m1FQbyf!)4A<>x2zQ8KVPh~RB6@2YIO{17F}B6N6bjaGzdBu+=q zG|Swm2qgm-fv#M`&>Cd!<2{rNT&}*KEwP`@N(ozul7UMJxxYyHiTu{;F0hC zbnljO=*c6fEO40+tB)Qb)*jHhh00=FzA_)u=5ZfCxeS#hAh96z9r^s~4Q>yB{CuLuH{jTf4PxYl|Lbi|j>ZflvsuBr&U7R6lSWl?6gg zYqox^5Ak_b=a0$)Ax(Y`_Ajq_o?X6)$^oIfscAvcT^lX8o1=0-Xzlr{TP{Q?p6wb$ z<$w@bI!wS^T!}%v8kK|M9KI`AHmH1>ADhI6CZZnr>maBPsRq5 z$_>1ErKmixnl3SSSr_+tWneEV53JgE;Pm!v2s@Uy8=7-=LQsOt5JDiHQzRNxyqts>We)p53IzrZ5ot}YX~o6QF&nX zWlB0$Vx&cwBOjGlK$voPN1Y5R`w|bL@<6Its=yp|Wvu16jw(naj2buzGv?ro?FchZ ztk9w7pCXcX@t_Ja2tDTk#=XxF<|oD-XNlgr@hbhO0`Op8BYdN`&77`&22}taGU*by zqgAD9m!wby;NdM#(iq7we#NhcDgY0sZr+biimcz_$4~{R4nIAKf4661g!DC30f;0D z+V-WWtu$44LlvPqA;p1D`pJrgO^T=@R3{*ar|wD!Tj`o!R1vC!s?XYqH{02FE})7~ zootmM{YIaS$C{3#il8_NS;txP)-l8|po&0>h3mwqbkfL!Tq3F{hcKyWw35OqPv34u z6@ioP?YU=8ub3xoY*9tvG+Q7Uywx^5oNS3IDk2O?a&31RFCqArSoytQ_ll~tZ3ZzU zBKS4qDH@}O>~s1}fMpOo{_u-Q@`Kp*(`x{e5Pa&G(`f-q+kkDAfXN8Hm7jN&&H>u; zOgvy&1W%`azK>4Yv}eErFchjM&$vT+`T_9@KzRf|X2Pcx;6~T1AP-mpp||z=^5r># zj}{#--E1BibSeTgSQ^3esBCPvOb>B4umcRNc7A*Msl6ztv%m?k48m~Lonz>b9D=Wx zawqy8ty8Nhfa|~}gs)6|53)Ass4!qz1W`ipCr65qte}fhZ=&E5Fc7n0J91`x*mjjE zU5T`QSf7)t57m{tL!Nu zg?Xs>OmBzxDu?DSl&6__5)%h+iLA$7JRT+&Fb&a;W5!uCC+hq2v>wCTRy0`Tm&?%lCY?eotmz?VS!p{<*tL zvZ48FcSqTpoj>m{{G%@Q;!qbH4f3Ndy4IeW*7mk`l)AiNPu&c`)SaHOx~nQ>E2zA` zl>_?_R=HVy|1OB2E*j%K1@Khi&QEuGl@i6K5K7(Y8C!j)YEK zlJmol2tKltZ;t7t?(_^A4`k2kJzf*S#X+odvE0y*H{!SJlH$6g{E$_OB7zHtq#YT> zd<$iKzS8qA^sKX7edAP6Os>Eyg-&|)y2(_yCuqQQ%6 zk}7;i{>a!47(`?JhL?xEn`u+Jy#a%i9o8K6+mkonyVz#txH zu1K~@uiGm+!37wkOe_?UfA)-%LDLd2h>h^|nng#p;X50!ogpYm1Y1@o_Z7X@hIN2J zNvcWNmRt@(#&Vi~6%qV6{pu7CCO$qr*bidct9P8E9tJtqIM6WIUg6m%*5<44Yr5OD zA284xIc>2)93{X&Pu|@@&`UazFRltO7}5xT;!Vjr{hUJpWf4Y(;*%ncH%g8p z_zx$RS$L-(>g?JHcYv4Tz}CaM*JBP?`TcT%{4ZD#!bI z2>xX%QBpwHp|{Bat^$SG$i=S{277kD+X@&cNWMyE{(N=%&`CDHnAq2~3A-8K^5-`K zh6=G;XcV6RdhD7(JYc9$>Z8XD8JiQ9U0M$qm~;x2`ghGYS{Vfah6*JFXGoQYh5Ds- z0EPN}3Li1L-)b(gGCuRfPvn=>uZ{G zA2fEWtp`JT@x z>Gl2)qaYvE|A^6d2dl0Sz(FY0!HgmFH;RvG6v5PKsLSsQld(jvj6U&&zQ&+carm#K z3;Dzyn3QuULzH*WdEg`a5gP6H4@aJQC+-#zj=kAN7oo6+)6DqtJ4?x@b5_a%9@iR3 zXH)&+U%cO$Q+aY?p6~t&N;cUuBHTc69aXh1mInMwsr6j{=3x0%w&FkXf#Bcf1CM7( z`OMx~zoP$mdP}v@ROy=I&4cle?UL8qw{WkddnrzD-gCi7C2h@YdfQdbgp3@XKtiX0 zlu9($hw>}E69T5zV=I^_`7rk&i$Ia;V8$umDk%B<%)y8nGh~RQIv5uDmxJLj%L(+b zQT$Pt=31x=?JpgyM&;}u5KMKjC~YPhs)N}Ecw-;JhU1T{YKI7_gGKGrRizB~QwK{m z&y;~ss)J#nn1jJEuskNW{IOWLQpljy6}K#M%0=va=M`mJGB1$H8bZ2g9<(9BfHXnlPNl9b3kqUwwznpS$>Q z)$P6)(J;pTBt?~xFx5$i+6_(m@10sI*bmYF%E4v|x)%1wbbPCLH6L795pZ?j_3SIC z5jfbTtDDQ@G_%Xfcf*MR=wF?!w_goeZE=SG9;SgdZW-%wqgJzr#R#$=!5?4s zBrrKZv)PLm?g54Exi+hv2gXjt!`ThAZTN#P1U(VgeFo|?fI)lTh#b>e6|Kn~1Lh2k z8{TB#iT&x;g|RNMFX-Ix{tQwQfe(5dHh|Vb>xS=o_~N?2y`gQr&?rFjhTl2GO*c3e zU0^Yc5}|v;Gqk4Je7!-K(gZhD0Bh)tJkzt&@O^oz4`67UK5>6E-C)8jaS|Z3Ou*frMn_e`A?JeO<%7}#wr*zU|NaLzhH09?-ktH*aC6%A(^Fwg-!W=54vw1i|Bs|A&T?2>#ZIeGH0Q-Gxkx;Fb)+=rrQu zWudz=9KpZi?S8A0uP{b{wgkF1Jh#{*-S~oS1s?Q(VfrtpZ2F&{AsdT8aiDR-pBAri zH>=9oa1pi!(7NG$-Ux^7z-P}-l>mm~^ougqsznROYqbKV7}Jv0F!p`?xiR5r{@<04 z^7WN^*@~&qpBm%UR`DN=`EE=MNmpGVlxj>-5c(S-$E1gFa!-)qvcsKP_x1ER?q1zI^mp{eUDr*$iv)#NI%b9V*bBmgp{^nbgZ%zy+_Dm8K zWW=tEg^mZtf7?+V)#tbV#>K)k73#rhZdvk2phz{Qs7uDG{}SMT7?aOtWi0s5d?5I@ z`M~2@Qa+UjH4mAj9vQf_sF`KFi#;X3SlHz8wRT4l1a;9ghu^NC4EIxG z(#v{L3!zkF!a^})f?;5JG&e1`avG>Q_mB@=mA9~BiZP;yOf@DY9qt>U+qzp-N|+VW+;D=urjwRvCN+qP*F zwaxB%cc%4wYekuv9h>-9trZ<+OwIm|0hlo@=}8j`zuMbg&rh!&KfL4oyYYDLNRM41 z2qE$C+jAo-hHh_t4K_?Xa~RYQVHo?BF>O4pdS{VvkTlugb$fF9f;lJ0)l8@n7*nr; zjAfl%_XmO+98!b&PJHFiaTYv(U*;}gBC=f9vanp(HrV7bf-e=a-^=h;Z0PYaxJ5z` z@P=vMkUK%XQfT1_o zUX^ONzG$@k$~(Y7f;E6c`XgcP=}LdVpuuhy_2+GGn0Q}Z2N+15IoBa}-K8a!ClWAd zu&;2?Mn9Du8lSQIpuw}LLsR&uz=`ZGz@WjVod+z8Ge3H`r~n3jcA>`)#_Kf)T?dSH zpJbI)WM#QKOco7*fuBpJM#r1yX(u&B0Rz8X!P6VuiU%KXcLD}}pKW?1lu7!6&&&Y> zzxTy-EhSML4<*h427ZR*OSQhr9Iht20K-9AV-!I@vP~>+blI=jl`+%#aI zM`zuvVlC4sp=Ash=rzAgd_GBM@sdKaY%eP(WfPvknJ@Qf)PbMk!TLK1l^1T%uUJYM)-vT@6KJlcFL}A*=;g6mY zfT4N;xx3pl`%ZtB@Bs|;TmuNt^X^Uz?+E}5^pb33b?HO|W-3zwQygqbi~ZLQMvFPv zlE(U?y&E=J!SEX8=x9zv;#-wdI@QG86J2n(CS1P`w0P%!d+~#l z-S_Hly3chEv);?P-^fec&p#0MAqFS#}BSk`9Sb*^MS{+q22(27dg)QHz)d5ASgEha~M149xb1>p(?=uid zb+G1nzwZu4hgpt6?5geG>(U~;9mb*mdtLrNpUbrDCS@g49la2l{=lUinYF}@eh0> z*L=`{m9A`52iqg&$UbwaxcBDTbg_-?n|bS=MVO`Aak;fE=Q+XM8GGVHp8CD}k_S!e zYnB-@wO$Cun^g)oJFXV0yk@8yeS=r-C>&}9M4P&hpb!K*KVsNlFnEgMk6+I;k zvc(*1Nl%&}5Nz^Fb?d$x89Q?XX6_f>6PI!$B7}Xn*i4T|OYYlNp3WGi*ZnPgzH0DS z4wjP;>h-czzaW(9qoBXgi*LHM?3Pd?a4>aab47cR`l+vSs0azP z;WZ-OMSpRXo$HM-Pmp`;=!;fp{f-$e-7Q5mfI$GAbxO_ho}Odz$KpW%zvi8)e*MYM z&+#>25Wq3dytK1ZH};JofI$HJd9@!_KCas94mOA#eVLLY6SeoENk?S?g8(+TDi}(x zXOnnH1PuBdpw42~@E7OB$=4I7g z0T@Ubv*hIl=RVCIy$%>i-R64mEYb35rW`$BAT_#QfVm+vIOd5Tyh{dB^pQy$;)rW_ zuI2#-QpLjkdf&(z3yt8x@O}~B(v`Wn#&Kjce?4F*8h^&|2eZD-A)mnmp)bW>4N)L+ zOL1@C_82hmW8V~~c#XNk1#to_htS7z2fsiO{IIdxUR|3s+}AO<479cdI9+mV3VFz8 z0~lx#G$LEo?ih44gu^>+pfx02R{d@9QMjA|V4!7Rd!~y2vyWk~4q%|A@wH7dZvU9o zE?2-n%VBTu>r|OX=_0&qpKw7Ig+gB zoaX^U*|-NcHe1I$czHDeFz^fF*)?jlGpOx7yhWF%nAwunJ@y^`c{@gjnc0%Y`k&R3 z@|C~*DaGQ4E>*EmW$y2~^xe#=q)UPzlp_9VDF9EM+-8RHWNEeNKxV`&;$f`6G=h6wf{Y`6-X0`{_$YGxW8 z?^P+o{e97VH?y9w^_mb$H8U&}Gcy8dVcli2d28%h7vWbtAb`Tgot@iVb*rpq0D}OgU2Hmh z`FS&28MJ3`7>3{RY%SAaHM%`HV195KhQGu5rER-lnx_FPVE7dQy=aD(uZT8uO4wo8 zv;f~4tLRsC0;+&P{;uCBv>mH`{^0E>V35C4yRAPszc2Z`1v(}8bpxI>7DfIrV24k) z0}S%qBAMx5a(>`=0n83ZWO$)aOY?|LpTqj20Ry|udx}rECw`kVkpT?s9HU$b!M0iH zC5nK7UD}1%4M+I6WT(JNvC}f^WT6v-nfZadCHannbz-Mw3k$iK-PI!H+)n@lKNprFZkA$}d+B)il@svWD|X(6 zy=~}Td>~*V!r1Um*RW>7+ZsNphB)CW%inz&gFC=1VOV_IoiaO4&nmzevzrSxX6|Gg z+YNw$nNp6&*q!bz z<$)r=^02m+w#KpV>(7mg9y6{beP8}(Wu!=b;p%FW70&>BxasjR_EcB|6x-?O5c-&z{@k)zfvqjYcPm?Ee=2G%aw-XC3?hui&x3n zZ`sQ+Sa&S@+0bq`K^^pTl7*m5;oYaAnzQj?nuliig(9|Y66^cJz;;BIDpuYag4coK zI*M3nvd^ggOQ8Q@TPTflNVZ zz3%!xJk#UJHc^|VNev#AXh|IHJ$8vZ--cutd(B3q?50v{Zl!$>(Rr^?!2^)p-B_ literal 0 HcmV?d00001 From c2585b5525ecb3cfa785d8923480ab1f55be8201 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 00:52:02 +0000 Subject: [PATCH 25/35] evidence_pool.go -> pool.go. remove old test files --- evidence/evidence_pool_test.go | 202 ------------------------- evidence/{evidence_pool.go => pool.go} | 0 evidence/reactor_test.go | 108 ------------- 3 files changed, 310 deletions(-) delete mode 100644 evidence/evidence_pool_test.go rename evidence/{evidence_pool.go => pool.go} (100%) delete mode 100644 evidence/reactor_test.go diff --git a/evidence/evidence_pool_test.go b/evidence/evidence_pool_test.go deleted file mode 100644 index fba5941c..00000000 --- a/evidence/evidence_pool_test.go +++ /dev/null @@ -1,202 +0,0 @@ -package evpool - -import ( - "crypto/rand" - "encoding/binary" - "testing" - "time" - - "github.com/tendermint/abci/example/counter" - "github.com/tendermint/abci/example/dummy" - "github.com/tendermint/tmlibs/log" - - cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" -) - -func newMempoolWithApp(cc proxy.ClientCreator) *Mempool { - config := cfg.ResetTestRoot("mempool_test") - - appConnMem, _ := cc.NewABCIClient() - appConnMem.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "mempool")) - appConnMem.Start() - mempool := NewMempool(config.Mempool, appConnMem, 0) - mempool.SetLogger(log.TestingLogger()) - return mempool -} - -func ensureNoFire(t *testing.T, ch <-chan int, timeoutMS int) { - timer := time.NewTimer(time.Duration(timeoutMS) * time.Millisecond) - select { - case <-ch: - t.Fatal("Expected not to fire") - case <-timer.C: - } -} - -func ensureFire(t *testing.T, ch <-chan int, timeoutMS int) { - timer := time.NewTimer(time.Duration(timeoutMS) * time.Millisecond) - select { - case <-ch: - case <-timer.C: - t.Fatal("Expected to fire") - } -} - -func checkTxs(t *testing.T, mempool *Mempool, count int) types.Txs { - txs := make(types.Txs, count) - for i := 0; i < count; i++ { - txBytes := make([]byte, 20) - txs[i] = txBytes - rand.Read(txBytes) - err := mempool.CheckTx(txBytes, nil) - if err != nil { - t.Fatal("Error after CheckTx: %v", err) - } - } - return txs -} - -func TestTxsAvailable(t *testing.T) { - app := dummy.NewDummyApplication() - cc := proxy.NewLocalClientCreator(app) - mempool := newMempoolWithApp(cc) - mempool.EnableTxsAvailable() - - timeoutMS := 500 - - // with no txs, it shouldnt fire - ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) - - // send a bunch of txs, it should only fire once - txs := checkTxs(t, mempool, 100) - ensureFire(t, mempool.TxsAvailable(), timeoutMS) - ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) - - // call update with half the txs. - // it should fire once now for the new height - // since there are still txs left - committedTxs, txs := txs[:50], txs[50:] - mempool.Update(1, committedTxs) - ensureFire(t, mempool.TxsAvailable(), timeoutMS) - ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) - - // send a bunch more txs. we already fired for this height so it shouldnt fire again - moreTxs := checkTxs(t, mempool, 50) - ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) - - // now call update with all the txs. it should not fire as there are no txs left - committedTxs = append(txs, moreTxs...) - mempool.Update(2, committedTxs) - ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) - - // send a bunch more txs, it should only fire once - checkTxs(t, mempool, 100) - ensureFire(t, mempool.TxsAvailable(), timeoutMS) - ensureNoFire(t, mempool.TxsAvailable(), timeoutMS) -} - -func TestSerialReap(t *testing.T) { - app := counter.NewCounterApplication(true) - app.SetOption("serial", "on") - cc := proxy.NewLocalClientCreator(app) - - mempool := newMempoolWithApp(cc) - appConnCon, _ := cc.NewABCIClient() - appConnCon.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "consensus")) - if _, err := appConnCon.Start(); err != nil { - t.Fatalf("Error starting ABCI client: %v", err.Error()) - } - - deliverTxsRange := func(start, end int) { - // Deliver some txs. - for i := start; i < end; i++ { - - // This will succeed - txBytes := make([]byte, 8) - binary.BigEndian.PutUint64(txBytes, uint64(i)) - err := mempool.CheckTx(txBytes, nil) - if err != nil { - t.Fatal("Error after CheckTx: %v", err) - } - - // This will fail because not serial (incrementing) - // However, error should still be nil. - // It just won't show up on Reap(). - err = mempool.CheckTx(txBytes, nil) - if err != nil { - t.Fatal("Error after CheckTx: %v", err) - } - - } - } - - reapCheck := func(exp int) { - txs := mempool.Reap(-1) - if len(txs) != exp { - t.Fatalf("Expected to reap %v txs but got %v", exp, len(txs)) - } - } - - updateRange := func(start, end int) { - txs := make([]types.Tx, 0) - for i := start; i < end; i++ { - txBytes := make([]byte, 8) - binary.BigEndian.PutUint64(txBytes, uint64(i)) - txs = append(txs, txBytes) - } - mempool.Update(0, txs) - } - - commitRange := func(start, end int) { - // Deliver some txs. - for i := start; i < end; i++ { - txBytes := make([]byte, 8) - binary.BigEndian.PutUint64(txBytes, uint64(i)) - res := appConnCon.DeliverTxSync(txBytes) - if !res.IsOK() { - t.Errorf("Error committing tx. Code:%v result:%X log:%v", - res.Code, res.Data, res.Log) - } - } - res := appConnCon.CommitSync() - if len(res.Data) != 8 { - t.Errorf("Error committing. Hash:%X log:%v", res.Data, res.Log) - } - } - - //---------------------------------------- - - // Deliver some txs. - deliverTxsRange(0, 100) - - // Reap the txs. - reapCheck(100) - - // Reap again. We should get the same amount - reapCheck(100) - - // Deliver 0 to 999, we should reap 900 new txs - // because 100 were already counted. - deliverTxsRange(0, 1000) - - // Reap the txs. - reapCheck(1000) - - // Reap again. We should get the same amount - reapCheck(1000) - - // Commit from the conensus AppConn - commitRange(0, 500) - updateRange(0, 500) - - // We should have 500 left. - reapCheck(500) - - // Deliver 100 invalid txs and 100 valid txs - deliverTxsRange(900, 1100) - - // We should have 600 now. - reapCheck(600) -} diff --git a/evidence/evidence_pool.go b/evidence/pool.go similarity index 100% rename from evidence/evidence_pool.go rename to evidence/pool.go diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go deleted file mode 100644 index e488311b..00000000 --- a/evidence/reactor_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package evpool - -import ( - "fmt" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/go-kit/kit/log/term" - - "github.com/tendermint/abci/example/dummy" - "github.com/tendermint/tmlibs/log" - - cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" -) - -// evpoolLogger is a TestingLogger which uses a different -// color for each validator ("validator" key must exist). -func evpoolLogger() log.Logger { - return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor { - for i := 0; i < len(keyvals)-1; i += 2 { - if keyvals[i] == "validator" { - return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))} - } - } - return term.FgBgColor{} - }) -} - -// connect N evpool reactors through N switches -func makeAndConnectEvidencePoolReactors(config *cfg.Config, N int) []*EvidencePoolReactor { - reactors := make([]*EvidencePoolReactor, N) - logger := evpoolLogger() - for i := 0; i < N; i++ { - app := dummy.NewDummyApplication() - cc := proxy.NewLocalClientCreator(app) - evpool := newEvidencePoolWithApp(cc) - - reactors[i] = NewEvidencePoolReactor(config.EvidencePool, evpool) // so we dont start the consensus states - reactors[i].SetLogger(logger.With("validator", i)) - } - - p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("MEMPOOL", reactors[i]) - return s - - }, p2p.Connect2Switches) - return reactors -} - -// wait for all evidences on all reactors -func waitForTxs(t *testing.T, evidences types.Txs, reactors []*EvidencePoolReactor) { - // wait for the evidences in all evpools - wg := new(sync.WaitGroup) - for i := 0; i < len(reactors); i++ { - wg.Add(1) - go _waitForTxs(t, wg, evidences, i, reactors) - } - - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - - timer := time.After(TIMEOUT) - select { - case <-timer: - t.Fatal("Timed out waiting for evidences") - case <-done: - } -} - -// wait for all evidences on a single evpool -func _waitForTxs(t *testing.T, wg *sync.WaitGroup, evidences types.Txs, reactorIdx int, reactors []*EvidencePoolReactor) { - - evpool := reactors[reactorIdx].EvidencePool - for evpool.Size() != len(evidences) { - time.Sleep(time.Second) - } - - reapedTxs := evpool.Reap(len(evidences)) - for i, evidence := range evidences { - assert.Equal(t, evidence, reapedTxs[i], fmt.Sprintf("evidences at index %d on reactor %d don't match: %v vs %v", i, reactorIdx, evidence, reapedTxs[i])) - } - wg.Done() -} - -var ( - NUM_TXS = 1000 - TIMEOUT = 120 * time.Second // ridiculously high because CircleCI is slow -) - -func TestReactorBroadcastTxMessage(t *testing.T) { - config := cfg.TestConfig() - N := 4 - reactors := makeAndConnectEvidencePoolReactors(config, N) - - // send a bunch of evidences to the first reactor's evpool - // and wait for them all to be received in the others - evidences := checkTxs(t, reactors[0].EvidencePool, NUM_TXS) - waitForTxs(t, evidences, reactors) -} From c13e93d63e0ace43bfcf36149e40b7a1e7fe3f8c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 03:55:59 +0000 Subject: [PATCH 26/35] evidence: store tests and fixes --- evidence/store.go | 56 ++++++++------ evidence/store_test.go | 166 +++++++++++++++++++++++++++++++++++++++++ types/evidence.go | 12 +++ 3 files changed, 212 insertions(+), 22 deletions(-) create mode 100644 evidence/store_test.go diff --git a/evidence/store.go b/evidence/store.go index 4a6eb024..1b4708ea 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -25,12 +25,12 @@ Impl: Schema for indexing evidence (note you need both height and hash to find a piece of evidence): -"evidence-lookup"// -> evidenceInfo -"evidence-outqueue"/// -> evidenceInfo -"evidence-pending"// -> evidenceInfo +"evidence-lookup"// -> EvidenceInfo +"evidence-outqueue"/// -> EvidenceInfo +"evidence-pending"// -> EvidenceInfo */ -type evidenceInfo struct { +type EvidenceInfo struct { Committed bool Priority int Evidence types.Evidence @@ -46,16 +46,21 @@ func keyLookup(evidence types.Evidence) []byte { return keyLookupFromHeightAndHash(evidence.Height(), evidence.Hash()) } +// big endian padded hex +func be(h int) string { + return fmt.Sprintf("%0.16X", h) +} + func keyLookupFromHeightAndHash(height int, hash []byte) []byte { - return _key("%s/%d/%X", baseKeyLookup, height, hash) + return _key("%s/%s/%X", baseKeyLookup, be(height), hash) } func keyOutqueue(evidence types.Evidence, priority int) []byte { - return _key("%s/%d/%d/%X", baseKeyOutqueue, priority, evidence.Height(), evidence.Hash()) + return _key("%s/%s/%s/%X", baseKeyOutqueue, be(priority), be(evidence.Height()), evidence.Hash()) } func keyPending(evidence types.Evidence) []byte { - return _key("%s/%d/%X", baseKeyPending, evidence.Height(), evidence.Hash()) + return _key("%s/%s/%X", baseKeyPending, be(evidence.Height()), evidence.Hash()) } func _key(fmt_ string, o ...interface{}) []byte { @@ -77,8 +82,13 @@ func NewEvidenceStore(db dbm.DB) *EvidenceStore { // PriorityEvidence returns the evidence from the outqueue, sorted by highest priority. func (store *EvidenceStore) PriorityEvidence() (evidence []types.Evidence) { - // TODO: revert order for highest first - return store.ListEvidence(baseKeyOutqueue) + // reverse the order so highest priority is first + l := store.ListEvidence(baseKeyOutqueue) + l2 := make([]types.Evidence, len(l)) + for i, _ := range l { + l2[i] = l[len(l)-1-i] + } + return l2 } // PendingEvidence returns all known uncommitted evidence. @@ -93,7 +103,7 @@ func (store *EvidenceStore) ListEvidence(prefixKey string) (evidence []types.Evi for iter.Next() { val := iter.Value() - var ei evidenceInfo + var ei EvidenceInfo wire.ReadBinaryBytes(val, &ei) evidence = append(evidence, ei.Evidence) } @@ -101,26 +111,27 @@ func (store *EvidenceStore) ListEvidence(prefixKey string) (evidence []types.Evi } // GetEvidence fetches the evidence with the given height and hash. -func (store *EvidenceStore) GetEvidence(height int, hash []byte) types.Evidence { +func (store *EvidenceStore) GetEvidence(height int, hash []byte) *EvidenceInfo { key := keyLookupFromHeightAndHash(height, hash) val := store.db.Get(key) + if len(val) == 0 { return nil } - var ei evidenceInfo - wire.ReadBinaryBytes(val, &ei) - return ei.Evidence + ei := new(EvidenceInfo) + wire.ReadBinaryBytes(val, ei) + return ei } // AddNewEvidence adds the given evidence to the database. func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) (bool, error) { // check if we already have seen it - ev := store.GetEvidence(evidence.Height(), evidence.Hash()) - if ev != nil { + ei_ := store.GetEvidence(evidence.Height(), evidence.Hash()) + if ei_ != nil && ei_.Evidence != nil { return false, nil } - ei := evidenceInfo{ + ei := EvidenceInfo{ Committed: false, Priority: priority, Evidence: evidence, @@ -152,8 +163,8 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { // if its committed, its been broadcast store.MarkEvidenceAsBroadcasted(evidence) - key := keyPending(evidence) - store.db.Delete(key) + pendingKey := keyPending(evidence) + store.db.Delete(pendingKey) ei := store.getEvidenceInfo(evidence) ei.Committed = true @@ -161,15 +172,16 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { // TODO: we should use the state db and db.Sync in state.Save instead. // Else, if we call this before state.Save, we may never mark committed evidence as committed. // Else, if we call this after state.Save, we may get stuck broadcasting evidence we never know we committed. - store.db.SetSync(key, wire.BinaryBytes(ei)) + lookupKey := keyLookup(evidence) + store.db.SetSync(lookupKey, wire.BinaryBytes(ei)) } //--------------------------------------------------- // utils -func (store *EvidenceStore) getEvidenceInfo(evidence types.Evidence) evidenceInfo { +func (store *EvidenceStore) getEvidenceInfo(evidence types.Evidence) EvidenceInfo { key := keyLookup(evidence) - var ei evidenceInfo + var ei EvidenceInfo b := store.db.Get(key) wire.ReadBinaryBytes(b, &ei) return ei diff --git a/evidence/store_test.go b/evidence/store_test.go new file mode 100644 index 00000000..76cb794a --- /dev/null +++ b/evidence/store_test.go @@ -0,0 +1,166 @@ +package evpool + +import ( + "bytes" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + wire "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/types" + dbm "github.com/tendermint/tmlibs/db" +) + +//------------------------------------------- + +func TestStoreAddDuplicate(t *testing.T) { + assert := assert.New(t) + + db := dbm.NewMemDB() + store := NewEvidenceStore(db) + + priority := 10 + ev := newMockGoodEvidence(2, 1, []byte("val1")) + + added, err := store.AddNewEvidence(ev, priority) + assert.Nil(err) + assert.True(added) + + // cant add twice + added, err = store.AddNewEvidence(ev, priority) + assert.Nil(err) + assert.False(added) +} + +func TestStoreMark(t *testing.T) { + assert := assert.New(t) + + db := dbm.NewMemDB() + store := NewEvidenceStore(db) + + // before we do anything, priority/pending are empty + priorityEv := store.PriorityEvidence() + pendingEv := store.PendingEvidence() + assert.Equal(0, len(priorityEv)) + assert.Equal(0, len(pendingEv)) + + priority := 10 + ev := newMockGoodEvidence(2, 1, []byte("val1")) + + added, err := store.AddNewEvidence(ev, priority) + assert.Nil(err) + assert.True(added) + + // get the evidence. verify. should be uncommitted + ei := store.GetEvidence(ev.Height(), ev.Hash()) + assert.Equal(ev, ei.Evidence) + assert.Equal(priority, ei.Priority) + assert.False(ei.Committed) + + // new evidence should be returns in priority/pending + priorityEv = store.PriorityEvidence() + pendingEv = store.PendingEvidence() + assert.Equal(1, len(priorityEv)) + assert.Equal(1, len(pendingEv)) + + // priority is now empty + store.MarkEvidenceAsBroadcasted(ev) + priorityEv = store.PriorityEvidence() + pendingEv = store.PendingEvidence() + assert.Equal(0, len(priorityEv)) + assert.Equal(1, len(pendingEv)) + + // priority and pending are now empty + store.MarkEvidenceAsCommitted(ev) + priorityEv = store.PriorityEvidence() + pendingEv = store.PendingEvidence() + assert.Equal(0, len(priorityEv)) + assert.Equal(0, len(pendingEv)) + + // evidence should show committed + ei = store.GetEvidence(ev.Height(), ev.Hash()) + assert.Equal(ev, ei.Evidence) + assert.Equal(priority, ei.Priority) + assert.True(ei.Committed) +} + +func TestStorePriority(t *testing.T) { + assert := assert.New(t) + + db := dbm.NewMemDB() + store := NewEvidenceStore(db) + + // sorted by priority and then height + cases := []struct { + ev MockGoodEvidence + priority int + }{ + {newMockGoodEvidence(2, 1, []byte("val1")), 17}, + {newMockGoodEvidence(5, 2, []byte("val2")), 15}, + {newMockGoodEvidence(10, 2, []byte("val2")), 13}, + {newMockGoodEvidence(100, 2, []byte("val2")), 11}, + {newMockGoodEvidence(90, 2, []byte("val2")), 11}, + {newMockGoodEvidence(80, 2, []byte("val2")), 11}, + } + + for _, c := range cases { + added, err := store.AddNewEvidence(c.ev, c.priority) + assert.True(added) + assert.Nil(err) + } + + evList := store.PriorityEvidence() + for i, ev := range evList { + assert.Equal(ev, cases[i].ev) + } +} + +//------------------------------------------- +const ( + evidenceTypeMock = byte(0x01) +) + +var _ = wire.RegisterInterface( + struct{ types.Evidence }{}, + wire.ConcreteType{MockGoodEvidence{}, evidenceTypeMock}, +) + +type MockGoodEvidence struct { + Height_ int + Address_ []byte + Index_ int +} + +func newMockGoodEvidence(height, index int, address []byte) MockGoodEvidence { + return MockGoodEvidence{height, address, index} +} + +func (e MockGoodEvidence) Height() int { return e.Height_ } +func (e MockGoodEvidence) Address() []byte { return e.Address_ } +func (e MockGoodEvidence) Index() int { return e.Index_ } +func (e MockGoodEvidence) Hash() []byte { return []byte{byte(e.Index_)} } +func (e MockGoodEvidence) Verify(chainID string) error { return nil } +func (e MockGoodEvidence) Equal(ev types.Evidence) bool { + e2 := ev.(MockGoodEvidence) + return e.Height_ == e2.Height_ && + bytes.Equal(e.Address_, e2.Address_) && + e.Index_ == e2.Index_ +} +func (e MockGoodEvidence) String() string { + return fmt.Sprintf("GoodEvidence: %d/%s/%d", e.Height_, e.Address_, e.Index_) +} + +type MockBadEvidence struct { + MockGoodEvidence +} + +func (e MockBadEvidence) Verify(chainID string) error { return fmt.Errorf("MockBadEvidence") } +func (e MockBadEvidence) Equal(ev types.Evidence) bool { + e2 := ev.(MockBadEvidence) + return e.Height_ == e2.Height_ && + bytes.Equal(e.Address_, e2.Address_) && + e.Index_ == e2.Index_ +} +func (e MockBadEvidence) String() string { + return fmt.Sprintf("BadEvidence: %d/%s/%d", e.Height_, e.Address_, e.Index_) +} diff --git a/types/evidence.go b/types/evidence.go index cb433ced..e270a0de 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/tendermint/go-crypto" + wire "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/merkle" ) @@ -78,6 +79,17 @@ func (evl EvidenceList) Has(evidence Evidence) bool { //------------------------------------------- +const ( + evidenceTypeDuplicateVote = byte(0x01) +) + +var _ = wire.RegisterInterface( + struct{ Evidence }{}, + wire.ConcreteType{&DuplicateVoteEvidence{}, evidenceTypeDuplicateVote}, +) + +//------------------------------------------- + // DuplicateVoteEvidence contains evidence a validator signed two conflicting votes. type DuplicateVoteEvidence struct { PubKey crypto.PubKey From 666ae244b367bc2ab50aedb181e829f126488cc4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 04:22:25 +0000 Subject: [PATCH 27/35] evidence: pool test --- evidence/pool.go | 8 +++---- evidence/pool_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++ evidence/reactor.go | 2 +- evidence/store.go | 8 +++---- evidence/store_test.go | 14 ++++------- node/node.go | 2 +- 6 files changed, 68 insertions(+), 20 deletions(-) create mode 100644 evidence/pool_test.go diff --git a/evidence/pool.go b/evidence/pool.go index 0f296f46..bfc04f44 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -1,4 +1,4 @@ -package evpool +package evidence import ( "github.com/tendermint/tmlibs/log" @@ -60,10 +60,8 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { return err } - added, err := evpool.evidenceStore.AddNewEvidence(evidence, priority) - if err != nil { - return err - } else if !added { + added := evpool.evidenceStore.AddNewEvidence(evidence, priority) + if !added { // evidence already known, just ignore return } diff --git a/evidence/pool_test.go b/evidence/pool_test.go new file mode 100644 index 00000000..2908bb42 --- /dev/null +++ b/evidence/pool_test.go @@ -0,0 +1,54 @@ +package evidence + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/types" + dbm "github.com/tendermint/tmlibs/db" +) + +type mockState struct{} + +func (m mockState) VerifyEvidence(ev types.Evidence) (int, error) { + err := ev.Verify("") + return 10, err +} + +func TestEvidencePool(t *testing.T) { + assert := assert.New(t) + + config := &cfg.EvidenceConfig{} + store := NewEvidenceStore(dbm.NewMemDB()) + state := mockState{} + pool := NewEvidencePool(config, store, state) + + goodEvidence := newMockGoodEvidence(5, 1, []byte("val1")) + badEvidence := MockBadEvidence{goodEvidence} + + err := pool.AddEvidence(badEvidence) + assert.NotNil(err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + <-pool.EvidenceChan() + wg.Done() + }() + + err = pool.AddEvidence(goodEvidence) + assert.Nil(err) + wg.Wait() + + // if we send it again it wont fire on the chan + err = pool.AddEvidence(goodEvidence) + assert.Nil(err) + select { + case <-pool.EvidenceChan(): + t.Fatal("unexpected read on EvidenceChan") + default: + } +} diff --git a/evidence/reactor.go b/evidence/reactor.go index f6e7501a..9ea5f9a5 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -1,4 +1,4 @@ -package evpool +package evidence import ( "bytes" diff --git a/evidence/store.go b/evidence/store.go index 1b4708ea..fa070303 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -1,4 +1,4 @@ -package evpool +package evidence import ( "fmt" @@ -124,11 +124,11 @@ func (store *EvidenceStore) GetEvidence(height int, hash []byte) *EvidenceInfo { } // AddNewEvidence adds the given evidence to the database. -func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) (bool, error) { +func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) bool { // check if we already have seen it ei_ := store.GetEvidence(evidence.Height(), evidence.Hash()) if ei_ != nil && ei_.Evidence != nil { - return false, nil + return false } ei := EvidenceInfo{ @@ -148,7 +148,7 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int key = keyLookup(evidence) store.db.SetSync(key, eiBytes) - return true, nil + return true } // MarkEvidenceAsBroadcasted removes evidence from Outqueue. diff --git a/evidence/store_test.go b/evidence/store_test.go index 76cb794a..6cd9b244 100644 --- a/evidence/store_test.go +++ b/evidence/store_test.go @@ -1,4 +1,4 @@ -package evpool +package evidence import ( "bytes" @@ -22,13 +22,11 @@ func TestStoreAddDuplicate(t *testing.T) { priority := 10 ev := newMockGoodEvidence(2, 1, []byte("val1")) - added, err := store.AddNewEvidence(ev, priority) - assert.Nil(err) + added := store.AddNewEvidence(ev, priority) assert.True(added) // cant add twice - added, err = store.AddNewEvidence(ev, priority) - assert.Nil(err) + added = store.AddNewEvidence(ev, priority) assert.False(added) } @@ -47,8 +45,7 @@ func TestStoreMark(t *testing.T) { priority := 10 ev := newMockGoodEvidence(2, 1, []byte("val1")) - added, err := store.AddNewEvidence(ev, priority) - assert.Nil(err) + added := store.AddNewEvidence(ev, priority) assert.True(added) // get the evidence. verify. should be uncommitted @@ -104,9 +101,8 @@ func TestStorePriority(t *testing.T) { } for _, c := range cases { - added, err := store.AddNewEvidence(c.ev, c.priority) + added := store.AddNewEvidence(c.ev, c.priority) assert.True(added) - assert.Nil(err) } evList := store.PriorityEvidence() diff --git a/node/node.go b/node/node.go index 0d20df66..8134074a 100644 --- a/node/node.go +++ b/node/node.go @@ -19,7 +19,7 @@ import ( bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/consensus" - evidence "github.com/tendermint/tendermint/evidence" + "github.com/tendermint/tendermint/evidence" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/p2p/trust" From cfbedec719dbc218ad85db0d23e8ef33d92fd370 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 04:32:53 +0000 Subject: [PATCH 28/35] evidence: reactor test --- evidence/reactor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evidence/reactor.go b/evidence/reactor.go index 9ea5f9a5..d4d58f7b 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -71,7 +71,7 @@ func (evR *EvidenceReactor) AddPeer(peer p2p.Peer) { // send the peer our high-priority evidence. // the rest will be sent by the broadcastRoutine evidence := evR.evpool.PriorityEvidence() - msg := EvidenceListMessage{evidence} + msg := &EvidenceListMessage{evidence} success := peer.Send(EvidenceChannel, struct{ EvidenceMessage }{msg}) if !success { // TODO: remove peer ? @@ -120,7 +120,7 @@ func (evR *EvidenceReactor) broadcastRoutine() { select { case evidence := <-evR.evpool.EvidenceChan(): // broadcast some new evidence - msg := EvidenceListMessage{[]types.Evidence{evidence}} + msg := &EvidenceListMessage{[]types.Evidence{evidence}} evR.Switch.Broadcast(EvidenceChannel, struct{ EvidenceMessage }{msg}) // TODO: Broadcast runs asynchronously, so this should wait on the successChan @@ -128,7 +128,7 @@ func (evR *EvidenceReactor) broadcastRoutine() { evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(evidence) case <-ticker.C: // broadcast all pending evidence - msg := EvidenceListMessage{evR.evpool.PendingEvidence()} + msg := &EvidenceListMessage{evR.evpool.PendingEvidence()} evR.Switch.Broadcast(EvidenceChannel, struct{ EvidenceMessage }{msg}) case <-evR.Quit: return From 0f293bfc2b64e76fda7aaed3f4a15201aa759736 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 04:40:09 +0000 Subject: [PATCH 29/35] remove some TODOs --- state/execution.go | 1 - types/evidence.go | 3 --- 2 files changed, 4 deletions(-) diff --git a/state/execution.go b/state/execution.go index f6f6a314..a55bc5f8 100644 --- a/state/execution.go +++ b/state/execution.go @@ -313,7 +313,6 @@ func (s *State) validateBlock(b *types.Block) error { if _, err := s.VerifyEvidence(ev); err != nil { return types.NewEvidenceInvalidErr(ev, err) } - // TODO: mark evidence as committed } return nil diff --git a/types/evidence.go b/types/evidence.go index e270a0de..48c2d189 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -126,9 +126,6 @@ func (dve *DuplicateVoteEvidence) Hash() []byte { // Verify returns an error if the two votes aren't conflicting. // To be conflicting, they must be from the same validator, for the same H/R/S, but for different blocks. func (dve *DuplicateVoteEvidence) Verify(chainID string) error { - - // TODO: verify (cs.Height - dve.Height) < MaxHeightDiff - // H/R/S must be the same if dve.VoteA.Height != dve.VoteB.Height || dve.VoteA.Round != dve.VoteB.Round || From 5904f6df8ba0957e822b779c9c22da0cb5c3880d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 20 Nov 2017 18:59:10 +0000 Subject: [PATCH 30/35] minor fixes from review --- evidence/pool.go | 2 +- evidence/store.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/evidence/pool.go b/evidence/pool.go index bfc04f44..097b74a8 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -36,7 +36,7 @@ func (evpool *EvidencePool) SetLogger(l log.Logger) { } // EvidenceChan returns an unbuffered channel on which new evidence can be received. -func (evpool *EvidencePool) EvidenceChan() chan types.Evidence { +func (evpool *EvidencePool) EvidenceChan() <-chan types.Evidence { return evpool.evidenceChan } diff --git a/evidence/store.go b/evidence/store.go index fa070303..e4246cfa 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -85,7 +85,7 @@ func (store *EvidenceStore) PriorityEvidence() (evidence []types.Evidence) { // reverse the order so highest priority is first l := store.ListEvidence(baseKeyOutqueue) l2 := make([]types.Evidence, len(l)) - for i, _ := range l { + for i := range l { l2[i] = l[len(l)-1-i] } return l2 @@ -118,12 +118,13 @@ func (store *EvidenceStore) GetEvidence(height int, hash []byte) *EvidenceInfo { if len(val) == 0 { return nil } - ei := new(EvidenceInfo) - wire.ReadBinaryBytes(val, ei) - return ei + var ei EvidenceInfo + wire.ReadBinaryBytes(val, &ei) + return &ei } // AddNewEvidence adds the given evidence to the database. +// It returns false if the evidence is already stored. func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) bool { // check if we already have seen it ei_ := store.GetEvidence(evidence.Height(), evidence.Hash()) From 014b0b9944e76c0661e9a9770b333c006216a377 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 22 Nov 2017 23:15:10 +0000 Subject: [PATCH 31/35] evidence: reactor test --- evidence/reactor_test.go | 127 +++++++++++++++++++++++++++++++++++++++ evidence/store_test.go | 10 +-- 2 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 evidence/reactor_test.go diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go new file mode 100644 index 00000000..d6e8653b --- /dev/null +++ b/evidence/reactor_test.go @@ -0,0 +1,127 @@ +package evidence + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/go-kit/kit/log/term" + + dbm "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/types" +) + +// evidenceLogger is a TestingLogger which uses a different +// color for each validator ("validator" key must exist). +func evidenceLogger() log.Logger { + return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor { + for i := 0; i < len(keyvals)-1; i += 2 { + if keyvals[i] == "validator" { + return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))} + } + } + return term.FgBgColor{} + }) +} + +// connect N evidence reactors through N switches +func makeAndConnectEvidenceReactors(config *cfg.Config, N int) []*EvidenceReactor { + reactors := make([]*EvidenceReactor, N) + logger := evidenceLogger() + for i := 0; i < N; i++ { + + config := &cfg.EvidenceConfig{} + store := NewEvidenceStore(dbm.NewMemDB()) + state := mockState{} + pool := NewEvidencePool(config, store, state) + reactors[i] = NewEvidenceReactor(config, pool) + reactors[i].SetLogger(logger.With("validator", i)) + } + + p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("EVIDENCE", reactors[i]) + return s + + }, p2p.Connect2Switches) + return reactors +} + +// wait for all evidence on all reactors +func waitForEvidence(t *testing.T, evs types.EvidenceList, reactors []*EvidenceReactor) { + // wait for the evidence in all evpools + wg := new(sync.WaitGroup) + for i := 0; i < len(reactors); i++ { + wg.Add(1) + go _waitForEvidence(t, wg, evs, i, reactors) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + timer := time.After(TIMEOUT) + select { + case <-timer: + t.Fatal("Timed out waiting for evidence") + case <-done: + } +} + +// wait for all evidence on a single evpool +func _waitForEvidence(t *testing.T, wg *sync.WaitGroup, evs types.EvidenceList, reactorIdx int, reactors []*EvidenceReactor) { + + evpool := reactors[reactorIdx].evpool + for len(evpool.PendingEvidence()) != len(evs) { + time.Sleep(time.Millisecond * 100) + } + + reapedEv := evpool.PendingEvidence() + // put the reaped evidence is a map so we can quickly check we got everything + evMap := make(map[string]types.Evidence) + for _, e := range reapedEv { + evMap[string(e.Hash())] = e + } + for i, expectedEv := range evs { + gotEv := evMap[string(expectedEv.Hash())] + assert.Equal(t, expectedEv, gotEv, + fmt.Sprintf("evidence at index %d on reactor %d don't match: %v vs %v", + i, reactorIdx, expectedEv, gotEv)) + } + wg.Done() +} + +func sendEvidence(t *testing.T, evpool *EvidencePool, n int) types.EvidenceList { + evList := make([]types.Evidence, n) + for i := 0; i < n; i++ { + ev := newMockGoodEvidence(i, 2, []byte("val")) + err := evpool.AddEvidence(ev) + assert.Nil(t, err) + evList[i] = ev + } + return evList +} + +var ( + NUM_EVIDENCE = 1000 + TIMEOUT = 120 * time.Second // ridiculously high because CircleCI is slow +) + +func TestReactorBroadcastEvidence(t *testing.T) { + config := cfg.TestConfig() + N := 7 + reactors := makeAndConnectEvidenceReactors(config, N) + + // send a bunch of evidence to the first reactor's evpool + // and wait for them all to be received in the others + evList := sendEvidence(t, reactors[0].evpool, NUM_EVIDENCE) + waitForEvidence(t, evList, reactors) +} diff --git a/evidence/store_test.go b/evidence/store_test.go index 6cd9b244..ca55a98d 100644 --- a/evidence/store_test.go +++ b/evidence/store_test.go @@ -131,10 +131,12 @@ func newMockGoodEvidence(height, index int, address []byte) MockGoodEvidence { return MockGoodEvidence{height, address, index} } -func (e MockGoodEvidence) Height() int { return e.Height_ } -func (e MockGoodEvidence) Address() []byte { return e.Address_ } -func (e MockGoodEvidence) Index() int { return e.Index_ } -func (e MockGoodEvidence) Hash() []byte { return []byte{byte(e.Index_)} } +func (e MockGoodEvidence) Height() int { return e.Height_ } +func (e MockGoodEvidence) Address() []byte { return e.Address_ } +func (e MockGoodEvidence) Index() int { return e.Index_ } +func (e MockGoodEvidence) Hash() []byte { + return []byte(fmt.Sprintf("%d-%d", e.Height_, e.Index_)) +} func (e MockGoodEvidence) Verify(chainID string) error { return nil } func (e MockGoodEvidence) Equal(ev types.Evidence) bool { e2 := ev.(MockGoodEvidence) From b01b1e4758d704668b7f853076390b3ab07e634d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 30 Nov 2017 06:50:51 +0000 Subject: [PATCH 32/35] remove unused var --- evidence/reactor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/evidence/reactor.go b/evidence/reactor.go index d4d58f7b..54bc37a1 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -18,7 +18,6 @@ const ( EvidenceChannel = byte(0x38) maxEvidenceMessageSize = 1048576 // 1MB TODO make it configurable - peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount broadcastEvidenceIntervalS = 60 // broadcast uncommitted evidence this often ) From 6a4fd4647989677d231a86e21813334996b37ab6 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 26 Dec 2017 20:34:57 -0500 Subject: [PATCH 33/35] fixes from rebase --- consensus/replay_test.go | 5 ++--- consensus/wal_generator.go | 3 ++- evidence/pool.go | 7 +++---- evidence/pool_test.go | 7 +++---- evidence/reactor.go | 5 +---- evidence/reactor_test.go | 8 ++++---- evidence/store.go | 12 ++++++------ evidence/store_test.go | 12 ++++++------ node/node.go | 5 ++--- state/execution.go | 2 +- state/state.go | 6 +++--- types/block_test.go | 15 +++++++++++++-- types/evidence.go | 4 ++-- types/evidence_test.go | 13 +------------ types/params.go | 2 +- types/services.go | 2 +- 16 files changed, 51 insertions(+), 57 deletions(-) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 495fc42c..f1a060ec 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -261,9 +261,8 @@ const ( ) var ( - NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal - mempool = types.MockMempool{} - evpool = types.MockEvidencePool{} + mempool = types.MockMempool{} + evpool = types.MockEvidencePool{} ) //--------------------------------------- diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index b46ef93d..b4efb5a9 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -66,7 +66,8 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { } defer eventBus.Stop() mempool := types.MockMempool{} - consensusState := NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool) + evpool := types.MockEvidencePool{} + consensusState := NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool, evpool) consensusState.SetLogger(logger) consensusState.SetEventBus(eventBus) if privValidator != nil { diff --git a/evidence/pool.go b/evidence/pool.go index 097b74a8..821fdf3c 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -3,14 +3,13 @@ package evidence import ( "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" ) // EvidencePool maintains a pool of valid evidence // in an EvidenceStore. type EvidencePool struct { - config *cfg.EvidenceConfig + params types.EvidenceParams logger log.Logger state types.State @@ -19,9 +18,9 @@ type EvidencePool struct { evidenceChan chan types.Evidence } -func NewEvidencePool(config *cfg.EvidenceConfig, evidenceStore *EvidenceStore, state types.State) *EvidencePool { +func NewEvidencePool(params types.EvidenceParams, evidenceStore *EvidenceStore, state types.State) *EvidencePool { evpool := &EvidencePool{ - config: config, + params: params, logger: log.NewNopLogger(), evidenceStore: evidenceStore, state: state, diff --git a/evidence/pool_test.go b/evidence/pool_test.go index 2908bb42..0997505c 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -6,14 +6,13 @@ import ( "github.com/stretchr/testify/assert" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" dbm "github.com/tendermint/tmlibs/db" ) type mockState struct{} -func (m mockState) VerifyEvidence(ev types.Evidence) (int, error) { +func (m mockState) VerifyEvidence(ev types.Evidence) (int64, error) { err := ev.Verify("") return 10, err } @@ -21,10 +20,10 @@ func (m mockState) VerifyEvidence(ev types.Evidence) (int, error) { func TestEvidencePool(t *testing.T) { assert := assert.New(t) - config := &cfg.EvidenceConfig{} + params := types.EvidenceParams{} store := NewEvidenceStore(dbm.NewMemDB()) state := mockState{} - pool := NewEvidencePool(config, store, state) + pool := NewEvidencePool(params, store, state) goodEvidence := newMockGoodEvidence(5, 1, []byte("val1")) badEvidence := MockBadEvidence{goodEvidence} diff --git a/evidence/reactor.go b/evidence/reactor.go index 54bc37a1..ff502ed0 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -9,7 +9,6 @@ import ( wire "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -24,15 +23,13 @@ const ( // EvidenceReactor handles evpool evidence broadcasting amongst peers. type EvidenceReactor struct { p2p.BaseReactor - config *cfg.EvidenceConfig evpool *EvidencePool eventBus *types.EventBus } // NewEvidenceReactor returns a new EvidenceReactor with the given config and evpool. -func NewEvidenceReactor(config *cfg.EvidenceConfig, evpool *EvidencePool) *EvidenceReactor { +func NewEvidenceReactor(evpool *EvidencePool) *EvidenceReactor { evR := &EvidenceReactor{ - config: config, evpool: evpool, } evR.BaseReactor = *p2p.NewBaseReactor("EvidenceReactor", evR) diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go index d6e8653b..fb83667c 100644 --- a/evidence/reactor_test.go +++ b/evidence/reactor_test.go @@ -37,11 +37,11 @@ func makeAndConnectEvidenceReactors(config *cfg.Config, N int) []*EvidenceReacto logger := evidenceLogger() for i := 0; i < N; i++ { - config := &cfg.EvidenceConfig{} + params := types.EvidenceParams{} store := NewEvidenceStore(dbm.NewMemDB()) state := mockState{} - pool := NewEvidencePool(config, store, state) - reactors[i] = NewEvidenceReactor(config, pool) + pool := NewEvidencePool(params, store, state) + reactors[i] = NewEvidenceReactor(pool) reactors[i].SetLogger(logger.With("validator", i)) } @@ -102,7 +102,7 @@ func _waitForEvidence(t *testing.T, wg *sync.WaitGroup, evs types.EvidenceList, func sendEvidence(t *testing.T, evpool *EvidencePool, n int) types.EvidenceList { evList := make([]types.Evidence, n) for i := 0; i < n; i++ { - ev := newMockGoodEvidence(i, 2, []byte("val")) + ev := newMockGoodEvidence(int64(i), 2, []byte("val")) err := evpool.AddEvidence(ev) assert.Nil(t, err) evList[i] = ev diff --git a/evidence/store.go b/evidence/store.go index e4246cfa..5b677c34 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -32,7 +32,7 @@ Schema for indexing evidence (note you need both height and hash to find a piece type EvidenceInfo struct { Committed bool - Priority int + Priority int64 Evidence types.Evidence } @@ -47,15 +47,15 @@ func keyLookup(evidence types.Evidence) []byte { } // big endian padded hex -func be(h int) string { +func be(h int64) string { return fmt.Sprintf("%0.16X", h) } -func keyLookupFromHeightAndHash(height int, hash []byte) []byte { +func keyLookupFromHeightAndHash(height int64, hash []byte) []byte { return _key("%s/%s/%X", baseKeyLookup, be(height), hash) } -func keyOutqueue(evidence types.Evidence, priority int) []byte { +func keyOutqueue(evidence types.Evidence, priority int64) []byte { return _key("%s/%s/%s/%X", baseKeyOutqueue, be(priority), be(evidence.Height()), evidence.Hash()) } @@ -111,7 +111,7 @@ func (store *EvidenceStore) ListEvidence(prefixKey string) (evidence []types.Evi } // GetEvidence fetches the evidence with the given height and hash. -func (store *EvidenceStore) GetEvidence(height int, hash []byte) *EvidenceInfo { +func (store *EvidenceStore) GetEvidence(height int64, hash []byte) *EvidenceInfo { key := keyLookupFromHeightAndHash(height, hash) val := store.db.Get(key) @@ -125,7 +125,7 @@ func (store *EvidenceStore) GetEvidence(height int, hash []byte) *EvidenceInfo { // AddNewEvidence adds the given evidence to the database. // It returns false if the evidence is already stored. -func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int) bool { +func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int64) bool { // check if we already have seen it ei_ := store.GetEvidence(evidence.Height(), evidence.Hash()) if ei_ != nil && ei_.Evidence != nil { diff --git a/evidence/store_test.go b/evidence/store_test.go index ca55a98d..7828d37b 100644 --- a/evidence/store_test.go +++ b/evidence/store_test.go @@ -19,7 +19,7 @@ func TestStoreAddDuplicate(t *testing.T) { db := dbm.NewMemDB() store := NewEvidenceStore(db) - priority := 10 + priority := int64(10) ev := newMockGoodEvidence(2, 1, []byte("val1")) added := store.AddNewEvidence(ev, priority) @@ -42,7 +42,7 @@ func TestStoreMark(t *testing.T) { assert.Equal(0, len(priorityEv)) assert.Equal(0, len(pendingEv)) - priority := 10 + priority := int64(10) ev := newMockGoodEvidence(2, 1, []byte("val1")) added := store.AddNewEvidence(ev, priority) @@ -90,7 +90,7 @@ func TestStorePriority(t *testing.T) { // sorted by priority and then height cases := []struct { ev MockGoodEvidence - priority int + priority int64 }{ {newMockGoodEvidence(2, 1, []byte("val1")), 17}, {newMockGoodEvidence(5, 2, []byte("val2")), 15}, @@ -122,16 +122,16 @@ var _ = wire.RegisterInterface( ) type MockGoodEvidence struct { - Height_ int + Height_ int64 Address_ []byte Index_ int } -func newMockGoodEvidence(height, index int, address []byte) MockGoodEvidence { +func newMockGoodEvidence(height int64, index int, address []byte) MockGoodEvidence { return MockGoodEvidence{height, address, index} } -func (e MockGoodEvidence) Height() int { return e.Height_ } +func (e MockGoodEvidence) Height() int64 { return e.Height_ } func (e MockGoodEvidence) Address() []byte { return e.Address_ } func (e MockGoodEvidence) Index() int { return e.Index_ } func (e MockGoodEvidence) Hash() []byte { diff --git a/node/node.go b/node/node.go index 8134074a..1f51354a 100644 --- a/node/node.go +++ b/node/node.go @@ -210,16 +210,15 @@ func NewNode(config *cfg.Config, } // Make Evidence Reactor - evidenceConfig := &cfg.EvidenceConfig{} // TODO evidenceDB, err := dbProvider(&DBContext{"evidence", config}) if err != nil { return nil, err } evidenceLogger := logger.With("module", "evidence") evidenceStore := evidence.NewEvidenceStore(evidenceDB) - evidencePool := evidence.NewEvidencePool(evidenceConfig, evidenceStore, state) + evidencePool := evidence.NewEvidencePool(state.ConsensusParams.EvidenceParams, evidenceStore, state) evidencePool.SetLogger(evidenceLogger) - evidenceReactor := evidence.NewEvidenceReactor(evidenceConfig, evidencePool) + evidenceReactor := evidence.NewEvidenceReactor(evidencePool) evidenceReactor.SetLogger(evidenceLogger) // Make ConsensusReactor diff --git a/state/execution.go b/state/execution.go index a55bc5f8..c9686152 100644 --- a/state/execution.go +++ b/state/execution.go @@ -309,7 +309,7 @@ func (s *State) validateBlock(b *types.Block) error { } } - for _, ev := range block.Evidence.Evidence { + for _, ev := range b.Evidence.Evidence { if _, err := s.VerifyEvidence(ev); err != nil { return types.NewEvidenceInvalidErr(ev, err) } diff --git a/state/state.go b/state/state.go index 54ad0b10..773b46fc 100644 --- a/state/state.go +++ b/state/state.go @@ -388,9 +388,9 @@ func (s *State) GetValidators() (last *types.ValidatorSet, current *types.Valida // It returns the priority of this evidence, or an error. // NOTE: return error may be ErrNoValSetForHeight, in which case the validator set // for the evidence height could not be loaded. -func (s *State) VerifyEvidence(evidence types.Evidence) (priority int, err error) { +func (s *State) VerifyEvidence(evidence types.Evidence) (priority int64, err error) { evidenceAge := s.LastBlockHeight - evidence.Height() - maxAge := s.Params.EvidenceParams.MaxAge + maxAge := s.ConsensusParams.EvidenceParams.MaxAge if evidenceAge > maxAge { return priority, fmt.Errorf("Evidence from height %d is too old. Min height is %d", evidence.Height(), s.LastBlockHeight-maxAge) @@ -416,7 +416,7 @@ func (s *State) VerifyEvidence(evidence types.Evidence) (priority int, err error return priority, fmt.Errorf("Address %X was validator %d at height %d, not %d", addr, valIdx, height, idx) } - priority = int(val.VotingPower) + priority = val.VotingPower return priority, nil } diff --git a/types/block_test.go b/types/block_test.go index bde47440..1fcfa469 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -10,7 +10,7 @@ import ( func TestValidateBlock(t *testing.T) { txs := []Tx{Tx("foo"), Tx("bar")} - lastID := makeBlockID() + lastID := makeBlockIDRandom() h := int64(3) voteSet, _, vals := randVoteSet(h-1, 1, VoteTypePrecommit, @@ -58,7 +58,18 @@ func TestValidateBlock(t *testing.T) { require.Error(t, err) } -func makeBlockID() BlockID { +func makeBlockIDRandom() BlockID { blockHash, blockPartsHeader := crypto.CRandBytes(32), PartSetHeader{123, crypto.CRandBytes(32)} return BlockID{blockHash, blockPartsHeader} } + +func makeBlockID(hash string, partSetSize int, partSetHash string) BlockID { + return BlockID{ + Hash: []byte(hash), + PartsHeader: PartSetHeader{ + Total: partSetSize, + Hash: []byte(partSetHash), + }, + } + +} diff --git a/types/evidence.go b/types/evidence.go index 48c2d189..d6b40811 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -28,7 +28,7 @@ func (err *ErrEvidenceInvalid) Error() string { // Evidence represents any provable malicious activity by a validator type Evidence interface { - Height() int // height of the equivocation + Height() int64 // height of the equivocation Address() []byte // address of the equivocating validator Index() int // index of the validator in the validator set Hash() []byte // hash of the evidence @@ -104,7 +104,7 @@ func (dve *DuplicateVoteEvidence) String() string { } // Height returns the height this evidence refers to. -func (dve *DuplicateVoteEvidence) Height() int { +func (dve *DuplicateVoteEvidence) Height() int64 { return dve.VoteA.Height } diff --git a/types/evidence_test.go b/types/evidence_test.go index a69040d4..876b68ad 100644 --- a/types/evidence_test.go +++ b/types/evidence_test.go @@ -13,7 +13,7 @@ type voteData struct { valid bool } -func makeVote(val *PrivValidatorFS, chainID string, valIndex, height, round, step int, blockID BlockID) *Vote { +func makeVote(val *PrivValidatorFS, chainID string, valIndex int, height int64, round, step int, blockID BlockID) *Vote { v := &Vote{ ValidatorAddress: val.PubKey.Address(), ValidatorIndex: valIndex, @@ -28,17 +28,6 @@ func makeVote(val *PrivValidatorFS, chainID string, valIndex, height, round, ste } -func makeBlockID(hash string, partSetSize int, partSetHash string) BlockID { - return BlockID{ - Hash: []byte(hash), - PartsHeader: PartSetHeader{ - Total: partSetSize, - Hash: []byte(partSetHash), - }, - } - -} - func TestEvidence(t *testing.T) { _, tmpFilePath := cmn.Tempfile("priv_validator_") val := GenPrivValidatorFS(tmpFilePath) diff --git a/types/params.go b/types/params.go index 216f1390..90ab5f65 100644 --- a/types/params.go +++ b/types/params.go @@ -40,7 +40,7 @@ type BlockGossip struct { // EvidenceParams determine how we handle evidence of malfeasance type EvidenceParams struct { - MaxAge int `json:"max_age"` // only accept new evidence more recent than this + MaxAge int64 `json:"max_age"` // only accept new evidence more recent than this } // DefaultConsensusParams returns a default ConsensusParams. diff --git a/types/services.go b/types/services.go index b23ccff4..787b1b99 100644 --- a/types/services.go +++ b/types/services.go @@ -76,7 +76,7 @@ type BlockStore interface { // State defines the stateful interface used to verify evidence. // UNSTABLE type State interface { - VerifyEvidence(Evidence) (priority int, err error) + VerifyEvidence(Evidence) (priority int64, err error) } //------------------------------------------------------ From 7d81a3f4a5fba31a08f13e0f0180c9f9c8b934ae Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 27 Dec 2017 01:27:03 -0500 Subject: [PATCH 34/35] address some comments from review --- consensus/replay.go | 1 + evidence/pool.go | 9 +++++++-- evidence/reactor.go | 8 ++++---- evidence/store.go | 11 ++++------- node/node.go | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 56f09d12..209ea597 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -323,6 +323,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store // Note that we don't have an old version of the state, // so we by-pass state validation/mutation using sm.ExecCommitBlock. // This also means we won't be saving validator sets if they change during this period. + // TODO: Load the historical information to fix this and just use state.ApplyBlock // // If mutateState == true, the final block is replayed with h.replayBlock() diff --git a/evidence/pool.go b/evidence/pool.go index 821fdf3c..2296ac02 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -12,9 +12,10 @@ type EvidencePool struct { params types.EvidenceParams logger log.Logger - state types.State + state types.State // TODO: update this on commit! evidenceStore *EvidenceStore + // never close evidenceChan chan types.Evidence } @@ -52,10 +53,13 @@ func (evpool *EvidencePool) PendingEvidence() []types.Evidence { // AddEvidence checks the evidence is valid and adds it to the pool. // Blocks on the EvidenceChan. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { + // TODO: check if we already have evidence for this + // validator at this height so we dont get spammed - // XXX: is this thread safe ? priority, err := evpool.state.VerifyEvidence(evidence) if err != nil { + // TODO: if err is just that we cant find it cuz we pruned, ignore. + // TODO: if its actually bad evidence, punish peer return err } @@ -67,6 +71,7 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence) + // never closes. always safe to send on evpool.evidenceChan <- evidence return nil } diff --git a/evidence/reactor.go b/evidence/reactor.go index ff502ed0..cb9706a3 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -66,8 +66,8 @@ func (evR *EvidenceReactor) GetChannels() []*p2p.ChannelDescriptor { func (evR *EvidenceReactor) AddPeer(peer p2p.Peer) { // send the peer our high-priority evidence. // the rest will be sent by the broadcastRoutine - evidence := evR.evpool.PriorityEvidence() - msg := &EvidenceListMessage{evidence} + evidences := evR.evpool.PriorityEvidence() + msg := &EvidenceListMessage{evidences} success := peer.Send(EvidenceChannel, struct{ EvidenceMessage }{msg}) if !success { // TODO: remove peer ? @@ -108,8 +108,8 @@ func (evR *EvidenceReactor) SetEventBus(b *types.EventBus) { evR.eventBus = b } -// broadcast new evidence to all peers. -// broadcasts must be non-blocking so routine is always available to read off EvidenceChan. +// Broadcast new evidence to all peers. +// Broadcasts must be non-blocking so routine is always available to read off EvidenceChan. func (evR *EvidenceReactor) broadcastRoutine() { ticker := time.NewTicker(time.Second * broadcastEvidenceIntervalS) for { diff --git a/evidence/store.go b/evidence/store.go index 5b677c34..fd40b533 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -47,20 +47,20 @@ func keyLookup(evidence types.Evidence) []byte { } // big endian padded hex -func be(h int64) string { +func bE(h int64) string { return fmt.Sprintf("%0.16X", h) } func keyLookupFromHeightAndHash(height int64, hash []byte) []byte { - return _key("%s/%s/%X", baseKeyLookup, be(height), hash) + return _key("%s/%s/%X", baseKeyLookup, bE(height), hash) } func keyOutqueue(evidence types.Evidence, priority int64) []byte { - return _key("%s/%s/%s/%X", baseKeyOutqueue, be(priority), be(evidence.Height()), evidence.Hash()) + return _key("%s/%s/%s/%X", baseKeyOutqueue, bE(priority), bE(evidence.Height()), evidence.Hash()) } func keyPending(evidence types.Evidence) []byte { - return _key("%s/%s/%X", baseKeyPending, be(evidence.Height()), evidence.Hash()) + return _key("%s/%s/%X", baseKeyPending, bE(evidence.Height()), evidence.Hash()) } func _key(fmt_ string, o ...interface{}) []byte { @@ -170,9 +170,6 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { ei := store.getEvidenceInfo(evidence) ei.Committed = true - // TODO: we should use the state db and db.Sync in state.Save instead. - // Else, if we call this before state.Save, we may never mark committed evidence as committed. - // Else, if we call this after state.Save, we may get stuck broadcasting evidence we never know we committed. lookupKey := keyLookup(evidence) store.db.SetSync(lookupKey, wire.BinaryBytes(ei)) } diff --git a/node/node.go b/node/node.go index 1f51354a..53eab6e0 100644 --- a/node/node.go +++ b/node/node.go @@ -216,7 +216,7 @@ func NewNode(config *cfg.Config, } evidenceLogger := logger.With("module", "evidence") evidenceStore := evidence.NewEvidenceStore(evidenceDB) - evidencePool := evidence.NewEvidencePool(state.ConsensusParams.EvidenceParams, evidenceStore, state) + evidencePool := evidence.NewEvidencePool(state.ConsensusParams.EvidenceParams, evidenceStore, state.Copy()) evidencePool.SetLogger(evidenceLogger) evidenceReactor := evidence.NewEvidenceReactor(evidencePool) evidenceReactor.SetLogger(evidenceLogger) From d0e0ac5fac0fee3bea81b314e5c4ebe19f09db65 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 27 Dec 2017 14:46:24 -0500 Subject: [PATCH 35/35] types: better error messages for votes --- types/evidence.go | 4 ++-- types/vote.go | 13 +++++++------ types/vote_set.go | 21 ++++++++++++++------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/types/evidence.go b/types/evidence.go index d6b40811..94bab1a3 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -149,10 +149,10 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string) error { // Signatures must be valid if !dve.PubKey.VerifyBytes(SignBytes(chainID, dve.VoteA), dve.VoteA.Signature) { - return ErrVoteInvalidSignature + return fmt.Errorf("DuplicateVoteEvidence Error verifying VoteA: %v", ErrVoteInvalidSignature) } if !dve.PubKey.VerifyBytes(SignBytes(chainID, dve.VoteB), dve.VoteB.Signature) { - return ErrVoteInvalidSignature + return fmt.Errorf("DuplicateVoteEvidence Error verifying VoteB: %v", ErrVoteInvalidSignature) } return nil diff --git a/types/vote.go b/types/vote.go index 3136e4bb..4397152c 100644 --- a/types/vote.go +++ b/types/vote.go @@ -14,12 +14,13 @@ import ( ) var ( - ErrVoteUnexpectedStep = errors.New("Unexpected step") - ErrVoteInvalidValidatorIndex = errors.New("Invalid validator index") - ErrVoteInvalidValidatorAddress = errors.New("Invalid validator address") - ErrVoteInvalidSignature = errors.New("Invalid signature") - ErrVoteInvalidBlockHash = errors.New("Invalid block hash") - ErrVoteNil = errors.New("Nil vote") + ErrVoteUnexpectedStep = errors.New("Unexpected step") + ErrVoteInvalidValidatorIndex = errors.New("Invalid validator index") + ErrVoteInvalidValidatorAddress = errors.New("Invalid validator address") + ErrVoteInvalidSignature = errors.New("Invalid signature") + ErrVoteInvalidBlockHash = errors.New("Invalid block hash") + ErrVoteNonDeterministicSignature = errors.New("Non-deterministic signature") + ErrVoteNil = errors.New("Nil vote") ) type ErrVoteConflictingVotes struct { diff --git a/types/vote_set.go b/types/vote_set.go index 3f8a96e3..34f98956 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -6,6 +6,8 @@ import ( "strings" "sync" + "github.com/pkg/errors" + cmn "github.com/tendermint/tmlibs/common" ) @@ -145,27 +147,32 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { // Ensure that validator index was set if valIndex < 0 { - return false, ErrVoteInvalidValidatorIndex + return false, errors.Wrap(ErrVoteInvalidValidatorIndex, "Index < 0") } else if len(valAddr) == 0 { - return false, ErrVoteInvalidValidatorAddress + return false, errors.Wrap(ErrVoteInvalidValidatorAddress, "Empty address") } // Make sure the step matches. if (vote.Height != voteSet.height) || (vote.Round != voteSet.round) || (vote.Type != voteSet.type_) { - return false, ErrVoteUnexpectedStep + return false, errors.Wrapf(ErrVoteUnexpectedStep, "Got %d/%d/%d, expected %d/%d/%d", + voteSet.height, voteSet.round, voteSet.type_, + vote.Height, vote.Round, vote.Type) } // Ensure that signer is a validator. lookupAddr, val := voteSet.valSet.GetByIndex(valIndex) if val == nil { - return false, ErrVoteInvalidValidatorIndex + return false, errors.Wrapf(ErrVoteInvalidValidatorIndex, + "Cannot find validator %d in valSet of size %d", valIndex, voteSet.valSet.Size()) } // Ensure that the signer has the right address if !bytes.Equal(valAddr, lookupAddr) { - return false, ErrVoteInvalidValidatorAddress + return false, errors.Wrapf(ErrVoteInvalidValidatorAddress, + "vote.ValidatorAddress (%X) does not match address (%X) for vote.ValidatorIndex (%d)", + valAddr, lookupAddr, valIndex) } // If we already know of this vote, return false. @@ -173,13 +180,13 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { if existing.Signature.Equals(vote.Signature) { return false, nil // duplicate } else { - return false, ErrVoteInvalidSignature // NOTE: assumes deterministic signatures + return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } } // Check signature. if err := vote.Verify(voteSet.chainID, val.PubKey); err != nil { - return false, err + return false, errors.Wrapf(err, "Failed to verify vote with ChainID %s and PubKey %s", voteSet.chainID, val.PubKey) } // Add vote and get conflicting vote if any