fluencelabs/tendermint
1
0
Fork 0
mirror of https://github.com/fluencelabs/tendermint synced 2025-06-14 13:51:21 +00:00
Code Issues Projects Releases Wiki Activity

consensus: handshake replay test using wal

Browse Source
This commit is contained in:
Ethan Buchman
2017-02-17 19:12:05 -05:00
parent 3c5adebcd3
commit 0bec99fbd4
11 changed files with 496 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/tendermint/reset_priv_validator.go
View File

@ -6,15 +6,15 @@ import (
"github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/types"
) )
// NOTE: this is totally unsafe. // XXX: this is totally unsafe.
// it's only suitable for testnets. // it's only suitable for testnets.
func reset_all() { func reset_all() {
reset_priv_validator() reset_priv_validator()
os.RemoveAll(config.GetString("db_dir")) os.RemoveAll(config.GetString("db_dir"))
os.RemoveAll(config.GetString("cs_wal_dir")) os.Remove(config.GetString("cs_wal_file"))
} }
// NOTE: this is totally unsafe. // XXX: this is totally unsafe.
// it's only suitable for testnets. // it's only suitable for testnets.
func reset_priv_validator() { func reset_priv_validator() {
// Get PrivValidator // Get PrivValidator

2
config/tendermint/config.go
View File

@ -72,7 +72,7 @@ func GetConfig(rootDir string) cfg.Config {
mapConfig.SetDefault("grpc_laddr", "") mapConfig.SetDefault("grpc_laddr", "")
mapConfig.SetDefault("prof_laddr", "") mapConfig.SetDefault("prof_laddr", "")
mapConfig.SetDefault("revision_file", rootDir+"/revision") mapConfig.SetDefault("revision_file", rootDir+"/revision")
mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal") mapConfig.SetDefault("cs_wal_file", rootDir+"/data/cs.wal/wal")
mapConfig.SetDefault("cs_wal_light", false) mapConfig.SetDefault("cs_wal_light", false)
mapConfig.SetDefault("filter_peers", false) mapConfig.SetDefault("filter_peers", false)

2
config/tendermint_test/config.go
View File

@ -86,7 +86,7 @@ func ResetConfig(localPath string) cfg.Config {
mapConfig.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658") mapConfig.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658")
mapConfig.SetDefault("prof_laddr", "") mapConfig.SetDefault("prof_laddr", "")
mapConfig.SetDefault("revision_file", rootDir+"/revision") mapConfig.SetDefault("revision_file", rootDir+"/revision")
mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal") mapConfig.SetDefault("cs_wal_file", rootDir+"/data/cs.wal/wal")
mapConfig.SetDefault("cs_wal_light", false) mapConfig.SetDefault("cs_wal_light", false)
mapConfig.SetDefault("filter_peers", false) mapConfig.SetDefault("filter_peers", false)

8
consensus/common_test.go
View File

@ -11,6 +11,8 @@ import (
"testing" "testing"
"time" "time"
abcicli "github.com/tendermint/abci/client"
abci "github.com/tendermint/abci/types"
. "github.com/tendermint/go-common" . "github.com/tendermint/go-common"
cfg "github.com/tendermint/go-config" cfg "github.com/tendermint/go-config"
dbm "github.com/tendermint/go-db" dbm "github.com/tendermint/go-db"
@ -20,8 +22,6 @@ import (
mempl "github.com/tendermint/tendermint/mempool" mempl "github.com/tendermint/tendermint/mempool"
sm "github.com/tendermint/tendermint/state" sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/types"
abcicli "github.com/tendermint/abci/client"
abci "github.com/tendermint/abci/types"
"github.com/tendermint/abci/example/counter" "github.com/tendermint/abci/example/counter"
"github.com/tendermint/abci/example/dummy" "github.com/tendermint/abci/example/dummy"
@ -320,7 +320,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
state := sm.MakeGenesisState(db, genDoc) state := sm.MakeGenesisState(db, genDoc)
state.Save() state.Save()
thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal ensureDir(path.Dir(thisConfig.GetString("cs_wal_file")), 0700) // dir for wal
css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc()) css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc())
css[i].SetTimeoutTicker(tickerFunc()) css[i].SetTimeoutTicker(tickerFunc())
} }
@ -336,7 +336,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
state := sm.MakeGenesisState(db, genDoc) state := sm.MakeGenesisState(db, genDoc)
state.Save() state.Save()
thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal ensureDir(path.Dir(thisConfig.GetString("cs_wal_file")), 0700) // dir for wal
var privVal *types.PrivValidator var privVal *types.PrivValidator
if i < nValidators { if i < nValidators {
privVal = privVals[i] privVal = privVals[i]

216
consensus/handshake_test.go_
View File

@ -1,216 +0,0 @@
package consensus
import (
"bytes"
"fmt"
"path"
"testing"
"github.com/tendermint/tendermint/config/tendermint_test"
// . "github.com/tendermint/go-common"
"github.com/tendermint/abci/example/dummy"
cfg "github.com/tendermint/go-config"
"github.com/tendermint/go-crypto"
dbm "github.com/tendermint/go-db"
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
)
var (
privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test"))
chainID = "handshake_chain"
nBlocks = 5
mempool = sm.MockMempool{}
testPartSize = 65536
)
//---------------------------------------
// Test handshake/replay
// Sync from scratch
func _TestHandshakeReplayAll(t *testing.T) {
testHandshakeReplay(t, 0)
}
// Sync many, not from scratch
func _TestHandshakeReplaySome(t *testing.T) {
testHandshakeReplay(t, 1)
}
// Sync from lagging by one
func _TestHandshakeReplayOne(t *testing.T) {
testHandshakeReplay(t, nBlocks-1)
}
// Sync from caught up
func TestHandshakeReplayNone(t *testing.T) {
testHandshakeReplay(t, nBlocks)
}
// Make some blocks. Start a fresh app and apply n blocks. Then restart the app and sync it up with the remaining blocks
func testHandshakeReplay(t *testing.T, n int) {
config := tendermint_test.ResetConfig("proxy_test_")
config.Set("chain_id", chainID)
state, store := stateAndStore(config)
clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, store, ReplayLastBlock))
if _, err := proxyApp.Start(); err != nil {
t.Fatalf("Error starting proxy app connections: %v", err)
}
chain, commits := makeBlockchain(t, proxyApp, state)
store.chain = chain //
store.commits = commits
latestAppHash := state.AppHash
proxyApp.Stop()
if n > 0 {
// start a new app without handshake, play n blocks
proxyApp = proxy.NewAppConns(config, clientCreator2, nil)
if _, err := proxyApp.Start(); err != nil {
t.Fatalf("Error starting proxy app connections: %v", err)
}
state2, _ := stateAndStore(config)
for i := 0; i < n; i++ {
block := chain[i]
err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
if err != nil {
t.Fatal(err)
}
}
proxyApp.Stop()
}
// now start it with the handshake
handshaker := sm.NewHandshaker(config, state, store, ReplayLastBlock)
proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
if _, err := proxyApp.Start(); err != nil {
t.Fatalf("Error starting proxy app connections: %v", err)
}
// get the latest app hash from the app
res, err := proxyApp.Query().InfoSync()
if err != nil {
t.Fatal(err)
}
// the app hash should be synced up
if !bytes.Equal(latestAppHash, res.LastBlockAppHash) {
t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
}
if handshaker.NBlocks() != nBlocks-n {
t.Fatalf("Expected handshake to sync %d blocks, got %d", nBlocks-n, handshaker.NBlocks())
}
}
//--------------------------
// utils for making blocks
// make some bogus txs
func txsFunc(blockNum int) (txs []types.Tx) {
for i := 0; i < 10; i++ {
txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)}))
}
return txs
}
// sign a commit vote
func signCommit(height, round int, hash []byte, header types.PartSetHeader) *types.Vote {
vote := &types.Vote{
ValidatorIndex: 0,
ValidatorAddress: privKey.PubKey().Address(),
Height: height,
Round: round,
Type: types.VoteTypePrecommit,
BlockID: types.BlockID{hash, header},
}
sig := privKey.Sign(types.SignBytes(chainID, vote))
vote.Signature = sig
return vote
}
// make a blockchain with one validator
func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *sm.State) (blockchain []*types.Block, commits []*types.Commit) {
prevHash := state.LastBlockID.Hash
lastCommit := new(types.Commit)
prevParts := types.PartSetHeader{}
valHash := state.Validators.Hash()
prevBlockID := types.BlockID{prevHash, prevParts}
for i := 1; i < nBlocks+1; i++ {
block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit,
prevBlockID, valHash, state.AppHash, testPartSize)
fmt.Println(i)
fmt.Println(block.LastBlockID)
err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
if err != nil {
t.Fatal(i, err)
}
voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators)
vote := signCommit(i, 0, block.Hash(), parts.Header())
_, err = voteSet.AddVote(vote)
if err != nil {
t.Fatal(err)
}
prevHash = block.Hash()
prevParts = parts.Header()
lastCommit = voteSet.MakeCommit()
prevBlockID = types.BlockID{prevHash, prevParts}
blockchain = append(blockchain, block)
commits = append(commits, lastCommit)
}
return blockchain, commits
}
// fresh state and mock store
func stateAndStore(config cfg.Config) (*sm.State, *mockBlockStore) {
stateDB := dbm.NewMemDB()
return sm.MakeGenesisState(stateDB, &types.GenesisDoc{
ChainID: chainID,
Validators: []types.GenesisValidator{
types.GenesisValidator{privKey.PubKey(), 10000, "test"},
},
AppHash: nil,
}), NewMockBlockStore(config)
}
//----------------------------------
// mock block store
type mockBlockStore struct {
config cfg.Config
chain []*types.Block
commits []*types.Commit
}
func NewMockBlockStore(config cfg.Config) *mockBlockStore {
return &mockBlockStore{config, nil, nil}
}
func (bs *mockBlockStore) Height() int { return len(bs.chain) }
func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
block := bs.chain[height-1]
return &types.BlockMeta{
BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.GetInt("block_part_size")).Header()},
Header: block.Header,
}
}
func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
}
func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
return bs.commits[height-1]
}
func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
return bs.commits[height-1]
}

377
consensus/replay_test.go
View File

@ -1,7 +1,10 @@
package consensus package consensus
import ( import (
"bytes"
"errors"
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
@ -11,8 +14,14 @@ import (
"github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/config/tendermint_test"
"github.com/tendermint/abci/example/dummy"
. "github.com/tendermint/go-common" . "github.com/tendermint/go-common"
cfg "github.com/tendermint/go-config"
"github.com/tendermint/go-crypto"
dbm "github.com/tendermint/go-db"
"github.com/tendermint/go-wire" "github.com/tendermint/go-wire"
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/types"
) )
@ -20,14 +29,23 @@ func init() {
config = tendermint_test.ResetConfig("consensus_replay_test") config = tendermint_test.ResetConfig("consensus_replay_test")
} }
// TODO: these tests ensure we can always recover from any state of the wal, // These tests ensure we can always recover from failure at any part of the consensus process.
// assuming it comes with a correct related state for the priv_validator.json. // There are two general failure scenarios: failure during consensus, and failure while applying the block.
// It would be better to verify explicitly which states we can recover from without the wal // Only the latter interacts with the app and store,
// and which ones we need the wal for - then we'd also be able to only flush the // but the former has to deal with restrictions on re-use of priv_validator keys.
// wal writer when we need to, instead of with every message. // The `WAL Tests` are for failures during the consensus;
// the `Handshake Tests` are for failures in applying the block.
// With the help of the WAL, we can recover from it all!
var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data") var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
//------------------------------------------------------------------------------------------
// WAL Tests
// TODO: It would be better to verify explicitly which states we can recover from without the wal
// and which ones we need the wal for - then we'd also be able to only flush the
// wal writer when we need to, instead of with every message.
// the priv validator changes step at these lines for a block with 1 val and 1 part // the priv validator changes step at these lines for a block with 1 val and 1 part
var baseStepChanges = []int{3, 6, 8} var baseStepChanges = []int{3, 6, 8}
@ -85,18 +103,19 @@ func readWAL(p string) string {
func writeWAL(walMsgs string) string { func writeWAL(walMsgs string) string {
tempDir := os.TempDir() tempDir := os.TempDir()
walDir := tempDir + "/wal" + RandStr(12) walDir := path.Join(tempDir, "/wal"+RandStr(12))
walFile := path.Join(walDir, "wal")
// Create WAL directory // Create WAL directory
err := EnsureDir(walDir, 0700) err := EnsureDir(walDir, 0700)
if err != nil { if err != nil {
panic(err) panic(err)
} }
// Write the needed WAL to file // Write the needed WAL to file
err = WriteFile(walDir+"/wal", []byte(walMsgs), 0600) err = WriteFile(walFile, []byte(walMsgs), 0600)
if err != nil { if err != nil {
panic(err) panic(err)
} }
return walDir return walFile
} }
func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) { func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
@ -108,10 +127,10 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
} }
} }
func runReplayTest(t *testing.T, cs *ConsensusState, walDir string, newBlockCh chan interface{}, func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{},
thisCase *testCase, i int) { thisCase *testCase, i int) {
cs.config.Set("cs_wal_dir", walDir) cs.config.Set("cs_wal_file", walFile)
cs.Start() cs.Start()
// Wait to make a new block. // Wait to make a new block.
// This is just a signal that we haven't halted; its not something contained in the WAL itself. // This is just a signal that we haven't halted; its not something contained in the WAL itself.
@ -148,7 +167,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu
lastMsg := split[nLines] lastMsg := split[nLines]
// we write those lines up to (not including) one with the signature // we write those lines up to (not including) one with the signature
walDir := writeWAL(strings.Join(split[:nLines], "\n") + "\n") walFile := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
cs := fixedConsensusStateDummy() cs := fixedConsensusStateDummy()
@ -160,7 +179,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu
newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1) newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
return cs, newBlockCh, lastMsg, walDir return cs, newBlockCh, lastMsg, walFile
} }
func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage { func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
@ -177,12 +196,12 @@ func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
// Test the log at every iteration, and set the privVal last step // Test the log at every iteration, and set the privVal last step
// as if the log was written after signing, before the crash // as if the log was written after signing, before the crash
func TestReplayCrashAfterWrite(t *testing.T) { func TestWALCrashAfterWrite(t *testing.T) {
for _, thisCase := range testCases { for _, thisCase := range testCases {
split := strings.Split(thisCase.log, "\n") split := strings.Split(thisCase.log, "\n")
for i := 0; i < len(split)-1; i++ { for i := 0; i < len(split)-1; i++ {
cs, newBlockCh, _, walDir := setupReplayTest(thisCase, i+1, true) cs, newBlockCh, _, walFile := setupReplayTest(thisCase, i+1, true)
runReplayTest(t, cs, walDir, newBlockCh, thisCase, i+1) runReplayTest(t, cs, walFile, newBlockCh, thisCase, i+1)
} }
} }
} }
@ -191,27 +210,27 @@ func TestReplayCrashAfterWrite(t *testing.T) {
// Test the log as if we crashed after signing but before writing. // Test the log as if we crashed after signing but before writing.
// This relies on privValidator.LastSignature being set // This relies on privValidator.LastSignature being set
func TestReplayCrashBeforeWritePropose(t *testing.T) { func TestWALCrashBeforeWritePropose(t *testing.T) {
for _, thisCase := range testCases { for _, thisCase := range testCases {
lineNum := thisCase.proposeLine lineNum := thisCase.proposeLine
// setup replay test where last message is a proposal // setup replay test where last message is a proposal
cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false) cs, newBlockCh, proposalMsg, walFile := setupReplayTest(thisCase, lineNum, false)
msg := readTimedWALMessage(t, proposalMsg) msg := readTimedWALMessage(t, proposalMsg)
proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage) proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage)
// Set LastSig // Set LastSig
toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal)
toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature
runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
} }
} }
func TestReplayCrashBeforeWritePrevote(t *testing.T) { func TestWALCrashBeforeWritePrevote(t *testing.T) {
for _, thisCase := range testCases { for _, thisCase := range testCases {
testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal()) testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal())
} }
} }
func TestReplayCrashBeforeWritePrecommit(t *testing.T) { func TestWALCrashBeforeWritePrecommit(t *testing.T) {
for _, thisCase := range testCases { for _, thisCase := range testCases {
testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka()) testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka())
} }
@ -219,7 +238,7 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) {
func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) { func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) {
// setup replay test where last message is a vote // setup replay test where last message is a vote
cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) cs, newBlockCh, voteMsg, walFile := setupReplayTest(thisCase, lineNum, false)
types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) { types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) {
msg := readTimedWALMessage(t, voteMsg) msg := readTimedWALMessage(t, voteMsg)
vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) vote := msg.Msg.(msgInfo).Msg.(*VoteMessage)
@ -227,5 +246,319 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in
toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote)
toPV(cs.privValidator).LastSignature = vote.Vote.Signature toPV(cs.privValidator).LastSignature = vote.Vote.Signature
}) })
runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
}
//------------------------------------------------------------------------------------------
// Handshake Tests
var (
NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal
mempool = sm.MockMempool{}
testPartSize int
)
//---------------------------------------
// Test handshake/replay
// Sync from scratch
func TestHandshakeReplayAll(t *testing.T) {
testHandshakeReplay(t, 0)
}
// Sync many, not from scratch
func TestHandshakeReplaySome(t *testing.T) {
testHandshakeReplay(t, 1)
}
// Sync from lagging by one
func TestHandshakeReplayOne(t *testing.T) {
testHandshakeReplay(t, NUM_BLOCKS-1)
}
// Sync from caught up
func TestHandshakeReplayNone(t *testing.T) {
testHandshakeReplay(t, NUM_BLOCKS)
}
// Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
func testHandshakeReplay(t *testing.T, nBlocks int) {
config := tendermint_test.ResetConfig("proxy_test_")
walFile := path.Join(data_dir, "many_blocks.cswal")
config.Set("cs_wal_file", walFile)
privVal := types.LoadPrivValidator(config.GetString("priv_validator_file"))
testPartSize = config.GetInt("block_part_size")
wal, err := NewWAL(walFile, false)
if err != nil {
t.Fatal(err)
}
chain, commits, err := makeBlockchainFromWAL(wal)
if err != nil {
t.Fatalf(err.Error())
}
state, store := stateAndStore(config, privVal.PubKey)
store.chain = chain
store.commits = commits
// run the whole chain against this client to build up the tendermint state
clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
proxyApp := proxy.NewAppConns(config, clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
if _, err := proxyApp.Start(); err != nil {
t.Fatalf("Error starting proxy app connections: %v", err)
}
for _, block := range chain {
err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
if err != nil {
t.Fatal(err)
}
}
proxyApp.Stop()
latestAppHash := state.AppHash
// run nBlocks against a new client to build up the app state.
// use a throwaway tendermint state
clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
if nBlocks > 0 {
// start a new app without handshake, play nBlocks blocks
proxyApp := proxy.NewAppConns(config, clientCreator2, nil)
if _, err := proxyApp.Start(); err != nil {
t.Fatalf("Error starting proxy app connections: %v", err)
}
state2, _ := stateAndStore(config, privVal.PubKey)
for i := 0; i < nBlocks; i++ {
block := chain[i]
err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
if err != nil {
t.Fatal(err)
}
}
proxyApp.Stop()
}
// now start the app using the handshake - it should sync
handshaker := sm.NewHandshaker(config, state, store, ReplayLastBlock)
proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
if _, err := proxyApp.Start(); err != nil {
t.Fatalf("Error starting proxy app connections: %v", err)
}
// get the latest app hash from the app
res, err := proxyApp.Query().InfoSync()
if err != nil {
t.Fatal(err)
}
// the app hash should be synced up
if !bytes.Equal(latestAppHash, res.LastBlockAppHash) {
t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
}
if handshaker.NBlocks() != NUM_BLOCKS-nBlocks {
t.Fatalf("Expected handshake to sync %d blocks, got %d", NUM_BLOCKS-nBlocks, handshaker.NBlocks())
}
}
//--------------------------
// utils for making blocks
func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
// Search for height marker
gr, found, err := wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(1))
if err != nil {
return nil, nil, err
}
if !found {
return nil, nil, errors.New(Fmt("WAL does not contain height %d.", 1))
}
defer gr.Close()
log.Notice("Build a blockchain by reading from the WAL")
var blockParts *types.PartSet
var blocks []*types.Block
var commits []*types.Commit
for {
line, err := gr.ReadLine()
if err != nil {
if err == io.EOF {
break
} else {
return nil, nil, err
}
}
piece, err := readPieceFromWAL([]byte(line))
if err != nil {
return nil, nil, err
}
if piece == nil {
continue
}
switch p := piece.(type) {
case *types.PartSetHeader:
// if its not the first one, we have a full block
if blockParts != nil {
var n int
block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
blocks = append(blocks, block)
}
blockParts = types.NewPartSetFromHeader(*p)
case *types.Part:
_, err := blockParts.AddPart(p, false)
if err != nil {
return nil, nil, err
}
case *types.Vote:
if p.Type == types.VoteTypePrecommit {
commit := &types.Commit{
BlockID: p.BlockID,
Precommits: []*types.Vote{p},
}
commits = append(commits, commit)
}
}
}
// grab the last block too
var n int
block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
blocks = append(blocks, block)
return blocks, commits, nil
}
func readPieceFromWAL(msgBytes []byte) (interface{}, error) {
// Skip over empty and meta lines
if len(msgBytes) == 0 || msgBytes[0] == '#' {
return nil, nil
}
var err error
var msg TimedWALMessage
wire.ReadJSON(&msg, msgBytes, &err)
if err != nil {
fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
return nil, fmt.Errorf("Error reading json data: %v", err)
}
// for logging
switch m := msg.Msg.(type) {
case msgInfo:
switch msg := m.Msg.(type) {
case *ProposalMessage:
return &msg.Proposal.BlockPartsHeader, nil
case *BlockPartMessage:
return msg.Part, nil
case *VoteMessage:
return msg.Vote, nil
}
}
return nil, nil
}
// make some bogus txs
func txsFunc(blockNum int) (txs []types.Tx) {
for i := 0; i < 10; i++ {
txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)}))
}
return txs
}
// sign a commit vote
func signCommit(chainID string, privVal *types.PrivValidator, height, round int, hash []byte, header types.PartSetHeader) *types.Vote {
vote := &types.Vote{
ValidatorIndex: 0,
ValidatorAddress: privVal.Address,
Height: height,
Round: round,
Type: types.VoteTypePrecommit,
BlockID: types.BlockID{hash, header},
}
sig := privVal.Sign(types.SignBytes(chainID, vote))
vote.Signature = sig
return vote
}
// make a blockchain with one validator
func makeBlockchain(t *testing.T, chainID string, nBlocks int, privVal *types.PrivValidator, proxyApp proxy.AppConns, state *sm.State) (blockchain []*types.Block, commits []*types.Commit) {
prevHash := state.LastBlockID.Hash
lastCommit := new(types.Commit)
prevParts := types.PartSetHeader{}
valHash := state.Validators.Hash()
prevBlockID := types.BlockID{prevHash, prevParts}
for i := 1; i < nBlocks+1; i++ {
block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit,
prevBlockID, valHash, state.AppHash, testPartSize)
fmt.Println(i)
fmt.Println(block.LastBlockID)
err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
if err != nil {
t.Fatal(i, err)
}
voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators)
vote := signCommit(chainID, privVal, i, 0, block.Hash(), parts.Header())
_, err = voteSet.AddVote(vote)
if err != nil {
t.Fatal(err)
}
prevHash = block.Hash()
prevParts = parts.Header()
lastCommit = voteSet.MakeCommit()
prevBlockID = types.BlockID{prevHash, prevParts}
blockchain = append(blockchain, block)
commits = append(commits, lastCommit)
}
return blockchain, commits
}
// fresh state and mock store
func stateAndStore(config cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) {
stateDB := dbm.NewMemDB()
return sm.MakeGenesisState(stateDB, &types.GenesisDoc{
ChainID: config.GetString("chain_id"),
Validators: []types.GenesisValidator{
types.GenesisValidator{pubKey, 10000, "test"},
},
AppHash: nil,
}), NewMockBlockStore(config)
}
//----------------------------------
// mock block store
type mockBlockStore struct {
config cfg.Config
chain []*types.Block
commits []*types.Commit
}
// TODO: NewBlockStore(db.NewMemDB) ...
func NewMockBlockStore(config cfg.Config) *mockBlockStore {
return &mockBlockStore{config, nil, nil}
}
func (bs *mockBlockStore) Height() int { return len(bs.chain) }
func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
block := bs.chain[height-1]
return &types.BlockMeta{
BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.GetInt("block_part_size")).Header()},
Header: block.Header,
}
}
func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
}
func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
return bs.commits[height-1]
}
func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
return bs.commits[height-1]
} }

11
consensus/state.go
View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"path"
"reflect" "reflect"
"sync" "sync"
"time" "time"
@ -354,13 +355,13 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit {
func (cs *ConsensusState) OnStart() error { func (cs *ConsensusState) OnStart() error {
cs.BaseService.OnStart() cs.BaseService.OnStart()
walDir := cs.config.GetString("cs_wal_dir") walFile := cs.config.GetString("cs_wal_file")
err := EnsureDir(walDir, 0700) err := EnsureDir(path.Dir(walFile), 0700)
if err != nil { if err != nil {
log.Error("Error ensuring ConsensusState wal dir", "error", err.Error()) log.Error("Error ensuring ConsensusState wal dir", "error", err.Error())
return err return err
} }
err = cs.OpenWAL(walDir) err = cs.OpenWAL(walFile)
if err != nil { if err != nil {
log.Error("Error loading ConsensusState wal", "error", err.Error()) log.Error("Error loading ConsensusState wal", "error", err.Error())
return err return err
@ -415,10 +416,10 @@ func (cs *ConsensusState) Wait() {
} }
// Open file to log all consensus messages and timeouts for deterministic accountability // Open file to log all consensus messages and timeouts for deterministic accountability
func (cs *ConsensusState) OpenWAL(walDir string) (err error) { func (cs *ConsensusState) OpenWAL(walFile string) (err error) {
cs.mtx.Lock() cs.mtx.Lock()
defer cs.mtx.Unlock() defer cs.mtx.Unlock()
wal, err := NewWAL(walDir, cs.config.GetBool("cs_wal_light")) wal, err := NewWAL(walFile, cs.config.GetBool("cs_wal_light"))
if err != nil { if err != nil {
return err return err
} }

53
consensus/test_data/build.sh
View File

@ -1,12 +1,13 @@
#! /bin/bash #! /bin/bash
# XXX: removes tendermint dir
cd $GOPATH/src/github.com/tendermint/tendermint cd $GOPATH/src/github.com/tendermint/tendermint
# specify a dir to copy # specify a dir to copy
# NOTE: eventually we should replace with `tendermint init --test` # TODO: eventually we should replace with `tendermint init --test`
DIR=$HOME/.tendermint_test/consensus_state_test DIR=$HOME/.tendermint_test/consensus_state_test
# XXX: remove tendermint dir
rm -rf $HOME/.tendermint rm -rf $HOME/.tendermint
cp -r $DIR $HOME/.tendermint cp -r $DIR $HOME/.tendermint
@ -18,6 +19,7 @@ function reset(){
reset reset
# empty block # empty block
function empty_block(){
tendermint node --proxy_app=dummy &> /dev/null & tendermint node --proxy_app=dummy &> /dev/null &
sleep 5 sleep 5
killall tendermint killall tendermint
@ -28,21 +30,40 @@ killall tendermint
sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/empty_block.cswal sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/empty_block.cswal
reset reset
}
# small block 1 # many blocks
function many_blocks(){
bash scripts/txs/random.sh 1000 36657 &> /dev/null & bash scripts/txs/random.sh 1000 36657 &> /dev/null &
PID=$! PID=$!
tendermint node --proxy_app=dummy &> /dev/null & tendermint node --proxy_app=dummy &> /dev/null &
sleep 5 sleep 7
killall tendermint
kill -9 $PID
sed '/HEIGHT: 7/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/many_blocks.cswal
reset
}
# small block 1
function small_block1(){
bash scripts/txs/random.sh 1000 36657 &> /dev/null &
PID=$!
tendermint node --proxy_app=dummy &> /dev/null &
sleep 10
killall tendermint killall tendermint
kill -9 $PID kill -9 $PID
sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block1.cswal sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block1.cswal
reset reset
}
# small block 2 (part size = 512) # small block 2 (part size = 512)
function small_block2(){
echo "" >> ~/.tendermint/config.toml echo "" >> ~/.tendermint/config.toml
echo "block_part_size = 512" >> ~/.tendermint/config.toml echo "block_part_size = 512" >> ~/.tendermint/config.toml
bash scripts/txs/random.sh 1000 36657 &> /dev/null & bash scripts/txs/random.sh 1000 36657 &> /dev/null &
@ -55,4 +76,28 @@ kill -9 $PID
sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block2.cswal sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block2.cswal
reset reset
}
case "$1" in
"small_block1")
small_block1
;;
"small_block2")
small_block2
;;
"empty_block")
empty_block
;;
"many_blocks")
many_blocks
;;
*)
small_block1
small_block2
empty_block
many_blocks
esac

65
consensus/test_data/many_blocks.cswal Normal file
View File

File diff suppressed because one or more lines are too long

4
consensus/wal.go
View File

@ -40,8 +40,8 @@ type WAL struct {
light bool // ignore block parts light bool // ignore block parts
} }
func NewWAL(walDir string, light bool) (*WAL, error) { func NewWAL(walFile string, light bool) (*WAL, error) {
group, err := auto.OpenGroup(walDir + "/wal") group, err := auto.OpenGroup(walFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }

14
state/execution.go
View File

@ -1,6 +1,7 @@
package state package state
import ( import (
"bytes"
"errors" "errors"
"github.com/ebuchman/fail-test" "github.com/ebuchman/fail-test"
@ -278,7 +279,7 @@ func applyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]
var eventCache types.Fireable // nil var eventCache types.Fireable // nil
_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
if err != nil { if err != nil {
log.Warn("Error executing block on proxy app", "height", i, "err", err) log.Warn("Error executing block on proxy app", "height", block.Height, "err", err)
return nil, err return nil, err
} }
// Commit block, get hash back // Commit block, get hash back
@ -388,13 +389,17 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
// if the app is ahead, there's nothing we can do // if the app is ahead, there's nothing we can do
return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight} return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight {
// all good!
return nil
} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight+1 { } else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight+1 {
// We already ran Commit, but didn't save the state, so run through consensus with mock app // We already ran Commit, but didn't save the state, so run through consensus with mock app
mockApp := newMockProxyApp(appHash) mockApp := newMockProxyApp(appHash)
log.Info("Replay last block using mock app") log.Info("Replay last block using mock app")
h.replayLastBlock(h.config, h.state, mockApp, h.store) h.replayLastBlock(h.config, h.state, mockApp, h.store)
} else if storeBlockHeight == appBlockHeight+1 { } else if storeBlockHeight == appBlockHeight+1 && storeBlockHeight == stateBlockHeight+1 {
// We crashed after saving the block // We crashed after saving the block
// but before Commit (both the state and app are behind), // but before Commit (both the state and app are behind),
// so run through consensus with the real app // so run through consensus with the real app
@ -409,7 +414,8 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
var appHash []byte var appHash []byte
var err error var err error
for i := appBlockHeight + 1; i <= storeBlockHeight-1; i++ { for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
log.Info("Applying block", "height", i)
h.nBlocks += 1 h.nBlocks += 1
block := h.store.LoadBlock(i) block := h.store.LoadBlock(i)
appHash, err = applyBlock(proxyApp.Consensus(), block) appHash, err = applyBlock(proxyApp.Consensus(), block)
@ -418,7 +424,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
} }
} }
h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store) // h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
if !bytes.Equal(h.state.AppHash, appHash) { if !bytes.Equal(h.state.AppHash, appHash) {
return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)) return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
} }