Merge pull request #618 from tendermint/historical_validators

Historical validators

consensus/replay.go

@@ -324,8 +324,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) {
 	// App is further behind than it should be, so we need to replay blocks.
 	// We replay all blocks from appBlockHeight+1.
+	//
 	// Note that we don't have an old version of the state,
 	// so we by-pass state validation/mutation using sm.ExecCommitBlock.
+	// This also means we won't be saving validator sets if they change during this period.
+	//
 	// If mutateState == true, the final block is replayed with h.replayBlock()
 
 	var appHash []byte

rpc/client/httpclient.go

@@ -143,7 +143,7 @@ func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) {
 	return result, nil
 }
 
-func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) {
+func (c *HTTP) Block(height *int) (*ctypes.ResultBlock, error) {
 	result := new(ctypes.ResultBlock)
 	_, err := c.rpc.Call("block", map[string]interface{}{"height": height}, result)
 	if err != nil {
@@ -152,7 +152,7 @@ func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) {
 	return result, nil
 }
 
-func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) {
+func (c *HTTP) Commit(height *int) (*ctypes.ResultCommit, error) {
 	result := new(ctypes.ResultCommit)
 	_, err := c.rpc.Call("commit", map[string]interface{}{"height": height}, result)
 	if err != nil {
@@ -174,9 +174,9 @@ func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
 	return result, nil
 }
 
-func (c *HTTP) Validators() (*ctypes.ResultValidators, error) {
+func (c *HTTP) Validators(height *int) (*ctypes.ResultValidators, error) {
 	result := new(ctypes.ResultValidators)
-	_, err := c.rpc.Call("validators", map[string]interface{}{}, result)
+	_, err := c.rpc.Call("validators", map[string]interface{}{"height": height}, result)
 	if err != nil {
 		return nil, errors.Wrap(err, "Validators")
 	}

rpc/client/interface.go

@@ -42,9 +42,9 @@ type ABCIClient interface {
 // SignClient groups together the interfaces need to get valid
 // signatures and prove anything about the chain
 type SignClient interface {
-	Block(height int) (*ctypes.ResultBlock, error)
+	Block(height *int) (*ctypes.ResultBlock, error)
-	Commit(height int) (*ctypes.ResultCommit, error)
+	Commit(height *int) (*ctypes.ResultCommit, error)
-	Validators() (*ctypes.ResultValidators, error)
+	Validators(height *int) (*ctypes.ResultValidators, error)
 	Tx(hash []byte, prove bool) (*ctypes.ResultTx, error)
 }
 

rpc/client/localclient.go

@@ -93,16 +93,16 @@ func (c Local) Genesis() (*ctypes.ResultGenesis, error) {
 	return core.Genesis()
 }
 
-func (c Local) Block(height int) (*ctypes.ResultBlock, error) {
+func (c Local) Block(height *int) (*ctypes.ResultBlock, error) {
 	return core.Block(height)
 }
 
-func (c Local) Commit(height int) (*ctypes.ResultCommit, error) {
+func (c Local) Commit(height *int) (*ctypes.ResultCommit, error) {
 	return core.Commit(height)
 }
 
-func (c Local) Validators() (*ctypes.ResultValidators, error) {
+func (c Local) Validators(height *int) (*ctypes.ResultValidators, error) {
-	return core.Validators()
+	return core.Validators(height)
 }
 
 func (c Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {

rpc/client/mock/client.go

@@ -116,14 +116,14 @@ func (c Client) Genesis() (*ctypes.ResultGenesis, error) {
 	return core.Genesis()
 }
 
-func (c Client) Block(height int) (*ctypes.ResultBlock, error) {
+func (c Client) Block(height *int) (*ctypes.ResultBlock, error) {
 	return core.Block(height)
 }
 
-func (c Client) Commit(height int) (*ctypes.ResultCommit, error) {
+func (c Client) Commit(height *int) (*ctypes.ResultCommit, error) {
 	return core.Commit(height)
 }
 
-func (c Client) Validators() (*ctypes.ResultValidators, error) {
+func (c Client) Validators(height *int) (*ctypes.ResultValidators, error) {
-	return core.Validators()
+	return core.Validators(height)
 }

rpc/client/rpc_test.go

@@ -87,7 +87,7 @@ func TestGenesisAndValidators(t *testing.T) {
 		gval := gen.Genesis.Validators[0]
 
 		// get the current validators
-		vals, err := c.Validators()
+		vals, err := c.Validators(nil)
 		require.Nil(t, err, "%d: %+v", i, err)
 		require.Equal(t, 1, len(vals.Validators))
 		val := vals.Validators[0]
@@ -110,7 +110,8 @@ func TestAppCalls(t *testing.T) {
 		sh := s.LatestBlockHeight
 
 		// look for the future
-		_, err = c.Block(sh + 2)
+		h := sh + 2
+		_, err = c.Block(&h)
 		assert.NotNil(err) // no block yet
 
 		// write something
@@ -137,7 +138,7 @@ func TestAppCalls(t *testing.T) {
 		assert.EqualValues(tx, ptx.Tx)
 
 		// and we can even check the block is added
-		block, err := c.Block(apph)
+		block, err := c.Block(&apph)
 		require.Nil(err, "%d: %+v", i, err)
 		appHash := block.BlockMeta.Header.AppHash
 		assert.True(len(appHash) > 0)
@@ -158,14 +159,15 @@ func TestAppCalls(t *testing.T) {
 		}
 
 		// and get the corresponding commit with the same apphash
-		commit, err := c.Commit(apph)
+		commit, err := c.Commit(&apph)
 		require.Nil(err, "%d: %+v", i, err)
 		cappHash := commit.Header.AppHash
 		assert.Equal(appHash, cappHash)
 		assert.NotNil(commit.Commit)
 
 		// compare the commits (note Commit(2) has commit from Block(3))
-		commit2, err := c.Commit(apph - 1)
+		h = apph - 1
+		commit2, err := c.Commit(&h)
 		require.Nil(err, "%d: %+v", i, err)
 		assert.Equal(block.Block.LastCommit, commit2.Commit)
 

rpc/core/blocks.go

@@ -85,6 +85,7 @@ func BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, err
 }
 
 // Get block at a given height.
+// If no height is provided, it will fetch the latest block.
 //
 // ```shell
 // curl 'localhost:46657/block?height=10'
@@ -183,8 +184,16 @@ func BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, err
 //   "jsonrpc": "2.0"
 // }
 // ```
-func Block(height int) (*ctypes.ResultBlock, error) {
+func Block(heightPtr *int) (*ctypes.ResultBlock, error) {
-	if height == 0 {
+	if heightPtr == nil {
+		height := blockStore.Height()
+		blockMeta := blockStore.LoadBlockMeta(height)
+		block := blockStore.LoadBlock(height)
+		return &ctypes.ResultBlock{blockMeta, block}, nil
+	}
+
+	height := *heightPtr
+	if height <= 0 {
 		return nil, fmt.Errorf("Height must be greater than 0")
 	}
 	if height > blockStore.Height() {
@@ -197,6 +206,7 @@ func Block(height int) (*ctypes.ResultBlock, error) {
 }
 
 // Get block commit at a given height.
+// If no height is provided, it will fetch the commit for the latest block.
 //
 // ```shell
 // curl 'localhost:46657/commit?height=11'
@@ -265,8 +275,16 @@ func Block(height int) (*ctypes.ResultBlock, error) {
 //   "jsonrpc": "2.0"
 // }
 // ```
-func Commit(height int) (*ctypes.ResultCommit, error) {
+func Commit(heightPtr *int) (*ctypes.ResultCommit, error) {
-	if height == 0 {
+	if heightPtr == nil {
+		height := blockStore.Height()
+		header := blockStore.LoadBlockMeta(height).Header
+		commit := blockStore.LoadSeenCommit(height)
+		return &ctypes.ResultCommit{header, commit, false}, nil
+	}
+
+	height := *heightPtr
+	if height <= 0 {
 		return nil, fmt.Errorf("Height must be greater than 0")
 	}
 	storeHeight := blockStore.Height()

rpc/core/consensus.go

@@ -7,7 +7,8 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-// Get current validators set along with a block height.
+// Get the validator set at the given block height.
+// If no height is provided, it will fetch the current validator set.
 //
 // ```shell
 // curl 'localhost:46657/validators'
@@ -41,9 +42,19 @@ import (
 // 	"jsonrpc": "2.0"
 // }
 // ```
-func Validators() (*ctypes.ResultValidators, error) {
+func Validators(heightPtr *int) (*ctypes.ResultValidators, error) {
-	blockHeight, validators := consensusState.GetValidators()
+	if heightPtr == nil {
-	return &ctypes.ResultValidators{blockHeight, validators}, nil
+		blockHeight, validators := consensusState.GetValidators()
+		return &ctypes.ResultValidators{blockHeight, validators}, nil
+	}
+
+	height := *heightPtr
+	state := consensusState.GetState()
+	validators, err := state.LoadValidators(height)
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultValidators{height, validators.Validators}, nil
 }
 
 // Dump consensus state.

rpc/core/pipe.go

@@ -2,18 +2,21 @@ package core
 
 import (
 	crypto "github.com/tendermint/go-crypto"
+	"github.com/tendermint/tmlibs/log"
+
 	"github.com/tendermint/tendermint/consensus"
 	p2p "github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/state/txindex"
 	"github.com/tendermint/tendermint/types"
-	"github.com/tendermint/tmlibs/log"
 )
 
 //----------------------------------------------
 // These interfaces are used by RPC and must be thread safe
 
 type Consensus interface {
+	GetState() *sm.State
 	GetValidators() (int, []*types.Validator)
 	GetRoundState() *consensus.RoundState
 }

rpc/core/routes.go

@@ -18,7 +18,7 @@ var Routes = map[string]*rpc.RPCFunc{
 	"block":                rpc.NewRPCFunc(Block, "height"),
 	"commit":               rpc.NewRPCFunc(Commit, "height"),
 	"tx":                   rpc.NewRPCFunc(Tx, "hash,prove"),
-	"validators":           rpc.NewRPCFunc(Validators, ""),
+	"validators":           rpc.NewRPCFunc(Validators, "height"),
 	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""),
 	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxs, ""),
 	"num_unconfirmed_txs":  rpc.NewRPCFunc(NumUnconfirmedTxs, ""),

state/errors.go

@@ -33,6 +33,10 @@ type (
 		Got      *State
 		Expected *State
 	}
+
+	ErrNoValSetForHeight struct {
+		Height int
+	}
 )
 
 func (e ErrUnknownBlock) Error() string {
@@ -53,3 +57,7 @@ func (e ErrLastStateMismatch) Error() string {
 func (e ErrStateMismatch) Error() string {
 	return cmn.Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.Got, e.Expected)
 }
+
+func (e ErrNoValSetForHeight) Error() string {
+	return cmn.Fmt("Could not find validator set for height #%d", e.Height)
+}

state/execution.go

@@ -239,7 +239,7 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn
 
 	fail.Fail() // XXX
 
-	// save the state
+	// save the state and the validators
 	s.Save()
 
 	return nil

state/state.go

@@ -2,6 +2,7 @@ package state
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"sync"
 	"time"
@@ -22,6 +23,10 @@ var (
 	abciResponsesKey = []byte("abciResponsesKey")
 )
 
+func calcValidatorsKey(height int) []byte {
+	return []byte(cmn.Fmt("validatorsKey:%v", height))
+}
+
 //-----------------------------------------------------------------------------
 
 // State represents the latest committed state of the Tendermint consensus,
@@ -49,6 +54,12 @@ type State struct {
 
 	TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer.
 
+	// When a block returns a validator set change via EndBlock,
+	// the change only applies to the next block.
+	// So, if s.LastBlockHeight causes a valset change,
+	// we set s.LastHeightValidatorsChanged = s.LastBlockHeight + 1
+	LastHeightValidatorsChanged int
+
 	logger log.Logger
 }
 
@@ -71,6 +82,7 @@ func loadState(db dbm.DB, key []byte) *State {
 		}
 		// TODO: ensure that buf is completely read.
 	}
+
 	return s
 }
 
@@ -82,17 +94,18 @@ func (s *State) SetLogger(l log.Logger) {
 // Copy makes a copy of the State for mutating.
 func (s *State) Copy() *State {
 	return &State{
-		db:              s.db,
+		db:                          s.db,
-		GenesisDoc:      s.GenesisDoc,
+		GenesisDoc:                  s.GenesisDoc,
-		ChainID:         s.ChainID,
+		ChainID:                     s.ChainID,
-		LastBlockHeight: s.LastBlockHeight,
+		LastBlockHeight:             s.LastBlockHeight,
-		LastBlockID:     s.LastBlockID,
+		LastBlockID:                 s.LastBlockID,
-		LastBlockTime:   s.LastBlockTime,
+		LastBlockTime:               s.LastBlockTime,
-		Validators:      s.Validators.Copy(),
+		Validators:                  s.Validators.Copy(),
-		LastValidators:  s.LastValidators.Copy(),
+		LastValidators:              s.LastValidators.Copy(),
-		AppHash:         s.AppHash,
+		AppHash:                     s.AppHash,
-		TxIndexer:       s.TxIndexer, // pointer here, not value
+		TxIndexer:                   s.TxIndexer, // pointer here, not value
-		logger:          s.logger,
+		LastHeightValidatorsChanged: s.LastHeightValidatorsChanged,
+		logger: s.logger,
 	}
 }
 
@@ -100,6 +113,7 @@ func (s *State) Copy() *State {
 func (s *State) Save() {
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
+	s.saveValidatorsInfo()
 	s.db.SetSync(stateKey, s.Bytes())
 }
 
@@ -126,6 +140,57 @@ func (s *State) LoadABCIResponses() *ABCIResponses {
 	return abciResponses
 }
 
+// LoadValidators loads the ValidatorSet for a given height.
+func (s *State) LoadValidators(height int) (*types.ValidatorSet, error) {
+	v := s.loadValidators(height)
+	if v == nil {
+		return nil, ErrNoValSetForHeight{height}
+	}
+
+	if v.ValidatorSet == nil {
+		v = s.loadValidators(v.LastHeightChanged)
+		if v == nil {
+			cmn.PanicSanity(fmt.Sprintf(`Couldn't find validators at 
+			height %d as last changed from height %d`, v.LastHeightChanged, height))
+		}
+	}
+
+	return v.ValidatorSet, nil
+}
+
+func (s *State) loadValidators(height int) *ValidatorsInfo {
+	buf := s.db.Get(calcValidatorsKey(height))
+	if len(buf) == 0 {
+		return nil
+	}
+
+	v := new(ValidatorsInfo)
+	r, n, err := bytes.NewReader(buf), new(int), new(error)
+	wire.ReadBinaryPtr(v, r, 0, n, err)
+	if *err != nil {
+		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
+		cmn.Exit(cmn.Fmt("LoadValidators: Data has been corrupted or its spec has changed: %v\n", *err))
+	}
+	// TODO: ensure that buf is completely read.
+	return v
+}
+
+// saveValidatorsInfo persists the validator set for the next block to disk.
+// It should be called from s.Save(), right before the state itself is persisted.
+// If the validator set did not change after processing the latest block,
+// only the last height for which the validators changed is persisted.
+func (s *State) saveValidatorsInfo() {
+	changeHeight := s.LastHeightValidatorsChanged
+	nextHeight := s.LastBlockHeight + 1
+	vi := &ValidatorsInfo{
+		LastHeightChanged: changeHeight,
+	}
+	if changeHeight == nextHeight {
+		vi.ValidatorSet = s.Validators
+	}
+	s.db.SetSync(calcValidatorsKey(nextHeight), vi.Bytes())
+}
+
 // Equals returns true if the States are identical.
 func (s *State) Equals(s2 *State) bool {
 	return bytes.Equal(s.Bytes(), s2.Bytes())
@@ -144,10 +209,15 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ
 	prevValSet := s.Validators.Copy()
 	nextValSet := prevValSet.Copy()
 
-	err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs)
+	// update the validator set with the latest abciResponses
-	if err != nil {
+	if len(abciResponses.EndBlock.Diffs) > 0 {
-		s.logger.Error("Error changing validator set", "err", err)
+		err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs)
-		// TODO: err or carry on?
+		if err != nil {
+			s.logger.Error("Error changing validator set", "err", err)
+			// TODO: err or carry on?
+		}
+		// change results from this height but only applies to the next height
+		s.LastHeightValidatorsChanged = header.Height + 1
 	}
 
 	// Update validator accums and set state variables
@@ -215,6 +285,19 @@ func (a *ABCIResponses) Bytes() []byte {
 	return wire.BinaryBytes(*a)
 }
 
+//-----------------------------------------------------------------------------
+
+// ValidatorsInfo represents the latest validator set, or the last time it changed
+type ValidatorsInfo struct {
+	ValidatorSet      *types.ValidatorSet
+	LastHeightChanged int
+}
+
+// Bytes serializes the ValidatorsInfo using go-wire
+func (vi *ValidatorsInfo) Bytes() []byte {
+	return wire.BinaryBytes(*vi)
+}
+
 //-----------------------------------------------------------------------------
 // Genesis
 
@@ -260,15 +343,16 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State {
 	}
 
 	return &State{
-		db:              db,
+		db:                          db,
-		GenesisDoc:      genDoc,
+		GenesisDoc:                  genDoc,
-		ChainID:         genDoc.ChainID,
+		ChainID:                     genDoc.ChainID,
-		LastBlockHeight: 0,
+		LastBlockHeight:             0,
-		LastBlockID:     types.BlockID{},
+		LastBlockID:                 types.BlockID{},
-		LastBlockTime:   genDoc.GenesisTime,
+		LastBlockTime:               genDoc.GenesisTime,
-		Validators:      types.NewValidatorSet(validators),
+		Validators:                  types.NewValidatorSet(validators),
-		LastValidators:  types.NewValidatorSet(nil),
+		LastValidators:              types.NewValidatorSet(nil),
-		AppHash:         genDoc.AppHash,
+		AppHash:                     genDoc.AppHash,
-		TxIndexer:       &null.TxIndex{}, // we do not need indexer during replay and in tests
+		TxIndexer:                   &null.TxIndex{}, // we do not need indexer during replay and in tests
+		LastHeightValidatorsChanged: 1,
 	}
 }

state/state_test.go

@@ -1,18 +1,24 @@
 package state
 
 import (
+	"bytes"
 	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
 	abci "github.com/tendermint/abci/types"
 	crypto "github.com/tendermint/go-crypto"
-	cfg "github.com/tendermint/tendermint/config"
+	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/types"
 )
 
 func TestStateCopyEquals(t *testing.T) {
+	assert := assert.New(t)
 	config := cfg.ResetTestRoot("state_")
 
 	// Get State db
@@ -22,18 +28,13 @@ func TestStateCopyEquals(t *testing.T) {
 
 	stateCopy := state.Copy()
 
-	if !state.Equals(stateCopy) {
+	assert.True(state.Equals(stateCopy), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state))
-		t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)
-	}
-
 	stateCopy.LastBlockHeight += 1
-
+	assert.False(state.Equals(stateCopy), cmn.Fmt("expected states to be different. got same %v", state))
-	if state.Equals(stateCopy) {
-		t.Fatal("expected states to be different. got same %v", state)
-	}
 }
 
 func TestStateSaveLoad(t *testing.T) {
+	assert := assert.New(t)
 	config := cfg.ResetTestRoot("state_")
 	// Get State db
 	stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir())
@@ -44,9 +45,7 @@ func TestStateSaveLoad(t *testing.T) {
 	state.Save()
 
 	loadedState := LoadState(stateDB)
-	if !state.Equals(loadedState) {
+	assert.True(state.Equals(loadedState), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state))
-		t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state)
-	}
 }
 
 func TestABCIResponsesSaveLoad(t *testing.T) {
@@ -74,5 +73,127 @@ func TestABCIResponsesSaveLoad(t *testing.T) {
 
 	state.SaveABCIResponses(abciResponses)
 	abciResponses2 := state.LoadABCIResponses()
-	assert.Equal(abciResponses, abciResponses2, fmt.Sprintf("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses))
+	assert.Equal(abciResponses, abciResponses2, cmn.Fmt("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses))
+}
+
+func TestValidatorSimpleSaveLoad(t *testing.T) {
+	assert := assert.New(t)
+	config := cfg.ResetTestRoot("state_")
+	// Get State db
+	stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir())
+	state := GetState(stateDB, config.GenesisFile())
+	state.SetLogger(log.TestingLogger())
+
+	// cant load anything for height 0
+	v, err := state.LoadValidators(0)
+	assert.IsType(ErrNoValSetForHeight{}, err, "expected err at height 0")
+
+	// should be able to load for height 1
+	v, err = state.LoadValidators(1)
+	assert.Nil(err, "expected no err at height 1")
+	assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match")
+
+	// increment height, save; should be able to load for next height
+	state.LastBlockHeight += 1
+	state.saveValidatorsInfo()
+	v, err = state.LoadValidators(state.LastBlockHeight + 1)
+	assert.Nil(err, "expected no err")
+	assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match")
+
+	// increment height, save; should be able to load for next height
+	state.LastBlockHeight += 10
+	state.saveValidatorsInfo()
+	v, err = state.LoadValidators(state.LastBlockHeight + 1)
+	assert.Nil(err, "expected no err")
+	assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match")
+
+	// should be able to load for next next height
+	_, err = state.LoadValidators(state.LastBlockHeight + 2)
+	assert.IsType(ErrNoValSetForHeight{}, err, "expected err at unknown height")
+}
+
+func TestValidatorChangesSaveLoad(t *testing.T) {
+	assert := assert.New(t)
+	config := cfg.ResetTestRoot("state_")
+	// Get State db
+	stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir())
+	state := GetState(stateDB, config.GenesisFile())
+	state.SetLogger(log.TestingLogger())
+
+	// change vals at these heights
+	changeHeights := []int{1, 2, 4, 5, 10, 15, 16, 17, 20}
+	N := len(changeHeights)
+
+	// each valset is just one validator.
+	// create list of them
+	pubkeys := make([]crypto.PubKey, N+1)
+	pubkeys[0] = state.GenesisDoc.Validators[0].PubKey
+	for i := 1; i < N+1; i++ {
+		pubkeys[i] = crypto.GenPrivKeyEd25519().PubKey()
+	}
+
+	// build the validator history by running SetBlockAndValidators
+	// with the right validator set for each height
+	highestHeight := changeHeights[N-1] + 5
+	changeIndex := 0
+	pubkey := pubkeys[changeIndex]
+	for i := 1; i < highestHeight; i++ {
+		// when we get to a change height,
+		// use the next pubkey
+		if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] {
+			changeIndex += 1
+			pubkey = pubkeys[changeIndex]
+		}
+		header, parts, responses := makeHeaderPartsResponses(state, i, pubkey)
+		state.SetBlockAndValidators(header, parts, responses)
+		state.saveValidatorsInfo()
+	}
+
+	// make all the test cases by using the same validator until after the change
+	testCases := make([]valChangeTestCase, highestHeight)
+	changeIndex = 0
+	pubkey = pubkeys[changeIndex]
+	for i := 1; i < highestHeight+1; i++ {
+		// we we get to the height after a change height
+		// use the next pubkey (note our counter starts at 0 this time)
+		if changeIndex < len(changeHeights) && i == changeHeights[changeIndex]+1 {
+			changeIndex += 1
+			pubkey = pubkeys[changeIndex]
+		}
+		testCases[i-1] = valChangeTestCase{i, pubkey}
+	}
+
+	for _, testCase := range testCases {
+		v, err := state.LoadValidators(testCase.height)
+		assert.Nil(err, fmt.Sprintf("expected no err at height %d", testCase.height))
+		assert.Equal(v.Size(), 1, "validator set size is greater than 1: %d", v.Size())
+		addr, _ := v.GetByIndex(0)
+
+		assert.Equal(addr, testCase.vals.Address(), fmt.Sprintf("unexpected pubkey at height %d", testCase.height))
+	}
+}
+
+func makeHeaderPartsResponses(state *State, height int, pubkey crypto.PubKey) (*types.Header, types.PartSetHeader, *ABCIResponses) {
+	block := makeBlock(height, state)
+	_, val := state.Validators.GetByIndex(0)
+	abciResponses := &ABCIResponses{
+		Height: height,
+	}
+
+	// if the pubkey is new, remove the old and add the new
+	if !bytes.Equal(pubkey.Bytes(), val.PubKey.Bytes()) {
+		abciResponses.EndBlock = abci.ResponseEndBlock{
+			Diffs: []*abci.Validator{
+				{val.PubKey.Bytes(), 0},
+				{pubkey.Bytes(), 10},
+			},
+		}
+	}
+
+	return block.Header, types.PartSetHeader{}, abciResponses
+}
+
+type valChangeTestCase struct {
+	height int
+	vals   crypto.PubKey
 }