diff --git a/CHANGELOG.md b/CHANGELOG.md index e9e6ecf7..eb532e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.19.9 + +*June 5th, 2018* + +BREAKING CHANGES + +- [types/priv_validator] Moved to top level `privval` package + +FEATURES + +- [config] Collapse PeerConfig into P2PConfig +- [docs] Add quick-install script +- [docs/spec] Add table of Amino prefixes + +BUG FIXES + +- [rpc] Return 404 for unknown endpoints +- [consensus] Flush WAL on stop +- [evidence] Don't send evidence to peers that are behind +- [p2p] Fix memory leak on peer disconnects +- [rpc] Fix panic when `per_page=0` + ## 0.19.8 *June 4th, 2018* @@ -34,7 +56,7 @@ FEATURES IMPROVEMENTS: -- [consensus] consensus reactor now receives events from a separate event bus, +- [consensus] Consensus reactor now receives events from a separate synchronous event bus, which is not dependant on external RPC load - [consensus/wal] do not look for height in older files if we've seen height - 1 - [docs] Various cleanup and link fixes @@ -47,6 +69,12 @@ BUG FIXES - [blockchain] Fix fast-sync deadlock during high peer turnover +BUG FIX: + +- [evidence] Dont send peers evidence from heights they haven't synced to yet +- [p2p] Refuse connections to more than one peer with the same IP +- [docs] Various fixes + ## 0.19.5 *May 20th, 2018* diff --git a/Gopkg.lock b/Gopkg.lock index 6e34258f..fff269b6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -5,7 +5,7 @@ branch = "master" name = "github.com/btcsuite/btcd" packages = ["btcec"] - revision = "675abc5df3c5531bc741b56a765e35623459da6d" + revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64" [[projects]] name = "github.com/davecgh/go-spew" @@ -82,7 +82,7 @@ branch = "master" name = "github.com/golang/snappy" packages = ["."] - revision = "553a641470496b2327abcac10b36396bd98e45c9" + revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" [[projects]] name = "github.com/gorilla/websocket" @@ -128,20 +128,20 @@ [[projects]] name = "github.com/magiconair/properties" packages = ["."] - revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" - version = "v1.7.6" + revision = "c2353362d570a7bfa228149c62842019201cfb71" + version = "v1.8.0" [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "00c29f56e2386353d58c599509e8dc3801b0d716" + revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" [[projects]] name = "github.com/pelletier/go-toml" packages = ["."] - revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" - version = "v1.1.0" + revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" + version = "v1.2.0" [[projects]] name = "github.com/pkg/errors" @@ -159,7 +159,7 @@ branch = "master" name = "github.com/rcrowley/go-metrics" packages = ["."] - revision = "d932a24a8ccb8fcadc993e5c6c58f93dac168294" + revision = "e2704e165165ec55d062f5919b4b29494e9fa790" [[projects]] name = "github.com/spf13/afero" @@ -179,8 +179,8 @@ [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" - version = "v0.0.2" + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" [[projects]] branch = "master" @@ -226,7 +226,7 @@ "leveldb/table", "leveldb/util" ] - revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad" + revision = "5d6fca44a948d2be89a9702de7717f0168403d3d" [[projects]] name = "github.com/tendermint/abci" @@ -266,8 +266,8 @@ [[projects]] name = "github.com/tendermint/go-wire" packages = ["."] - revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c" - version = "v0.7.3" + revision = "3c22a7a539411f89a96738fcfa14c1027e24e5ec" + version = "0.9.10" [[projects]] name = "github.com/tendermint/tmlibs" @@ -283,8 +283,8 @@ "merkle", "test" ] - revision = "cc5f287c4798ffe88c04d02df219ecb6932080fd" - version = "v0.8.3-rc0" + revision = "692f1d86a6e2c0efa698fd1e4541b68c74ffaf38" + version = "v0.8.4" [[projects]] branch = "master" @@ -299,7 +299,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "b0697eccbea9adec5b7ba8008f4c33d98d733388" + revision = "df8d4716b3472e4a531c33cedbe537dae921a1a9" [[projects]] branch = "master" @@ -311,16 +311,15 @@ "http2/hpack", "idna", "internal/timeseries", - "lex/httplex", "trace" ] - revision = "5f9ae10d9af5b1c89ae6904293b14b064d4ada23" + revision = "1e491301e022f8f977054da4c2d852decd59571f" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "bb9c189858d91f42db229b04d45a4c3d23a7662a" + revision = "c11f84a56e43e20a78cee75a7c034031ecf57d1f" [[projects]] name = "golang.org/x/text" @@ -344,7 +343,6 @@ version = "v0.3.0" [[projects]] - branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" @@ -382,6 +380,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d85c98dcac32cc1fe05d006aa75e8985f6447a150a041b972a673a65e7681da9" + inputs-digest = "bdcf814c0cd3b8d6cc11ad03da556abe169f872a45e6dcbd8b08588b4587ddde" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 74197977..0c34ea03 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -79,16 +79,21 @@ [[constraint]] name = "github.com/tendermint/go-amino" - version = "0.9.9" + version = "=0.9.9" [[override]] name = "github.com/tendermint/tmlibs" - version = "~0.8.3-rc0" + version = "~0.8.4" [[constraint]] name = "google.golang.org/grpc" version = "~1.7.3" +# this got updated and broke, so locked to an old working commit ... +[[override]] + name = "google.golang.org/genproto" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + [prune] go-tests = true unused-packages = true diff --git a/blockchain/reactor_test.go b/blockchain/reactor_test.go index 49913c10..c7f7e9af 100644 --- a/blockchain/reactor_test.go +++ b/blockchain/reactor_test.go @@ -36,7 +36,7 @@ func newBlockchainReactor(logger log.Logger, maxBlockHeight int64) *BlockchainRe fastSync := true var nilApp proxy.AppConnConsensus blockExec := sm.NewBlockExecutor(dbm.NewMemDB(), log.TestingLogger(), nilApp, - types.MockMempool{}, types.MockEvidencePool{}) + sm.MockMempool{}, sm.MockEvidencePool{}) bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync) bcReactor.SetLogger(logger.With("module", "blockchain")) diff --git a/cmd/priv_val_server/main.go b/cmd/priv_val_server/main.go index 6b10b817..98e7dcee 100644 --- a/cmd/priv_val_server/main.go +++ b/cmd/priv_val_server/main.go @@ -8,7 +8,7 @@ import ( cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" - priv_val "github.com/tendermint/tendermint/types/priv_validator" + "github.com/tendermint/tendermint/privval" ) func main() { @@ -30,13 +30,13 @@ func main() { "privPath", *privValPath, ) - privVal := priv_val.LoadFilePV(*privValPath) + pv := privval.LoadFilePV(*privValPath) - rs := priv_val.NewRemoteSigner( + rs := privval.NewRemoteSigner( logger, *chainID, *addr, - privVal, + pv, crypto.GenPrivKeyEd25519(), ) err := rs.Start() diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go index ff0d97d7..20d43d4d 100644 --- a/cmd/tendermint/commands/gen_validator.go +++ b/cmd/tendermint/commands/gen_validator.go @@ -5,7 +5,7 @@ import ( "github.com/spf13/cobra" - pvm "github.com/tendermint/tendermint/types/priv_validator" + "github.com/tendermint/tendermint/privval" ) // GenValidatorCmd allows the generation of a keypair for a @@ -17,7 +17,7 @@ var GenValidatorCmd = &cobra.Command{ } func genValidator(cmd *cobra.Command, args []string) { - pv := pvm.GenFilePV("") + pv := privval.GenFilePV("") jsbz, err := cdc.MarshalJSON(pv) if err != nil { panic(err) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 430a6c7c..45812b9e 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -7,8 +7,8 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/types" - pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" ) @@ -26,12 +26,12 @@ func initFiles(cmd *cobra.Command, args []string) error { func initFilesWithConfig(config *cfg.Config) error { // private validator privValFile := config.PrivValidatorFile() - var pv *pvm.FilePV + var pv *privval.FilePV if cmn.FileExists(privValFile) { - pv = pvm.LoadFilePV(privValFile) + pv = privval.LoadFilePV(privValFile) logger.Info("Found private validator", "path", privValFile) } else { - pv = pvm.GenFilePV(privValFile) + pv = privval.GenFilePV(privValFile) pv.Save() logger.Info("Generated private validator", "path", privValFile) } diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 78db87de..cf4b7c57 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -5,7 +5,7 @@ import ( "github.com/spf13/cobra" - pvm "github.com/tendermint/tendermint/types/priv_validator" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tmlibs/log" ) @@ -50,11 +50,11 @@ func resetPrivValidator(cmd *cobra.Command, args []string) { func resetFilePV(privValFile string, logger log.Logger) { // Get PrivValidator if _, err := os.Stat(privValFile); err == nil { - pv := pvm.LoadFilePV(privValFile) + pv := privval.LoadFilePV(privValFile) pv.Reset() logger.Info("Reset PrivValidator", "file", privValFile) } else { - pv := pvm.GenFilePV(privValFile) + pv := privval.GenFilePV(privValFile) pv.Save() logger.Info("Generated PrivValidator", "file", privValFile) } diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index b354683b..54765164 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -5,7 +5,7 @@ import ( "github.com/spf13/cobra" - privval "github.com/tendermint/tendermint/types/priv_validator" + "github.com/tendermint/tendermint/privval" ) // ShowValidatorCmd adds capabilities for showing the validator info. diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 3b29e20a..660bab15 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -12,8 +12,8 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/types" - pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" ) @@ -89,7 +89,7 @@ func testnetFiles(cmd *cobra.Command, args []string) error { initFilesWithConfig(config) pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator) - pv := pvm.LoadFilePV(pvFile) + pv := privval.LoadFilePV(pvFile) genVals[i] = types.GenesisValidator{ PubKey: pv.GetPubKey(), Power: 1, diff --git a/config/config.go b/config/config.go index 57655480..a5a212f5 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,15 @@ import ( "os" "path/filepath" "time" + + tmconn "github.com/tendermint/tendermint/p2p/conn" +) + +const ( + // FuzzModeDrop is a mode in which we randomly drop reads/writes, connections or sleep + FuzzModeDrop = iota + // FuzzModeDelay is a mode in which we randomly sleep + FuzzModeDelay ) // NOTE: Most of the structs & relevant comments + the @@ -287,11 +296,24 @@ type P2PConfig struct { // Does not work if the peer-exchange reactor is disabled. SeedMode bool `mapstructure:"seed_mode"` - // Comma separated list of peer IDs to keep private (will not be gossiped to other peers) + // Comma separated list of peer IDs to keep private (will not be gossiped to + // other peers) PrivatePeerIDs string `mapstructure:"private_peer_ids"` // Toggle to disable guard against peers connecting from the same ip. AllowDuplicateIP bool `mapstructure:"allow_duplicate_ip"` + + // Peer connection configuration. + HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"` + DialTimeout time.Duration `mapstructure:"dial_timeout"` + MConfig tmconn.MConnConfig `mapstructure:"connection"` + + // Testing params. + // Force dial to fail + TestDialFail bool `mapstructure:"test_dial_fail"` + // FUzz connection + TestFuzz bool `mapstructure:"test_fuzz"` + TestFuzzConfig *FuzzConnConfig `mapstructure:"test_fuzz_config"` } // DefaultP2PConfig returns a default configuration for the peer-to-peer layer @@ -308,6 +330,12 @@ func DefaultP2PConfig() *P2PConfig { PexReactor: true, SeedMode: false, AllowDuplicateIP: true, // so non-breaking yet + HandshakeTimeout: 20 * time.Second, + DialTimeout: 3 * time.Second, + MConfig: tmconn.DefaultMConnConfig(), + TestDialFail: false, + TestFuzz: false, + TestFuzzConfig: DefaultFuzzConnConfig(), } } @@ -326,6 +354,26 @@ func (cfg *P2PConfig) AddrBookFile() string { return rootify(cfg.AddrBook, cfg.RootDir) } +// FuzzConnConfig is a FuzzedConnection configuration. +type FuzzConnConfig struct { + Mode int + MaxDelay time.Duration + ProbDropRW float64 + ProbDropConn float64 + ProbSleep float64 +} + +// DefaultFuzzConnConfig returns the default config. +func DefaultFuzzConnConfig() *FuzzConnConfig { + return &FuzzConnConfig{ + Mode: FuzzModeDrop, + MaxDelay: 3 * time.Second, + ProbDropRW: 0.2, + ProbDropConn: 0.00, + ProbSleep: 0.00, + } +} + //----------------------------------------------------------------------------- // MempoolConfig diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index f18f1623..a8f559f6 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -105,15 +105,18 @@ func TestByzantine(t *testing.T) { p2p.Connect2Switches(sws, i, j) }) - // start the state machines - byzR := reactors[0].(*ByzantineReactor) - s := byzR.reactor.conS.GetState() - byzR.reactor.SwitchToConsensus(s, 0) + // start the non-byz state machines. + // note these must be started before the byz for i := 1; i < N; i++ { cr := reactors[i].(*ConsensusReactor) cr.SwitchToConsensus(cr.conS.GetState(), 0) } + // start the byzantine state machine + byzR := reactors[0].(*ByzantineReactor) + s := byzR.reactor.conS.GetState() + byzR.reactor.SwitchToConsensus(s, 0) + // byz proposer sends one block to peers[0] // and the other block to peers[1] and peers[2]. // note peers and switches order don't match. diff --git a/consensus/common_test.go b/consensus/common_test.go index 3eaeea70..f50e5769 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -19,9 +19,9 @@ import ( cstypes "github.com/tendermint/tendermint/consensus/types" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - pvm "github.com/tendermint/tendermint/types/priv_validator" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -262,7 +262,7 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S } // mock the evidence pool - evpool := types.MockEvidencePool{} + evpool := sm.MockEvidencePool{} // Make ConsensusState stateDB := dbm.NewMemDB() @@ -278,10 +278,10 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S return cs } -func loadPrivValidator(config *cfg.Config) *pvm.FilePV { +func loadPrivValidator(config *cfg.Config) *privval.FilePV { privValidatorFile := config.PrivValidatorFile() ensureDir(path.Dir(privValidatorFile), 0700) - privValidator := pvm.LoadOrGenFilePV(privValidatorFile) + privValidator := privval.LoadOrGenFilePV(privValidatorFile) privValidator.Reset() return privValidator } @@ -379,7 +379,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF privVal = privVals[i] } else { _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal = pvm.GenFilePV(tempFilePath) + privVal = privval.GenFilePV(tempFilePath) } app := appFunc() diff --git a/consensus/replay.go b/consensus/replay.go index 265ab538..13d665f7 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -196,7 +196,7 @@ func makeHeightSearchFunc(height int64) auto.SearchFunc { type Handshaker struct { stateDB dbm.DB initialState sm.State - store types.BlockStore + store sm.BlockStore appState json.RawMessage logger log.Logger @@ -204,7 +204,7 @@ type Handshaker struct { } func NewHandshaker(stateDB dbm.DB, state sm.State, - store types.BlockStore, appState json.RawMessage) *Handshaker { + store sm.BlockStore, appState json.RawMessage) *Handshaker { return &Handshaker{ stateDB: stateDB, @@ -390,7 +390,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap block := h.store.LoadBlock(height) meta := h.store.LoadBlockMeta(height) - blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, types.MockMempool{}, types.MockEvidencePool{}) + blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, sm.MockMempool{}, sm.MockEvidencePool{}) var err error state, err = blockExec.ApplyBlock(state, meta.BlockID, block) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 58f02227..4f834346 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -310,7 +310,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo cmn.Exit(cmn.Fmt("Failed to start event bus: %v", err)) } - mempool, evpool := types.MockMempool{}, types.MockEvidencePool{} + mempool, evpool := sm.MockMempool{}, sm.MockEvidencePool{} blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) consensusState := NewConsensusState(csConfig, state.Copy(), blockExec, diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 84b1e118..e0f8a4b9 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -23,10 +23,10 @@ import ( dbm "github.com/tendermint/tmlibs/db" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - pvm "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tmlibs/log" ) @@ -263,8 +263,8 @@ const ( ) var ( - mempool = types.MockMempool{} - evpool = types.MockEvidencePool{} + mempool = sm.MockMempool{} + evpool = sm.MockEvidencePool{} ) //--------------------------------------- @@ -329,7 +329,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { walFile := tempWALWithData(walBody) config.Consensus.SetWalFile(walFile) - privVal := pvm.LoadFilePV(config.PrivValidatorFile()) + privVal := privval.LoadFilePV(config.PrivValidatorFile()) wal, err := NewWAL(walFile) if err != nil { diff --git a/consensus/state.go b/consensus/state.go index 3b713e2e..d46ec583 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -76,9 +76,9 @@ type ConsensusState struct { // services for creating and executing blocks // TODO: encapsulate all of this in one "BlockManager" blockExec *sm.BlockExecutor - blockStore types.BlockStore - mempool types.Mempool - evpool types.EvidencePool + blockStore sm.BlockStore + mempool sm.Mempool + evpool sm.EvidencePool // internal state mtx sync.Mutex @@ -118,7 +118,7 @@ type ConsensusState struct { } // NewConsensusState returns a new ConsensusState. -func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *sm.BlockExecutor, blockStore types.BlockStore, mempool types.Mempool, evpool types.EvidencePool) *ConsensusState { +func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *sm.BlockExecutor, blockStore sm.BlockStore, mempool sm.Mempool, evpool sm.EvidencePool) *ConsensusState { cs := &ConsensusState{ config: config, blockExec: blockExec, diff --git a/consensus/wal.go b/consensus/wal.go index 80cb8fc3..3d9bf8af 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -106,8 +106,8 @@ func (wal *baseWAL) OnStart() error { } func (wal *baseWAL) OnStop() { - wal.BaseService.OnStop() wal.group.Stop() + wal.group.Close() } // Write is called in newStep and for each receive on the diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index dc364df0..38bed4ac 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -13,10 +13,10 @@ import ( "github.com/tendermint/abci/example/kvstore" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - pvm "github.com/tendermint/tendermint/types/priv_validator" auto "github.com/tendermint/tmlibs/autofile" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/db" @@ -40,7 +40,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { // COPY PASTE FROM node.go WITH A FEW MODIFICATIONS // NOTE: we can't import node package because of circular dependency privValidatorFile := config.PrivValidatorFile() - privValidator := pvm.LoadOrGenFilePV(privValidatorFile) + privValidator := privval.LoadOrGenFilePV(privValidatorFile) genDoc, err := types.GenesisDocFromFile(config.GenesisFile()) if err != nil { return nil, errors.Wrap(err, "failed to read genesis file") @@ -65,8 +65,8 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { return nil, errors.Wrap(err, "failed to start event bus") } defer eventBus.Stop() - mempool := types.MockMempool{} - evpool := types.MockEvidencePool{} + mempool := sm.MockMempool{} + evpool := sm.MockEvidencePool{} blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) consensusState := NewConsensusState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetLogger(logger) diff --git a/docs/install.rst b/docs/install.rst index 3fc392a3..63a4bb65 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,6 +1,9 @@ Install Tendermint ================== +The fastest and easiest way to install the ``tendermint`` binary +is to run `this script `__ on a fresh Ubuntu instance. Read the comments / instructions carefully (i.e., reset your terminal after running the script). + From Binary ----------- diff --git a/evidence/pool.go b/evidence/pool.go index 07c35134..4bad355f 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + clist "github.com/tendermint/tmlibs/clist" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -17,6 +18,7 @@ type EvidencePool struct { logger log.Logger evidenceStore *EvidenceStore + evidenceList *clist.CList // concurrent linked-list of evidence // needed to load validators to verify evidence stateDB dbm.DB @@ -24,9 +26,6 @@ type EvidencePool struct { // latest state mtx sync.Mutex state sm.State - - // never close - evidenceChan chan types.Evidence } func NewEvidencePool(stateDB dbm.DB, evidenceStore *EvidenceStore) *EvidencePool { @@ -35,21 +34,24 @@ func NewEvidencePool(stateDB dbm.DB, evidenceStore *EvidenceStore) *EvidencePool state: sm.LoadState(stateDB), logger: log.NewNopLogger(), evidenceStore: evidenceStore, - evidenceChan: make(chan types.Evidence), + evidenceList: clist.New(), } return evpool } +func (evpool *EvidencePool) EvidenceFront() *clist.CElement { + return evpool.evidenceList.Front() +} + +func (evpool *EvidencePool) EvidenceWaitChan() <-chan struct{} { + return evpool.evidenceList.WaitChan() +} + // SetLogger sets the Logger. func (evpool *EvidencePool) SetLogger(l log.Logger) { evpool.logger = l } -// EvidenceChan returns an unbuffered channel on which new evidence can be received. -func (evpool *EvidencePool) EvidenceChan() <-chan types.Evidence { - return evpool.evidenceChan -} - // PriorityEvidence returns the priority evidence. func (evpool *EvidencePool) PriorityEvidence() []types.Evidence { return evpool.evidenceStore.PriorityEvidence() @@ -68,22 +70,23 @@ func (evpool *EvidencePool) State() sm.State { } // Update loads the latest -func (evpool *EvidencePool) Update(block *types.Block) { - evpool.mtx.Lock() - defer evpool.mtx.Unlock() +func (evpool *EvidencePool) Update(block *types.Block, state sm.State) { - state := sm.LoadState(evpool.stateDB) + // sanity check if state.LastBlockHeight != block.Height { - panic(fmt.Sprintf("EvidencePool.Update: loaded state with height %d when block.Height=%d", state.LastBlockHeight, block.Height)) + panic(fmt.Sprintf("Failed EvidencePool.Update sanity check: got state.Height=%d with block.Height=%d", state.LastBlockHeight, block.Height)) } - evpool.state = state - // NOTE: shouldn't need the mutex - evpool.MarkEvidenceAsCommitted(block.Evidence.Evidence) + // update the state + evpool.mtx.Lock() + evpool.state = state + evpool.mtx.Unlock() + + // remove evidence from pending and mark committed + evpool.MarkEvidenceAsCommitted(block.Height, block.Evidence.Evidence) } // AddEvidence checks the evidence is valid and adds it to the pool. -// Blocks on the EvidenceChan. func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { // TODO: check if we already have evidence for this @@ -107,14 +110,43 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence) - // never closes. always safe to send on - evpool.evidenceChan <- evidence + // add evidence to clist + evpool.evidenceList.PushBack(evidence) + return nil } -// MarkEvidenceAsCommitted marks all the evidence as committed. -func (evpool *EvidencePool) MarkEvidenceAsCommitted(evidence []types.Evidence) { +// MarkEvidenceAsCommitted marks all the evidence as committed and removes it from the queue. +func (evpool *EvidencePool) MarkEvidenceAsCommitted(height int64, evidence []types.Evidence) { + // make a map of committed evidence to remove from the clist + blockEvidenceMap := make(map[string]struct{}) for _, ev := range evidence { evpool.evidenceStore.MarkEvidenceAsCommitted(ev) + blockEvidenceMap[evMapKey(ev)] = struct{}{} + } + + // remove committed evidence from the clist + maxAge := evpool.State().ConsensusParams.EvidenceParams.MaxAge + evpool.removeEvidence(height, maxAge, blockEvidenceMap) + +} + +func (evpool *EvidencePool) removeEvidence(height, maxAge int64, blockEvidenceMap map[string]struct{}) { + for e := evpool.evidenceList.Front(); e != nil; e = e.Next() { + ev := e.Value.(types.Evidence) + + // Remove the evidence if it's already in a block + // or if it's now too old. + if _, ok := blockEvidenceMap[evMapKey(ev)]; ok || + ev.Height() < height-maxAge { + + // remove from clist + evpool.evidenceList.Remove(e) + e.DetachPrev() + } } } + +func evMapKey(ev types.Evidence) string { + return string(ev.Hash()) +} diff --git a/evidence/pool_test.go b/evidence/pool_test.go index 97a29a27..01907623 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -45,7 +45,6 @@ func initializeValidatorState(valAddr []byte, height int64) dbm.DB { } func TestEvidencePool(t *testing.T) { - assert := assert.New(t) valAddr := []byte("val1") height := int64(5) @@ -56,26 +55,25 @@ func TestEvidencePool(t *testing.T) { goodEvidence := types.NewMockGoodEvidence(height, 0, valAddr) badEvidence := types.MockBadEvidence{goodEvidence} + // bad evidence err := pool.AddEvidence(badEvidence) - assert.NotNil(err) + assert.NotNil(t, err) var wg sync.WaitGroup wg.Add(1) go func() { - <-pool.EvidenceChan() + <-pool.EvidenceWaitChan() wg.Done() }() err = pool.AddEvidence(goodEvidence) - assert.Nil(err) + assert.Nil(t, err) wg.Wait() - // if we send it again it wont fire on the chan + assert.Equal(t, 1, pool.evidenceList.Len()) + + // if we send it again, it shouldnt change the size err = pool.AddEvidence(goodEvidence) - assert.Nil(err) - select { - case <-pool.EvidenceChan(): - t.Fatal("unexpected read on EvidenceChan") - default: - } + assert.Nil(t, err) + assert.Equal(t, 1, pool.evidenceList.Len()) } diff --git a/evidence/reactor.go b/evidence/reactor.go index a6aa66b1..5159572e 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -6,6 +6,7 @@ import ( "time" "github.com/tendermint/go-amino" + clist "github.com/tendermint/tmlibs/clist" "github.com/tendermint/tmlibs/log" "github.com/tendermint/tendermint/p2p" @@ -15,8 +16,10 @@ import ( const ( EvidenceChannel = byte(0x38) - maxMsgSize = 1048576 // 1MB TODO make it configurable - broadcastEvidenceIntervalS = 60 // broadcast uncommitted evidence this often + maxMsgSize = 1048576 // 1MB TODO make it configurable + + broadcastEvidenceIntervalS = 60 // broadcast uncommitted evidence this often + peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount ) // EvidenceReactor handles evpool evidence broadcasting amongst peers. @@ -43,11 +46,7 @@ func (evR *EvidenceReactor) SetLogger(l log.Logger) { // OnStart implements cmn.Service func (evR *EvidenceReactor) OnStart() error { - if err := evR.BaseReactor.OnStart(); err != nil { - return err - } - go evR.broadcastRoutine() - return nil + return evR.BaseReactor.OnStart() } // GetChannels implements Reactor. @@ -63,14 +62,7 @@ func (evR *EvidenceReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor. func (evR *EvidenceReactor) AddPeer(peer p2p.Peer) { - // send the peer our high-priority evidence. - // the rest will be sent by the broadcastRoutine - evidences := evR.evpool.PriorityEvidence() - msg := &EvidenceListMessage{evidences} - success := peer.Send(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) - if !success { - // TODO: remove peer ? - } + go evR.broadcastEvidenceRoutine(peer) } // RemovePeer implements Reactor. @@ -109,30 +101,97 @@ func (evR *EvidenceReactor) SetEventBus(b *types.EventBus) { evR.eventBus = b } -// Broadcast new evidence to all peers. -// Broadcasts must be non-blocking so routine is always available to read off EvidenceChan. -func (evR *EvidenceReactor) broadcastRoutine() { - ticker := time.NewTicker(time.Second * broadcastEvidenceIntervalS) +// Modeled after the mempool routine. +// - Evidence accumulates in a clist. +// - Each peer has a routien that iterates through the clist, +// sending available evidence to the peer. +// - If we're waiting for new evidence and the list is not empty, +// start iterating from the beginning again. +func (evR *EvidenceReactor) broadcastEvidenceRoutine(peer p2p.Peer) { + var next *clist.CElement for { - select { - case evidence := <-evR.evpool.EvidenceChan(): - // broadcast some new evidence - msg := &EvidenceListMessage{[]types.Evidence{evidence}} - evR.Switch.Broadcast(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) + // This happens because the CElement we were looking at got garbage + // collected (removed). That is, .NextWait() returned nil. Go ahead and + // start from the beginning. + if next == nil { + select { + case <-evR.evpool.EvidenceWaitChan(): // Wait until evidence is available + if next = evR.evpool.EvidenceFront(); next == nil { + continue + } + case <-peer.Quit(): + return + case <-evR.Quit(): + return + } + } - // TODO: Broadcast runs asynchronously, so this should wait on the successChan - // in another routine before marking to be proper. - evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(evidence) - case <-ticker.C: - // broadcast all pending evidence - msg := &EvidenceListMessage{evR.evpool.PendingEvidence()} - evR.Switch.Broadcast(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) + ev := next.Value.(types.Evidence) + msg, retry := evR.checkSendEvidenceMessage(peer, ev) + if msg != nil { + success := peer.Send(EvidenceChannel, cdc.MustMarshalBinaryBare(msg)) + retry = !success + } + + if retry { + time.Sleep(peerCatchupSleepIntervalMS * time.Millisecond) + continue + } + + afterCh := time.After(time.Second * broadcastEvidenceIntervalS) + select { + case <-afterCh: + // start from the beginning every tick. + // TODO: only do this if we're at the end of the list! + next = nil + case <-next.NextWaitChan(): + // see the start of the for loop for nil check + next = next.Next() + case <-peer.Quit(): + return case <-evR.Quit(): return } } } +// Returns the message to send the peer, or nil if the evidence is invalid for the peer. +// If message is nil, return true if we should sleep and try again. +func (evR EvidenceReactor) checkSendEvidenceMessage(peer p2p.Peer, ev types.Evidence) (msg EvidenceMessage, retry bool) { + + // make sure the peer is up to date + evHeight := ev.Height() + peerState, ok := peer.Get(types.PeerStateKey).(PeerState) + if !ok { + evR.Logger.Info("Found peer without PeerState", "peer", peer) + return nil, true + } + + // NOTE: We only send evidence to peers where + // peerHeight - maxAge < evidenceHeight < peerHeight + maxAge := evR.evpool.State().ConsensusParams.EvidenceParams.MaxAge + peerHeight := peerState.GetHeight() + if peerHeight < evHeight { + // peer is behind. sleep while he catches up + return nil, true + } else if peerHeight > evHeight+maxAge { + // evidence is too old, skip + // NOTE: if evidence is too old for an honest peer, + // then we're behind and either it already got committed or it never will! + evR.Logger.Info("Not sending peer old evidence", "peerHeight", peerHeight, "evHeight", evHeight, "maxAge", maxAge, "peer", peer) + return nil, false + } + + // send evidence + msg = &EvidenceListMessage{[]types.Evidence{ev}} + return msg, false +} + +// PeerState describes the state of a peer. +type PeerState interface { + GetHeight() int64 +} + //----------------------------------------------------------------------------- // Messages diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go index 11c63929..2f1c34e6 100644 --- a/evidence/reactor_test.go +++ b/evidence/reactor_test.go @@ -84,7 +84,7 @@ func _waitForEvidence(t *testing.T, wg *sync.WaitGroup, evs types.EvidenceList, } reapedEv := evpool.PendingEvidence() - // put the reaped evidence is a map so we can quickly check we got everything + // put the reaped evidence in a map so we can quickly check we got everything evMap := make(map[string]types.Evidence) for _, e := range reapedEv { evMap[string(e.Hash())] = e @@ -95,6 +95,7 @@ func _waitForEvidence(t *testing.T, wg *sync.WaitGroup, evs types.EvidenceList, fmt.Sprintf("evidence at index %d on reactor %d don't match: %v vs %v", i, reactorIdx, expectedEv, gotEv)) } + wg.Done() } @@ -110,7 +111,7 @@ func sendEvidence(t *testing.T, evpool *EvidencePool, valAddr []byte, n int) typ } var ( - NUM_EVIDENCE = 1 + NUM_EVIDENCE = 10 TIMEOUT = 120 * time.Second // ridiculously high because CircleCI is slow ) @@ -130,8 +131,52 @@ func TestReactorBroadcastEvidence(t *testing.T) { // make reactors from statedb reactors := makeAndConnectEvidenceReactors(config, stateDBs) + // set the peer height on each reactor + for _, r := range reactors { + for _, peer := range r.Switch.Peers().List() { + ps := peerState{height} + peer.Set(types.PeerStateKey, ps) + } + } + // send a bunch of valid evidence to the first reactor's evpool // and wait for them all to be received in the others evList := sendEvidence(t, reactors[0].evpool, valAddr, NUM_EVIDENCE) waitForEvidence(t, evList, reactors) } + +type peerState struct { + height int64 +} + +func (ps peerState) GetHeight() int64 { + return ps.height +} + +func TestReactorSelectiveBroadcast(t *testing.T) { + config := cfg.TestConfig() + + valAddr := []byte("myval") + height1 := int64(NUM_EVIDENCE) + 10 + height2 := int64(NUM_EVIDENCE) / 2 + + // DB1 is ahead of DB2 + stateDB1 := initializeValidatorState(valAddr, height1) + stateDB2 := initializeValidatorState(valAddr, height2) + + // make reactors from statedb + reactors := makeAndConnectEvidenceReactors(config, []dbm.DB{stateDB1, stateDB2}) + peer := reactors[0].Switch.Peers().List()[0] + ps := peerState{height2} + peer.Set(types.PeerStateKey, ps) + + // send a bunch of valid evidence to the first reactor's evpool + evList := sendEvidence(t, reactors[0].evpool, valAddr, NUM_EVIDENCE) + + // only ones less than the peers height should make it through + waitForEvidence(t, evList[:NUM_EVIDENCE/2], reactors[1:2]) + + // peers should still be connected + peers := reactors[1].Switch.Peers().List() + assert.Equal(t, 1, len(peers)) +} diff --git a/evidence/store.go b/evidence/store.go index 081715e3..6af5d75d 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -17,10 +17,6 @@ Impl: - First commit atomically in outqueue, pending, lookup. - Once broadcast, remove from outqueue. No need to sync - Once committed, atomically remove from pending and update lookup. - - TODO: If we crash after committed but before removing/updating, - we'll be stuck broadcasting evidence we never know we committed. - so either share the state db and atomically MarkCommitted - with ApplyBlock, or check all outqueue/pending on Start to see if its committed Schema for indexing evidence (note you need both height and hash to find a piece of evidence): @@ -164,7 +160,7 @@ func (store *EvidenceStore) MarkEvidenceAsBroadcasted(evidence types.Evidence) { store.db.Delete(key) } -// MarkEvidenceAsPending removes evidence from pending and outqueue and sets the state to committed. +// MarkEvidenceAsCommitted removes evidence from pending and outqueue and sets the state to committed. func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { // if its committed, its been broadcast store.MarkEvidenceAsBroadcasted(evidence) diff --git a/node/node.go b/node/node.go index 5cae4a4b..5da57665 100644 --- a/node/node.go +++ b/node/node.go @@ -21,6 +21,7 @@ import ( mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/p2p/pex" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" rpccore "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" @@ -32,7 +33,6 @@ import ( "github.com/tendermint/tendermint/state/txindex/kv" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" - pvm "github.com/tendermint/tendermint/types/priv_validator" "github.com/tendermint/tendermint/version" _ "net/http/pprof" @@ -77,7 +77,7 @@ type NodeProvider func(*cfg.Config, log.Logger) (*Node, error) // It implements NodeProvider. func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { return NewNode(config, - pvm.LoadOrGenFilePV(config.PrivValidatorFile()), + privval.LoadOrGenFilePV(config.PrivValidatorFile()), proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), DefaultDBProvider, @@ -177,8 +177,8 @@ func NewNode(config *cfg.Config, // TODO: persist this key so external signer // can actually authenticate us privKey = crypto.GenPrivKeyEd25519() - pvsc = pvm.NewSocketPV( - logger.With("module", "pvm"), + pvsc = privval.NewSocketPV( + logger.With("module", "privval"), config.PrivValidatorListenAddr, privKey, ) @@ -447,7 +447,7 @@ func (n *Node) OnStop() { n.eventBus.Stop() n.indexerService.Stop() - if pvsc, ok := n.privValidator.(*pvm.SocketPV); ok { + if pvsc, ok := n.privValidator.(*privval.SocketPV); ok { if err := pvsc.Stop(); err != nil { n.Logger.Error("Error stopping priv validator socket client", "err", err) } diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 6e08c67f..5c7f19cf 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -83,7 +83,7 @@ type MConnection struct { onReceive receiveCbFunc onError errorCbFunc errored uint32 - config *MConnConfig + config MConnConfig quit chan struct{} flushTimer *cmn.ThrottleTimer // flush writes as necessary but throttled. @@ -121,8 +121,8 @@ func (cfg *MConnConfig) maxPacketMsgTotalSize() int { } // DefaultMConnConfig returns the default config. -func DefaultMConnConfig() *MConnConfig { - return &MConnConfig{ +func DefaultMConnConfig() MConnConfig { + return MConnConfig{ SendRate: defaultSendRate, RecvRate: defaultRecvRate, MaxPacketMsgPayloadSize: maxPacketMsgPayloadSizeDefault, @@ -143,7 +143,7 @@ func NewMConnection(conn net.Conn, chDescs []*ChannelDescriptor, onReceive recei } // NewMConnectionWithConfig wraps net.Conn and creates multiplex connection with a config -func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onReceive receiveCbFunc, onError errorCbFunc, config *MConnConfig) *MConnection { +func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onReceive receiveCbFunc, onError errorCbFunc, config MConnConfig) *MConnection { if config.PongTimeout >= config.PingInterval { panic("pongTimeout must be less than pingInterval (otherwise, next ping will reset pong timer)") } @@ -545,9 +545,7 @@ FOR_LOOP: // not goroutine-safe func (c *MConnection) stopPongTimer() { if c.pongTimer != nil { - if !c.pongTimer.Stop() { - <-c.pongTimer.C - } + _ = c.pongTimer.Stop() c.pongTimer = nil } } diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index a927d695..ade8e8e9 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -6,9 +6,11 @@ import ( "testing" "time" + "github.com/fortytw2/leaktest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/go-amino" + + amino "github.com/tendermint/go-amino" "github.com/tendermint/tmlibs/log" ) @@ -242,7 +244,11 @@ func TestMConnectionMultiplePings(t *testing.T) { } func TestMConnectionPingPongs(t *testing.T) { + // check that we are not leaking any go-routines + defer leaktest.CheckTimeout(t, 10*time.Second)() + server, client := net.Pipe() + defer server.Close() defer client.Close() diff --git a/p2p/fuzz.go b/p2p/fuzz.go index 6bfadc29..8d00ba40 100644 --- a/p2p/fuzz.go +++ b/p2p/fuzz.go @@ -5,16 +5,10 @@ import ( "sync" "time" + "github.com/tendermint/tendermint/config" cmn "github.com/tendermint/tmlibs/common" ) -const ( - // FuzzModeDrop is a mode in which we randomly drop reads/writes, connections or sleep - FuzzModeDrop = iota - // FuzzModeDelay is a mode in which we randomly sleep - FuzzModeDelay -) - // FuzzedConnection wraps any net.Conn and depending on the mode either delays // reads/writes or randomly drops reads/writes/connections. type FuzzedConnection struct { @@ -24,37 +18,17 @@ type FuzzedConnection struct { start <-chan time.Time active bool - config *FuzzConnConfig -} - -// FuzzConnConfig is a FuzzedConnection configuration. -type FuzzConnConfig struct { - Mode int - MaxDelay time.Duration - ProbDropRW float64 - ProbDropConn float64 - ProbSleep float64 -} - -// DefaultFuzzConnConfig returns the default config. -func DefaultFuzzConnConfig() *FuzzConnConfig { - return &FuzzConnConfig{ - Mode: FuzzModeDrop, - MaxDelay: 3 * time.Second, - ProbDropRW: 0.2, - ProbDropConn: 0.00, - ProbSleep: 0.00, - } + config *config.FuzzConnConfig } // FuzzConn creates a new FuzzedConnection. Fuzzing starts immediately. func FuzzConn(conn net.Conn) net.Conn { - return FuzzConnFromConfig(conn, DefaultFuzzConnConfig()) + return FuzzConnFromConfig(conn, config.DefaultFuzzConnConfig()) } // FuzzConnFromConfig creates a new FuzzedConnection from a config. Fuzzing // starts immediately. -func FuzzConnFromConfig(conn net.Conn, config *FuzzConnConfig) net.Conn { +func FuzzConnFromConfig(conn net.Conn, config *config.FuzzConnConfig) net.Conn { return &FuzzedConnection{ conn: conn, start: make(<-chan time.Time), @@ -66,12 +40,16 @@ func FuzzConnFromConfig(conn net.Conn, config *FuzzConnConfig) net.Conn { // FuzzConnAfter creates a new FuzzedConnection. Fuzzing starts when the // duration elapses. func FuzzConnAfter(conn net.Conn, d time.Duration) net.Conn { - return FuzzConnAfterFromConfig(conn, d, DefaultFuzzConnConfig()) + return FuzzConnAfterFromConfig(conn, d, config.DefaultFuzzConnConfig()) } // FuzzConnAfterFromConfig creates a new FuzzedConnection from a config. // Fuzzing starts when the duration elapses. -func FuzzConnAfterFromConfig(conn net.Conn, d time.Duration, config *FuzzConnConfig) net.Conn { +func FuzzConnAfterFromConfig( + conn net.Conn, + d time.Duration, + config *config.FuzzConnConfig, +) net.Conn { return &FuzzedConnection{ conn: conn, start: time.After(d), @@ -81,7 +59,7 @@ func FuzzConnAfterFromConfig(conn net.Conn, d time.Duration, config *FuzzConnCon } // Config returns the connection's config. -func (fc *FuzzedConnection) Config() *FuzzConnConfig { +func (fc *FuzzedConnection) Config() *config.FuzzConnConfig { return fc.config } @@ -136,7 +114,7 @@ func (fc *FuzzedConnection) fuzz() bool { } switch fc.config.Mode { - case FuzzModeDrop: + case config.FuzzModeDrop: // randomly drop the r/w, drop the conn, or sleep r := cmn.RandFloat64() if r <= fc.config.ProbDropRW { @@ -149,7 +127,7 @@ func (fc *FuzzedConnection) fuzz() bool { } else if r < fc.config.ProbDropRW+fc.config.ProbDropConn+fc.config.ProbSleep { time.Sleep(fc.randomDuration()) } - case FuzzModeDelay: + case config.FuzzModeDelay: // sleep a bit time.Sleep(fc.randomDuration()) } diff --git a/p2p/peer.go b/p2p/peer.go index 29f42465..73e2eac2 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -10,10 +10,11 @@ import ( cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/config" tmconn "github.com/tendermint/tendermint/p2p/conn" ) -var testIPSuffix uint32 = 0 +var testIPSuffix uint32 // Peer is an interface representing a peer connected on a reactor. type Peer interface { @@ -39,7 +40,7 @@ type Peer interface { type peerConn struct { outbound bool persistent bool - config *PeerConfig + config *config.P2PConfig conn net.Conn // source connection ip net.IP } @@ -99,94 +100,95 @@ type peer struct { Data *cmn.CMap } -func newPeer(pc peerConn, nodeInfo NodeInfo, - reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor, - onPeerError func(Peer, interface{})) *peer { - +func newPeer( + pc peerConn, + nodeInfo NodeInfo, + reactorsByCh map[byte]Reactor, + chDescs []*tmconn.ChannelDescriptor, + onPeerError func(Peer, interface{}), +) *peer { p := &peer{ peerConn: pc, nodeInfo: nodeInfo, channels: nodeInfo.Channels, Data: cmn.NewCMap(), } - p.mconn = createMConnection(pc.conn, p, reactorsByCh, chDescs, onPeerError, pc.config.MConfig) + + p.mconn = createMConnection( + pc.conn, + p, + reactorsByCh, + chDescs, + onPeerError, + pc.config.MConfig, + ) p.BaseService = *cmn.NewBaseService(nil, "Peer", p) + return p } -// PeerConfig is a Peer configuration. -type PeerConfig struct { - // times are in seconds - HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"` - DialTimeout time.Duration `mapstructure:"dial_timeout"` - - MConfig *tmconn.MConnConfig `mapstructure:"connection"` - - DialFail bool `mapstructure:"dial_fail"` // for testing - Fuzz bool `mapstructure:"fuzz"` // fuzz connection (for testing) - FuzzConfig *FuzzConnConfig `mapstructure:"fuzz_config"` -} - -// DefaultPeerConfig returns the default config. -func DefaultPeerConfig() *PeerConfig { - return &PeerConfig{ - HandshakeTimeout: 20, // * time.Second, - DialTimeout: 3, // * time.Second, - MConfig: tmconn.DefaultMConnConfig(), - DialFail: false, - Fuzz: false, - FuzzConfig: DefaultFuzzConnConfig(), - } -} - -func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool, ourNodePrivKey crypto.PrivKey) (peerConn, error) { - var pc peerConn - +func newOutboundPeerConn( + addr *NetAddress, + config *config.P2PConfig, + persistent bool, + ourNodePrivKey crypto.PrivKey, +) (peerConn, error) { conn, err := dial(addr, config) if err != nil { - return pc, cmn.ErrorWrap(err, "Error creating peer") + return peerConn{}, cmn.ErrorWrap(err, "Error creating peer") } - pc, err = newPeerConn(conn, config, true, persistent, ourNodePrivKey) + pc, err := newPeerConn(conn, config, true, persistent, ourNodePrivKey) if err != nil { - if err2 := conn.Close(); err2 != nil { - return pc, cmn.ErrorWrap(err, err2.Error()) + if cerr := conn.Close(); cerr != nil { + return peerConn{}, cmn.ErrorWrap(err, cerr.Error()) } - return pc, err + return peerConn{}, err } // ensure dialed ID matches connection ID if addr.ID != pc.ID() { - if err2 := conn.Close(); err2 != nil { - return pc, cmn.ErrorWrap(err, err2.Error()) + if cerr := conn.Close(); cerr != nil { + return peerConn{}, cmn.ErrorWrap(err, cerr.Error()) } - return pc, ErrSwitchAuthenticationFailure{addr, pc.ID()} + return peerConn{}, ErrSwitchAuthenticationFailure{addr, pc.ID()} } + return pc, nil } -func newInboundPeerConn(conn net.Conn, config *PeerConfig, ourNodePrivKey crypto.PrivKey) (peerConn, error) { +func newInboundPeerConn( + conn net.Conn, + config *config.P2PConfig, + ourNodePrivKey crypto.PrivKey, +) (peerConn, error) { // TODO: issue PoW challenge return newPeerConn(conn, config, false, false, ourNodePrivKey) } -func newPeerConn(rawConn net.Conn, - config *PeerConfig, outbound, persistent bool, - ourNodePrivKey crypto.PrivKey) (pc peerConn, err error) { - +func newPeerConn( + rawConn net.Conn, + cfg *config.P2PConfig, + outbound, persistent bool, + ourNodePrivKey crypto.PrivKey, +) (pc peerConn, err error) { conn := rawConn // Fuzz connection - if config.Fuzz { + if cfg.TestFuzz { // so we have time to do peer handshakes and get set up - conn = FuzzConnAfterFromConfig(conn, 10*time.Second, config.FuzzConfig) + conn = FuzzConnAfterFromConfig(conn, 10*time.Second, cfg.TestFuzzConfig) } // Set deadline for secret handshake - if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil { - return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection") + dl := time.Now().Add(cfg.HandshakeTimeout) + if err := conn.SetDeadline(dl); err != nil { + return pc, cmn.ErrorWrap( + err, + "Error setting deadline while encrypting connection", + ) } // Encrypt connection @@ -197,7 +199,7 @@ func newPeerConn(rawConn net.Conn, // Only the information we already have return peerConn{ - config: config, + config: cfg, outbound: outbound, persistent: persistent, conn: conn, @@ -300,22 +302,33 @@ func (p *peer) hasChannel(chID byte) bool { } // NOTE: probably will want to remove this // but could be helpful while the feature is new - p.Logger.Debug("Unknown channel for peer", "channel", chID, "channels", p.channels) + p.Logger.Debug( + "Unknown channel for peer", + "channel", + chID, + "channels", + p.channels, + ) return false } //--------------------------------------------------- // methods used by the Switch -// CloseConn should be called by the Switch if the peer was created but never started. +// CloseConn should be called by the Switch if the peer was created but never +// started. func (pc *peerConn) CloseConn() { pc.conn.Close() // nolint: errcheck } -// HandshakeTimeout performs the Tendermint P2P handshake between a given node and the peer -// by exchanging their NodeInfo. It sets the received nodeInfo on the peer. +// HandshakeTimeout performs the Tendermint P2P handshake between a given node +// and the peer by exchanging their NodeInfo. It sets the received nodeInfo on +// the peer. // NOTE: blocking -func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration) (peerNodeInfo NodeInfo, err error) { +func (pc *peerConn) HandshakeTimeout( + ourNodeInfo NodeInfo, + timeout time.Duration, +) (peerNodeInfo NodeInfo, err error) { // Set deadline for handshake so we don't block forever on conn.ReadFull if err := pc.conn.SetDeadline(time.Now().Add(timeout)); err != nil { return peerNodeInfo, cmn.ErrorWrap(err, "Error setting deadline") @@ -327,7 +340,11 @@ func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration return }, func(_ int) (val interface{}, err error, abort bool) { - _, err = cdc.UnmarshalBinaryReader(pc.conn, &peerNodeInfo, int64(MaxNodeInfoSize())) + _, err = cdc.UnmarshalBinaryReader( + pc.conn, + &peerNodeInfo, + int64(MaxNodeInfoSize()), + ) return }, ) @@ -368,20 +385,26 @@ func (p *peer) String() string { //------------------------------------------------------------------ // helper funcs -func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) { - if config.DialFail { +func dial(addr *NetAddress, cfg *config.P2PConfig) (net.Conn, error) { + if cfg.TestDialFail { return nil, fmt.Errorf("dial err (peerConfig.DialFail == true)") } - conn, err := addr.DialTimeout(config.DialTimeout * time.Second) + conn, err := addr.DialTimeout(cfg.DialTimeout) if err != nil { return nil, err } return conn, nil } -func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor, - onPeerError func(Peer, interface{}), config *tmconn.MConnConfig) *tmconn.MConnection { +func createMConnection( + conn net.Conn, + p *peer, + reactorsByCh map[byte]Reactor, + chDescs []*tmconn.ChannelDescriptor, + onPeerError func(Peer, interface{}), + config tmconn.MConnConfig, +) *tmconn.MConnection { onReceive := func(chID byte, msgBytes []byte) { reactor := reactorsByCh[chID] @@ -397,5 +420,11 @@ func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, ch onPeerError(p, r) } - return tmconn.NewMConnectionWithConfig(conn, chDescs, onReceive, onError, config) + return tmconn.NewMConnectionWithConfig( + conn, + chDescs, + onReceive, + onError, + config, + ) } diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 435c941f..d4781c65 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -10,9 +10,11 @@ import ( "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" - tmconn "github.com/tendermint/tendermint/p2p/conn" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/tendermint/config" + tmconn "github.com/tendermint/tendermint/p2p/conn" ) const testCh = 0x01 @@ -21,11 +23,11 @@ func TestPeerBasic(t *testing.T) { assert, require := assert.New(t), require.New(t) // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: cfg} rp.Start() defer rp.Stop() - p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), DefaultPeerConfig()) + p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), cfg) require.Nil(err) err = p.Start() @@ -44,7 +46,7 @@ func TestPeerBasic(t *testing.T) { func TestPeerSend(t *testing.T) { assert, require := assert.New(t), require.New(t) - config := DefaultPeerConfig() + config := cfg // simulate remote peer rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config} @@ -63,7 +65,7 @@ func TestPeerSend(t *testing.T) { assert.True(p.Send(testCh, []byte("Asylum"))) } -func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) (*peer, error) { +func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *config.P2PConfig) (*peer, error) { chDescs := []*tmconn.ChannelDescriptor{ {ID: testCh, Priority: 1}, } @@ -91,7 +93,7 @@ func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) type remotePeer struct { PrivKey crypto.PrivKey - Config *PeerConfig + Config *config.P2PConfig addr *NetAddress quit chan struct{} channels cmn.HexBytes diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index a5fc0337..645e9317 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -13,21 +13,22 @@ import ( "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" - cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/p2p/conn" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/p2p/conn" ) var ( - config *cfg.P2PConfig + cfg *config.P2PConfig ) func init() { - config = cfg.DefaultP2PConfig() - config.PexReactor = true - config.AllowDuplicateIP = true + cfg = config.DefaultP2PConfig() + cfg.PexReactor = true + cfg.AllowDuplicateIP = true } func TestPEXReactorBasic(t *testing.T) { @@ -53,6 +54,7 @@ func TestPEXReactorAddRemovePeer(t *testing.T) { outboundPeer := p2p.CreateRandomPeer(true) r.AddPeer(outboundPeer) + assert.Equal(t, size+1, book.Size(), "outbound peers should not be added to the address book") r.RemovePeer(outboundPeer, "peer not available") } @@ -81,7 +83,7 @@ func TestPEXReactorRunning(t *testing.T) { // create switches for i := 0; i < N; i++ { - switches[i] = p2p.MakeSwitch(config, i, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + switches[i] = p2p.MakeSwitch(cfg, i, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false) books[i].SetLogger(logger.With("pex", i)) sw.SetAddrBook(books[i]) @@ -209,7 +211,7 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { // 1. create seed seed := p2p.MakeSwitch( - config, + cfg, 0, "127.0.0.1", "123.123.123", @@ -239,7 +241,7 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { // 2. create usual peer with only seed configured. peer := p2p.MakeSwitch( - config, + cfg, 1, "127.0.0.1", "123.123.123", @@ -425,7 +427,7 @@ func assertPeersWithTimeout( } } -func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) { +func createReactor(conf *PEXReactorConfig) (r *PEXReactor, book *addrBook) { // directory to store address book dir, err := ioutil.TempDir("", "pex_reactor") if err != nil { @@ -434,7 +436,7 @@ func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) { book = NewAddrBook(filepath.Join(dir, "addrbook.json"), true) book.SetLogger(log.TestingLogger()) - r = NewPEXReactor(book, config) + r = NewPEXReactor(book, conf) r.SetLogger(log.TestingLogger()) return } @@ -447,7 +449,7 @@ func teardownReactor(book *addrBook) { } func createSwitchAndAddReactors(reactors ...p2p.Reactor) *p2p.Switch { - sw := p2p.MakeSwitch(config, 0, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { return sw }) + sw := p2p.MakeSwitch(cfg, 0, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { return sw }) sw.SetLogger(log.TestingLogger()) for _, r := range reactors { sw.AddReactor(r.String(), r) diff --git a/p2p/switch.go b/p2p/switch.go index 939af0bb..9068aa11 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -7,7 +7,7 @@ import ( "sync" "time" - cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" cmn "github.com/tendermint/tmlibs/common" ) @@ -55,8 +55,7 @@ type AddrBook interface { type Switch struct { cmn.BaseService - config *cfg.P2PConfig - peerConfig *PeerConfig + config *config.P2PConfig listeners []Listener reactors map[string]Reactor chDescs []*conn.ChannelDescriptor @@ -75,10 +74,9 @@ type Switch struct { } // NewSwitch creates a new Switch with the given config. -func NewSwitch(config *cfg.P2PConfig) *Switch { +func NewSwitch(cfg *config.P2PConfig) *Switch { sw := &Switch{ - config: config, - peerConfig: DefaultPeerConfig(), + config: cfg, reactors: make(map[string]Reactor), chDescs: make([]*conn.ChannelDescriptor, 0), reactorsByCh: make(map[byte]Reactor), @@ -90,11 +88,10 @@ func NewSwitch(config *cfg.P2PConfig) *Switch { // Ensure we have a completely undeterministic PRNG. sw.rng = cmn.NewRand() - // TODO: collapse the peerConfig into the config ? - sw.peerConfig.MConfig.FlushThrottle = time.Duration(config.FlushThrottleTimeout) * time.Millisecond - sw.peerConfig.MConfig.SendRate = config.SendRate - sw.peerConfig.MConfig.RecvRate = config.RecvRate - sw.peerConfig.MConfig.MaxPacketMsgPayloadSize = config.MaxPacketMsgPayloadSize + sw.config.MConfig.FlushThrottle = time.Duration(cfg.FlushThrottleTimeout) * time.Millisecond + sw.config.MConfig.SendRate = cfg.SendRate + sw.config.MConfig.RecvRate = cfg.RecvRate + sw.config.MConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw) return sw @@ -419,7 +416,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { sw.dialing.Set(string(addr.ID), addr) defer sw.dialing.Delete(string(addr.ID)) - return sw.addOutboundPeerWithConfig(addr, sw.peerConfig, persistent) + return sw.addOutboundPeerWithConfig(addr, sw.config, persistent) } // sleep for interval plus some random amount of ms on [0, dialRandomizerIntervalMilliseconds] @@ -476,7 +473,7 @@ func (sw *Switch) listenerRoutine(l Listener) { } // New inbound connection! - err := sw.addInboundPeerWithConfig(inConn, sw.peerConfig) + err := sw.addInboundPeerWithConfig(inConn, sw.config) if err != nil { sw.Logger.Info("Ignoring inbound connection: error while adding peer", "address", inConn.RemoteAddr().String(), "err", err) continue @@ -486,7 +483,10 @@ func (sw *Switch) listenerRoutine(l Listener) { // cleanup } -func (sw *Switch) addInboundPeerWithConfig(conn net.Conn, config *PeerConfig) error { +func (sw *Switch) addInboundPeerWithConfig( + conn net.Conn, + config *config.P2PConfig, +) error { peerConn, err := newInboundPeerConn(conn, config, sw.nodeKey.PrivKey) if err != nil { conn.Close() // peer is nil @@ -503,10 +503,20 @@ func (sw *Switch) addInboundPeerWithConfig(conn net.Conn, config *PeerConfig) er // dial the peer; make secret connection; authenticate against the dialed ID; // add the peer. // if dialing fails, start the reconnect loop. If handhsake fails, its over. -// If peer is started succesffuly, reconnectLoop will start when StopPeerForError is called -func (sw *Switch) addOutboundPeerWithConfig(addr *NetAddress, config *PeerConfig, persistent bool) error { +// If peer is started succesffuly, reconnectLoop will start when +// StopPeerForError is called +func (sw *Switch) addOutboundPeerWithConfig( + addr *NetAddress, + config *config.P2PConfig, + persistent bool, +) error { sw.Logger.Info("Dialing peer", "address", addr) - peerConn, err := newOutboundPeerConn(addr, config, persistent, sw.nodeKey.PrivKey) + peerConn, err := newOutboundPeerConn( + addr, + config, + persistent, + sw.nodeKey.PrivKey, + ) if err != nil { if persistent { go sw.reconnectToPeer(addr) @@ -525,7 +535,8 @@ func (sw *Switch) addOutboundPeerWithConfig(addr *NetAddress, config *PeerConfig // that already has a SecretConnection. If all goes well, // it starts the peer and adds it to the switch. // NOTE: This performs a blocking handshake before the peer is added. -// NOTE: If error is returned, caller is responsible for calling peer.CloseConn() +// NOTE: If error is returned, caller is responsible for calling +// peer.CloseConn() func (sw *Switch) addPeer(pc peerConn) error { addr := pc.conn.RemoteAddr() @@ -534,7 +545,7 @@ func (sw *Switch) addPeer(pc peerConn) error { } // Exchange NodeInfo on the conn - peerNodeInfo, err := pc.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.peerConfig.HandshakeTimeout*time.Second)) + peerNodeInfo, err := pc.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.HandshakeTimeout)) if err != nil { return err } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index d33797a2..6157f45c 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -14,18 +14,18 @@ import ( crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" ) var ( - config *cfg.P2PConfig + cfg *config.P2PConfig ) func init() { - config = cfg.DefaultP2PConfig() - config.PexReactor = true - config.AllowDuplicateIP = true + cfg = config.DefaultP2PConfig() + cfg.PexReactor = true + cfg.AllowDuplicateIP = true } type PeerMessage struct { @@ -85,7 +85,7 @@ func (tr *TestReactor) getMsgs(chID byte) []PeerMessage { // XXX: note this uses net.Pipe and not a proper TCP conn func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switch, *Switch) { // Create two switches that will be interconnected. - switches := MakeConnectedSwitches(config, 2, initSwitch, Connect2Switches) + switches := MakeConnectedSwitches(cfg, 2, initSwitch, Connect2Switches) return switches[0], switches[1] } @@ -152,8 +152,8 @@ func assertMsgReceivedWithTimeout(t *testing.T, msgBytes []byte, channel byte, r } func TestConnAddrFilter(t *testing.T) { - s1 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) - s2 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) + s1 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) + s2 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) defer s1.Stop() defer s2.Stop() @@ -181,14 +181,14 @@ func TestConnAddrFilter(t *testing.T) { } func TestSwitchFiltersOutItself(t *testing.T) { - s1 := MakeSwitch(config, 1, "127.0.0.1", "123.123.123", initSwitchFunc) + s1 := MakeSwitch(cfg, 1, "127.0.0.1", "123.123.123", initSwitchFunc) // addr := s1.NodeInfo().NetAddress() // // add ourselves like we do in node.go#427 // s1.addrBook.AddOurAddress(addr) // simulate s1 having a public IP by creating a remote peer with the same ID - rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: cfg} rp.Start() // addr should be rejected in addPeer based on the same ID @@ -214,8 +214,8 @@ func assertNoPeersAfterTimeout(t *testing.T, sw *Switch, timeout time.Duration) } func TestConnIDFilter(t *testing.T) { - s1 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) - s2 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) + s1 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) + s2 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) defer s1.Stop() defer s2.Stop() @@ -251,7 +251,7 @@ func TestConnIDFilter(t *testing.T) { func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { assert, require := assert.New(t), require.New(t) - sw := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) + sw := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) err := sw.Start() if err != nil { t.Error(err) @@ -259,11 +259,11 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { defer sw.Stop() // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: cfg} rp.Start() defer rp.Stop() - pc, err := newOutboundPeerConn(rp.Addr(), DefaultPeerConfig(), false, sw.nodeKey.PrivKey) + pc, err := newOutboundPeerConn(rp.Addr(), cfg, false, sw.nodeKey.PrivKey) require.Nil(err) err = sw.addPeer(pc) require.Nil(err) @@ -281,7 +281,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { func TestSwitchReconnectsToPersistentPeer(t *testing.T) { assert, require := assert.New(t), require.New(t) - sw := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) + sw := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) err := sw.Start() if err != nil { t.Error(err) @@ -289,11 +289,11 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) { defer sw.Stop() // simulate remote peer - rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()} + rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: cfg} rp.Start() defer rp.Stop() - pc, err := newOutboundPeerConn(rp.Addr(), DefaultPeerConfig(), true, sw.nodeKey.PrivKey) + pc, err := newOutboundPeerConn(rp.Addr(), cfg, true, sw.nodeKey.PrivKey) // sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodeKey.PrivKey, require.Nil(err) @@ -320,7 +320,7 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) { // simulate another remote peer rp = &remotePeer{ PrivKey: crypto.GenPrivKeyEd25519(), - Config: DefaultPeerConfig(), + Config: cfg, // Use different interface to prevent duplicate IP filter, this will break // beyond two peers. listenAddr: "127.0.0.1:0", @@ -329,9 +329,9 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) { defer rp.Stop() // simulate first time dial failure - peerConfig := DefaultPeerConfig() - peerConfig.DialFail = true - err = sw.addOutboundPeerWithConfig(rp.Addr(), peerConfig, true) + conf := config.DefaultP2PConfig() + conf.TestDialFail = true + err = sw.addOutboundPeerWithConfig(rp.Addr(), conf, true) require.NotNil(err) // DialPeerWithAddres - sw.peerConfig resets the dialer @@ -348,7 +348,7 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) { } func TestSwitchFullConnectivity(t *testing.T) { - switches := MakeConnectedSwitches(config, 3, initSwitchFunc, Connect2Switches) + switches := MakeConnectedSwitches(cfg, 3, initSwitchFunc, Connect2Switches) defer func() { for _, sw := range switches { sw.Stop() diff --git a/p2p/test_util.go b/p2p/test_util.go index b5b739af..bea815f5 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -8,7 +8,7 @@ import ( cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" - cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p/conn" ) @@ -56,7 +56,7 @@ const TEST_HOST = "localhost" // If connect==Connect2Switches, the switches will be fully connected. // initSwitch defines how the i'th switch should be initialized (ie. with what reactors). // NOTE: panics if any switch fails to start. -func MakeConnectedSwitches(cfg *cfg.P2PConfig, n int, initSwitch func(int, *Switch) *Switch, connect func([]*Switch, int, int)) []*Switch { +func MakeConnectedSwitches(cfg *config.P2PConfig, n int, initSwitch func(int, *Switch) *Switch, connect func([]*Switch, int, int)) []*Switch { switches := make([]*Switch, n) for i := 0; i < n; i++ { switches[i] = MakeSwitch(cfg, i, TEST_HOST, "123.123.123", initSwitch) @@ -104,7 +104,7 @@ func Connect2Switches(switches []*Switch, i, j int) { } func (sw *Switch) addPeerWithConnection(conn net.Conn) error { - pc, err := newInboundPeerConn(conn, sw.peerConfig, sw.nodeKey.PrivKey) + pc, err := newInboundPeerConn(conn, sw.config, sw.nodeKey.PrivKey) if err != nil { if err := conn.Close(); err != nil { sw.Logger.Error("Error closing connection", "err", err) @@ -131,7 +131,7 @@ func StartSwitches(switches []*Switch) error { return nil } -func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch func(int, *Switch) *Switch) *Switch { +func MakeSwitch(cfg *config.P2PConfig, i int, network, version string, initSwitch func(int, *Switch) *Switch) *Switch { // new switch, add reactors // TODO: let the config be passed in? nodeKey := &NodeKey{ diff --git a/types/priv_validator/priv_validator.go b/privval/priv_validator.go similarity index 100% rename from types/priv_validator/priv_validator.go rename to privval/priv_validator.go diff --git a/types/priv_validator/priv_validator_test.go b/privval/priv_validator_test.go similarity index 100% rename from types/priv_validator/priv_validator_test.go rename to privval/priv_validator_test.go diff --git a/types/priv_validator/socket.go b/privval/socket.go similarity index 100% rename from types/priv_validator/socket.go rename to privval/socket.go diff --git a/types/priv_validator/socket_tcp.go b/privval/socket_tcp.go similarity index 100% rename from types/priv_validator/socket_tcp.go rename to privval/socket_tcp.go diff --git a/types/priv_validator/socket_tcp_test.go b/privval/socket_tcp_test.go similarity index 100% rename from types/priv_validator/socket_tcp_test.go rename to privval/socket_tcp_test.go diff --git a/types/priv_validator/socket_test.go b/privval/socket_test.go similarity index 100% rename from types/priv_validator/socket_test.go rename to privval/socket_test.go diff --git a/types/priv_validator/wire.go b/privval/wire.go similarity index 100% rename from types/priv_validator/wire.go rename to privval/wire.go diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index e4bb5a29..7a042362 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -51,9 +51,9 @@ var ( // interfaces defined in types and above stateDB dbm.DB - blockStore types.BlockStore - mempool types.Mempool - evidencePool types.EvidencePool + blockStore sm.BlockStore + mempool sm.Mempool + evidencePool sm.EvidencePool consensusState Consensus p2pSwitch P2P @@ -72,15 +72,15 @@ func SetStateDB(db dbm.DB) { stateDB = db } -func SetBlockStore(bs types.BlockStore) { +func SetBlockStore(bs sm.BlockStore) { blockStore = bs } -func SetMempool(mem types.Mempool) { +func SetMempool(mem sm.Mempool) { mempool = mem } -func SetEvidencePool(evpool types.EvidencePool) { +func SetEvidencePool(evpool sm.EvidencePool) { evidencePool = evpool } @@ -125,6 +125,10 @@ func SetEventBus(b *types.EventBus) { } func validatePage(page, perPage, totalCount int) int { + if perPage < 1 { + return 1 + } + pages := ((totalCount - 1) / perPage) + 1 if page < 1 { page = 1 diff --git a/rpc/core/pipe_test.go b/rpc/core/pipe_test.go index a33e17cf..225e3649 100644 --- a/rpc/core/pipe_test.go +++ b/rpc/core/pipe_test.go @@ -15,6 +15,8 @@ func TestPaginationPage(t *testing.T) { page int newPage int }{ + {0, 0, 1, 1}, + {0, 10, 0, 1}, {0, 10, 1, 1}, {0, 10, 2, 1}, diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 615136a9..2e27969c 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -189,8 +189,8 @@ func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSear } totalCount := len(results) - page = validatePage(page, perPage, totalCount) perPage = validatePerPage(perPage) + page = validatePage(page, perPage, totalCount) skipCount := (page - 1) * perPage apiResults := make([]*ctypes.ResultTx, cmn.MinInt(perPage, totalCount-skipCount)) diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 07ccfb6f..b478fd33 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -32,7 +32,7 @@ func RegisterRPCFuncs(mux *http.ServeMux, funcMap map[string]*RPCFunc, cdc *amin } // JSONRPC endpoints - mux.HandleFunc("/", makeJSONRPCHandler(funcMap, cdc, logger)) + mux.HandleFunc("/", handleInvalidJSONRPCPaths(makeJSONRPCHandler(funcMap, cdc, logger))) } //------------------------------------- @@ -153,6 +153,19 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, cdc *amino.Codec, logger lo } } +func handleInvalidJSONRPCPaths(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Since the pattern "/" matches all paths not matched by other registered patterns we check whether the path is indeed + // "/", otherwise return a 404 error + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + + next(w, r) + } +} + func mapParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, params map[string]json.RawMessage, argsOffset int) ([]reflect.Value, error) { values := make([]reflect.Value, len(rpcFunc.argNames)) for i, argName := range rpcFunc.argNames { diff --git a/rpc/lib/server/handlers_test.go b/rpc/lib/server/handlers_test.go index 92a2d990..86de0e4c 100644 --- a/rpc/lib/server/handlers_test.go +++ b/rpc/lib/server/handlers_test.go @@ -97,3 +97,14 @@ func TestRPCNotification(t *testing.T) { require.Nil(t, err, "reading from the body should not give back an error") require.Equal(t, len(blob), 0, "a notification SHOULD NOT be responded to by the server") } + +func TestUnknownRPCPath(t *testing.T) { + mux := testMux() + req, _ := http.NewRequest("GET", "http://localhost/unknownrpcpath", nil) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + res := rec.Result() + + // Always expecting back a 404 error + require.Equal(t, http.StatusNotFound, res.StatusCode, "should always return 404") +} diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 11cafa0c..5d18299b 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -15,11 +15,11 @@ import ( cfg "github.com/tendermint/tendermint/config" nm "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" ctypes "github.com/tendermint/tendermint/rpc/core/types" core_grpc "github.com/tendermint/tendermint/rpc/grpc" rpcclient "github.com/tendermint/tendermint/rpc/lib/client" - pvm "github.com/tendermint/tendermint/types/priv_validator" ) var globalConfig *cfg.Config @@ -118,7 +118,7 @@ func NewTendermint(app abci.Application) *nm.Node { logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) logger = log.NewFilter(logger, log.AllowError()) pvFile := config.PrivValidatorFile() - pv := pvm.LoadOrGenFilePV(pvFile) + pv := privval.LoadOrGenFilePV(pvFile) papp := proxy.NewLocalClientCreator(app) node, err := nm.NewNode(config, pv, papp, nm.DefaultGenesisDocProviderFunc(config), diff --git a/scripts/install_tendermint.sh b/scripts/install_tendermint.sh new file mode 100644 index 00000000..0e1de117 --- /dev/null +++ b/scripts/install_tendermint.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# XXX: this script is intended to be run from +# a fresh Digital Ocean droplet with Ubuntu + +# upon its completion, you must either reset +# your terminal or run `source ~/.profile` + +# as written, this script will install +# tendermint core from master branch +REPO=github.com/tendermint/tendermint + +# change this to a specific release or branch +BRANCH=master + +sudo apt-get update -y +sudo apt-get upgrade -y +sudo apt-get install -y make + +# get and unpack golang +curl -O https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz +tar -xvf go1.10.linux-amd64.tar.gz + +# move go binary and add to path +mv go /usr/local +echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.profile + +# create the goApps directory, set GOPATH, and put it on PATH +mkdir goApps +echo "export GOPATH=/root/goApps" >> ~/.profile +echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.profile + +source ~/.profile + +# get the code and move into repo +go get $REPO +cd $GOPATH/src/$REPO + +# build & install +git checkout $BRANCH +# XXX: uncomment if branch isn't master +# git fetch origin $BRANCH +make get_tools +make get_vendor_deps +make install + +# the binary is located in $GOPATH/bin +# run `source ~/.profile` or reset your terminal +# to persist the changes diff --git a/scripts/wire2amino.go b/scripts/wire2amino.go index 94908287..a942ceaa 100644 --- a/scripts/wire2amino.go +++ b/scripts/wire2amino.go @@ -13,8 +13,8 @@ import ( cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/types" - priv_val "github.com/tendermint/tendermint/types/priv_validator" ) type GenesisValidator struct { @@ -84,7 +84,7 @@ func convertPrivVal(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { var pubKey crypto.PubKeyEd25519 copy(pubKey[:], privVal.PubKey.Data) - privValNew := priv_val.FilePV{ + privValNew := privval.FilePV{ Address: pubKey.Address(), PubKey: pubKey, LastHeight: privVal.LastHeight, diff --git a/state/execution.go b/state/execution.go index 3fe35e2f..77091159 100644 --- a/state/execution.go +++ b/state/execution.go @@ -29,8 +29,8 @@ type BlockExecutor struct { eventBus types.BlockEventPublisher // update these with block results after commit - mempool types.Mempool - evpool types.EvidencePool + mempool Mempool + evpool EvidencePool logger log.Logger } @@ -38,7 +38,7 @@ type BlockExecutor struct { // NewBlockExecutor returns a new BlockExecutor with a NopEventBus. // Call SetEventBus to provide one. func NewBlockExecutor(db dbm.DB, logger log.Logger, proxyApp proxy.AppConnConsensus, - mempool types.Mempool, evpool types.EvidencePool) *BlockExecutor { + mempool Mempool, evpool EvidencePool) *BlockExecutor { return &BlockExecutor{ db: db, proxyApp: proxyApp, @@ -59,8 +59,8 @@ func (blockExec *BlockExecutor) SetEventBus(eventBus types.BlockEventPublisher) // If the block is invalid, it returns an error. // Validation does not mutate state, but does require historical information from the stateDB, // ie. to verify evidence from a validator at an old height. -func (blockExec *BlockExecutor) ValidateBlock(s State, block *types.Block) error { - return validateBlock(blockExec.db, s, block) +func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) error { + return validateBlock(blockExec.db, state, block) } // ApplyBlock validates the block against the state, executes it against the app, @@ -68,15 +68,15 @@ func (blockExec *BlockExecutor) ValidateBlock(s State, block *types.Block) error // It's the only function that needs to be called // from outside this package to process and commit an entire block. // It takes a blockID to avoid recomputing the parts hash. -func (blockExec *BlockExecutor) ApplyBlock(s State, blockID types.BlockID, block *types.Block) (State, error) { +func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, block *types.Block) (State, error) { - if err := blockExec.ValidateBlock(s, block); err != nil { - return s, ErrInvalidBlock(err) + if err := blockExec.ValidateBlock(state, block); err != nil { + return state, ErrInvalidBlock(err) } abciResponses, err := execBlockOnProxyApp(blockExec.logger, blockExec.proxyApp, block) if err != nil { - return s, ErrProxyAppConn(err) + return state, ErrProxyAppConn(err) } fail.Fail() // XXX @@ -87,35 +87,33 @@ func (blockExec *BlockExecutor) ApplyBlock(s State, blockID types.BlockID, block fail.Fail() // XXX // update the state with the block and responses - s, err = updateState(s, blockID, block.Header, abciResponses) + state, err = updateState(state, blockID, block.Header, abciResponses) if err != nil { - return s, fmt.Errorf("Commit failed for application: %v", err) + return state, fmt.Errorf("Commit failed for application: %v", err) } - // lock mempool, commit state, update mempoool + // lock mempool, commit app state, update mempoool appHash, err := blockExec.Commit(block) if err != nil { - return s, fmt.Errorf("Commit failed for application: %v", err) + return state, fmt.Errorf("Commit failed for application: %v", err) } + // Update evpool with the block and state. + blockExec.evpool.Update(block, state) + fail.Fail() // XXX // update the app hash and save the state - s.AppHash = appHash - SaveState(blockExec.db, s) + state.AppHash = appHash + SaveState(blockExec.db, state) fail.Fail() // XXX - // Update evpool now that state is saved - // TODO: handle the crash/recover scenario - // ie. (may need to call Update for last block) - blockExec.evpool.Update(block) - // events are fired after everything else // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses) - return s, nil + return state, nil } // Commit locks the mempool, runs the ABCI Commit message, and updates the mempool. @@ -283,20 +281,20 @@ func updateValidators(currentSet *types.ValidatorSet, updates []abci.Validator) } // updateState returns a new State updated according to the header and responses. -func updateState(s State, blockID types.BlockID, header *types.Header, +func updateState(state State, blockID types.BlockID, header *types.Header, abciResponses *ABCIResponses) (State, error) { // copy the valset so we can apply changes from EndBlock // and update s.LastValidators and s.Validators - prevValSet := s.Validators.Copy() + prevValSet := state.Validators.Copy() nextValSet := prevValSet.Copy() // update the validator set with the latest abciResponses - lastHeightValsChanged := s.LastHeightValidatorsChanged + lastHeightValsChanged := state.LastHeightValidatorsChanged if len(abciResponses.EndBlock.ValidatorUpdates) > 0 { err := updateValidators(nextValSet, abciResponses.EndBlock.ValidatorUpdates) if err != nil { - return s, fmt.Errorf("Error changing validator set: %v", err) + return state, fmt.Errorf("Error changing validator set: %v", err) } // change results from this height but only applies to the next height lastHeightValsChanged = header.Height + 1 @@ -306,14 +304,14 @@ func updateState(s State, blockID types.BlockID, header *types.Header, nextValSet.IncrementAccum(1) // update the params with the latest abciResponses - nextParams := s.ConsensusParams - lastHeightParamsChanged := s.LastHeightConsensusParamsChanged + nextParams := state.ConsensusParams + lastHeightParamsChanged := state.LastHeightConsensusParamsChanged if abciResponses.EndBlock.ConsensusParamUpdates != nil { // NOTE: must not mutate s.ConsensusParams - nextParams = s.ConsensusParams.Update(abciResponses.EndBlock.ConsensusParamUpdates) + nextParams = state.ConsensusParams.Update(abciResponses.EndBlock.ConsensusParamUpdates) err := nextParams.Validate() if err != nil { - return s, fmt.Errorf("Error updating consensus params: %v", err) + return state, fmt.Errorf("Error updating consensus params: %v", err) } // change results from this height but only applies to the next height lastHeightParamsChanged = header.Height + 1 @@ -322,13 +320,13 @@ func updateState(s State, blockID types.BlockID, header *types.Header, // NOTE: the AppHash has not been populated. // It will be filled on state.Save. return State{ - ChainID: s.ChainID, + ChainID: state.ChainID, LastBlockHeight: header.Height, - LastBlockTotalTx: s.LastBlockTotalTx + header.NumTxs, + LastBlockTotalTx: state.LastBlockTotalTx + header.NumTxs, LastBlockID: blockID, LastBlockTime: header.Time, Validators: nextValSet, - LastValidators: s.Validators.Copy(), + LastValidators: state.Validators.Copy(), LastHeightValidatorsChanged: lastHeightValsChanged, ConsensusParams: nextParams, LastHeightConsensusParamsChanged: lastHeightParamsChanged, diff --git a/state/execution_test.go b/state/execution_test.go index 09c40b5a..b6c7f9a6 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -10,11 +10,12 @@ import ( "github.com/tendermint/abci/example/kvstore" abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/types" ) var ( @@ -34,7 +35,7 @@ func TestApplyBlock(t *testing.T) { state, stateDB := state(), dbm.NewMemDB() blockExec := NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), - types.MockMempool{}, types.MockEvidencePool{}) + MockMempool{}, MockEvidencePool{}) block := makeBlock(state, 1) blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} diff --git a/types/services.go b/state/services.go similarity index 57% rename from types/services.go rename to state/services.go index 6b2be8a5..bef286b2 100644 --- a/types/services.go +++ b/state/services.go @@ -1,11 +1,10 @@ -package types +package state import ( abci "github.com/tendermint/abci/types" + "github.com/tendermint/tendermint/types" ) -// NOTE/XXX: all type definitions in this file are considered UNSTABLE - //------------------------------------------------------ // blockchain services types // NOTE: Interfaces used by RPC must be thread safe! @@ -17,15 +16,14 @@ import ( // Mempool defines the mempool interface as used by the ConsensusState. // Updates to the mempool need to be synchronized with committing a block // so apps can reset their transient state on Commit -// UNSTABLE type Mempool interface { Lock() Unlock() Size() int - CheckTx(Tx, func(*abci.Response)) error - Reap(int) Txs - Update(height int64, txs Txs) error + CheckTx(types.Tx, func(*abci.Response)) error + Reap(int) types.Txs + Update(height int64, txs types.Txs) error Flush() FlushAppConn() error @@ -34,60 +32,55 @@ type Mempool interface { } // MockMempool is an empty implementation of a Mempool, useful for testing. -// UNSTABLE type MockMempool struct { } -func (m MockMempool) Lock() {} -func (m MockMempool) Unlock() {} -func (m MockMempool) Size() int { return 0 } -func (m MockMempool) CheckTx(tx Tx, cb func(*abci.Response)) error { return nil } -func (m MockMempool) Reap(n int) Txs { return Txs{} } -func (m MockMempool) Update(height int64, txs Txs) error { return nil } -func (m MockMempool) Flush() {} -func (m MockMempool) FlushAppConn() error { return nil } -func (m MockMempool) TxsAvailable() <-chan int64 { return make(chan int64) } -func (m MockMempool) EnableTxsAvailable() {} +func (m MockMempool) Lock() {} +func (m MockMempool) Unlock() {} +func (m MockMempool) Size() int { return 0 } +func (m MockMempool) CheckTx(tx types.Tx, cb func(*abci.Response)) error { return nil } +func (m MockMempool) Reap(n int) types.Txs { return types.Txs{} } +func (m MockMempool) Update(height int64, txs types.Txs) error { return nil } +func (m MockMempool) Flush() {} +func (m MockMempool) FlushAppConn() error { return nil } +func (m MockMempool) TxsAvailable() <-chan int64 { return make(chan int64) } +func (m MockMempool) EnableTxsAvailable() {} //------------------------------------------------------ // blockstore // BlockStoreRPC is the block store interface used by the RPC. -// UNSTABLE type BlockStoreRPC interface { Height() int64 - LoadBlockMeta(height int64) *BlockMeta - LoadBlock(height int64) *Block - LoadBlockPart(height int64, index int) *Part + LoadBlockMeta(height int64) *types.BlockMeta + LoadBlock(height int64) *types.Block + LoadBlockPart(height int64, index int) *types.Part - LoadBlockCommit(height int64) *Commit - LoadSeenCommit(height int64) *Commit + LoadBlockCommit(height int64) *types.Commit + LoadSeenCommit(height int64) *types.Commit } // BlockStore defines the BlockStore interface used by the ConsensusState. -// UNSTABLE type BlockStore interface { BlockStoreRPC - SaveBlock(block *Block, blockParts *PartSet, seenCommit *Commit) + SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) } -//------------------------------------------------------ +//----------------------------------------------------------------------------------------------------- // evidence pool // EvidencePool defines the EvidencePool interface used by the ConsensusState. -// UNSTABLE type EvidencePool interface { - PendingEvidence() []Evidence - AddEvidence(Evidence) error - Update(*Block) + PendingEvidence() []types.Evidence + AddEvidence(types.Evidence) error + Update(*types.Block, State) } // MockMempool is an empty implementation of a Mempool, useful for testing. -// UNSTABLE type MockEvidencePool struct { } -func (m MockEvidencePool) PendingEvidence() []Evidence { return nil } -func (m MockEvidencePool) AddEvidence(Evidence) error { return nil } -func (m MockEvidencePool) Update(*Block) {} +func (m MockEvidencePool) PendingEvidence() []types.Evidence { return nil } +func (m MockEvidencePool) AddEvidence(types.Evidence) error { return nil } +func (m MockEvidencePool) Update(*types.Block, State) {} diff --git a/state/state.go b/state/state.go index aa6e04b6..3bc08dae 100644 --- a/state/state.go +++ b/state/state.go @@ -55,67 +55,67 @@ type State struct { } // Copy makes a copy of the State for mutating. -func (s State) Copy() State { +func (state State) Copy() State { return State{ - ChainID: s.ChainID, + ChainID: state.ChainID, - LastBlockHeight: s.LastBlockHeight, - LastBlockTotalTx: s.LastBlockTotalTx, - LastBlockID: s.LastBlockID, - LastBlockTime: s.LastBlockTime, + LastBlockHeight: state.LastBlockHeight, + LastBlockTotalTx: state.LastBlockTotalTx, + LastBlockID: state.LastBlockID, + LastBlockTime: state.LastBlockTime, - Validators: s.Validators.Copy(), - LastValidators: s.LastValidators.Copy(), - LastHeightValidatorsChanged: s.LastHeightValidatorsChanged, + Validators: state.Validators.Copy(), + LastValidators: state.LastValidators.Copy(), + LastHeightValidatorsChanged: state.LastHeightValidatorsChanged, - ConsensusParams: s.ConsensusParams, - LastHeightConsensusParamsChanged: s.LastHeightConsensusParamsChanged, + ConsensusParams: state.ConsensusParams, + LastHeightConsensusParamsChanged: state.LastHeightConsensusParamsChanged, - AppHash: s.AppHash, + AppHash: state.AppHash, - LastResultsHash: s.LastResultsHash, + LastResultsHash: state.LastResultsHash, } } // Equals returns true if the States are identical. -func (s State) Equals(s2 State) bool { - sbz, s2bz := s.Bytes(), s2.Bytes() +func (state State) Equals(state2 State) bool { + sbz, s2bz := state.Bytes(), state2.Bytes() return bytes.Equal(sbz, s2bz) } // Bytes serializes the State using go-amino. -func (s State) Bytes() []byte { - return cdc.MustMarshalBinaryBare(s) +func (state State) Bytes() []byte { + return cdc.MustMarshalBinaryBare(state) } // IsEmpty returns true if the State is equal to the empty State. -func (s State) IsEmpty() bool { - return s.Validators == nil // XXX can't compare to Empty +func (state State) IsEmpty() bool { + return state.Validators == nil // XXX can't compare to Empty } // GetValidators returns the last and current validator sets. -func (s State) GetValidators() (last *types.ValidatorSet, current *types.ValidatorSet) { - return s.LastValidators, s.Validators +func (state State) GetValidators() (last *types.ValidatorSet, current *types.ValidatorSet) { + return state.LastValidators, state.Validators } //------------------------------------------------------------------------ // Create a block from the latest state // MakeBlock builds a block with the given txs and commit from the current state. -func (s State) MakeBlock(height int64, txs []types.Tx, commit *types.Commit) (*types.Block, *types.PartSet) { +func (state State) MakeBlock(height int64, txs []types.Tx, commit *types.Commit) (*types.Block, *types.PartSet) { // build base block block := types.MakeBlock(height, txs, commit) // fill header with state data - block.ChainID = s.ChainID - block.TotalTxs = s.LastBlockTotalTx + block.NumTxs - block.LastBlockID = s.LastBlockID - block.ValidatorsHash = s.Validators.Hash() - block.AppHash = s.AppHash - block.ConsensusHash = s.ConsensusParams.Hash() - block.LastResultsHash = s.LastResultsHash + block.ChainID = state.ChainID + block.TotalTxs = state.LastBlockTotalTx + block.NumTxs + block.LastBlockID = state.LastBlockID + block.ValidatorsHash = state.Validators.Hash() + block.AppHash = state.AppHash + block.ConsensusHash = state.ConsensusParams.Hash() + block.LastResultsHash = state.LastResultsHash - return block, block.MakePartSet(s.ConsensusParams.BlockGossip.BlockPartSizeBytes) + return block, block.MakePartSet(state.ConsensusParams.BlockGossip.BlockPartSizeBytes) } //------------------------------------------------------------------------ diff --git a/state/store.go b/state/store.go index ee0619d3..60acf9e1 100644 --- a/state/store.go +++ b/state/store.go @@ -80,15 +80,15 @@ func loadState(db dbm.DB, key []byte) (state State) { } // SaveState persists the State, the ValidatorsInfo, and the ConsensusParamsInfo to the database. -func SaveState(db dbm.DB, s State) { - saveState(db, s, stateKey) +func SaveState(db dbm.DB, state State) { + saveState(db, state, stateKey) } -func saveState(db dbm.DB, s State, key []byte) { - nextHeight := s.LastBlockHeight + 1 - saveValidatorsInfo(db, nextHeight, s.LastHeightValidatorsChanged, s.Validators) - saveConsensusParamsInfo(db, nextHeight, s.LastHeightConsensusParamsChanged, s.ConsensusParams) - db.SetSync(stateKey, s.Bytes()) +func saveState(db dbm.DB, state State, key []byte) { + nextHeight := state.LastBlockHeight + 1 + saveValidatorsInfo(db, nextHeight, state.LastHeightValidatorsChanged, state.Validators) + saveConsensusParamsInfo(db, nextHeight, state.LastHeightConsensusParamsChanged, state.ConsensusParams) + db.SetSync(stateKey, state.Bytes()) } //------------------------------------------------------------------------ diff --git a/state/validation.go b/state/validation.go index fb3e8d13..0726b61e 100644 --- a/state/validation.go +++ b/state/validation.go @@ -12,69 +12,69 @@ import ( //----------------------------------------------------- // Validate block -func validateBlock(stateDB dbm.DB, s State, b *types.Block) error { +func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { // validate internal consistency - if err := b.ValidateBasic(); err != nil { + if err := block.ValidateBasic(); err != nil { return err } // validate basic info - if b.ChainID != s.ChainID { - return fmt.Errorf("Wrong Block.Header.ChainID. Expected %v, got %v", s.ChainID, b.ChainID) + if block.ChainID != state.ChainID { + return fmt.Errorf("Wrong Block.Header.ChainID. Expected %v, got %v", state.ChainID, block.ChainID) } - if b.Height != s.LastBlockHeight+1 { - return fmt.Errorf("Wrong Block.Header.Height. Expected %v, got %v", s.LastBlockHeight+1, b.Height) + if block.Height != state.LastBlockHeight+1 { + return fmt.Errorf("Wrong Block.Header.Height. Expected %v, got %v", state.LastBlockHeight+1, block.Height) } /* TODO: Determine bounds for Time See blockchain/reactor "stopSyncingDurationMinutes" - if !b.Time.After(lastBlockTime) { + if !block.Time.After(lastBlockTime) { return errors.New("Invalid Block.Header.Time") } */ // validate prev block info - if !b.LastBlockID.Equals(s.LastBlockID) { - return fmt.Errorf("Wrong Block.Header.LastBlockID. Expected %v, got %v", s.LastBlockID, b.LastBlockID) + if !block.LastBlockID.Equals(state.LastBlockID) { + return fmt.Errorf("Wrong Block.Header.LastBlockID. Expected %v, got %v", state.LastBlockID, block.LastBlockID) } - newTxs := int64(len(b.Data.Txs)) - if b.TotalTxs != s.LastBlockTotalTx+newTxs { - return fmt.Errorf("Wrong Block.Header.TotalTxs. Expected %v, got %v", s.LastBlockTotalTx+newTxs, b.TotalTxs) + newTxs := int64(len(block.Data.Txs)) + if block.TotalTxs != state.LastBlockTotalTx+newTxs { + return fmt.Errorf("Wrong Block.Header.TotalTxs. Expected %v, got %v", state.LastBlockTotalTx+newTxs, block.TotalTxs) } // validate app info - if !bytes.Equal(b.AppHash, s.AppHash) { - return fmt.Errorf("Wrong Block.Header.AppHash. Expected %X, got %v", s.AppHash, b.AppHash) + if !bytes.Equal(block.AppHash, state.AppHash) { + return fmt.Errorf("Wrong Block.Header.AppHash. Expected %X, got %v", state.AppHash, block.AppHash) } - if !bytes.Equal(b.ConsensusHash, s.ConsensusParams.Hash()) { - return fmt.Errorf("Wrong Block.Header.ConsensusHash. Expected %X, got %v", s.ConsensusParams.Hash(), b.ConsensusHash) + if !bytes.Equal(block.ConsensusHash, state.ConsensusParams.Hash()) { + return fmt.Errorf("Wrong Block.Header.ConsensusHash. Expected %X, got %v", state.ConsensusParams.Hash(), block.ConsensusHash) } - if !bytes.Equal(b.LastResultsHash, s.LastResultsHash) { - return fmt.Errorf("Wrong Block.Header.LastResultsHash. Expected %X, got %v", s.LastResultsHash, b.LastResultsHash) + if !bytes.Equal(block.LastResultsHash, state.LastResultsHash) { + return fmt.Errorf("Wrong Block.Header.LastResultsHash. Expected %X, got %v", state.LastResultsHash, block.LastResultsHash) } - if !bytes.Equal(b.ValidatorsHash, s.Validators.Hash()) { - return fmt.Errorf("Wrong Block.Header.ValidatorsHash. Expected %X, got %v", s.Validators.Hash(), b.ValidatorsHash) + if !bytes.Equal(block.ValidatorsHash, state.Validators.Hash()) { + return fmt.Errorf("Wrong Block.Header.ValidatorsHash. Expected %X, got %v", state.Validators.Hash(), block.ValidatorsHash) } // Validate block LastCommit. - if b.Height == 1 { - if len(b.LastCommit.Precommits) != 0 { + if block.Height == 1 { + if len(block.LastCommit.Precommits) != 0 { return errors.New("Block at height 1 (first block) should have no LastCommit precommits") } } else { - if len(b.LastCommit.Precommits) != s.LastValidators.Size() { + if len(block.LastCommit.Precommits) != state.LastValidators.Size() { return fmt.Errorf("Invalid block commit size. Expected %v, got %v", - s.LastValidators.Size(), len(b.LastCommit.Precommits)) + state.LastValidators.Size(), len(block.LastCommit.Precommits)) } - err := s.LastValidators.VerifyCommit( - s.ChainID, s.LastBlockID, b.Height-1, b.LastCommit) + err := state.LastValidators.VerifyCommit( + state.ChainID, state.LastBlockID, block.Height-1, block.LastCommit) if err != nil { return err } } - for _, ev := range b.Evidence.Evidence { - if err := VerifyEvidence(stateDB, s, ev); err != nil { + for _, ev := range block.Evidence.Evidence { + if err := VerifyEvidence(stateDB, state, ev); err != nil { return types.NewEvidenceInvalidErr(ev, err) } } @@ -87,17 +87,17 @@ func validateBlock(stateDB dbm.DB, s State, b *types.Block) error { // VerifyEvidence verifies the evidence fully by checking it is internally // consistent and sufficiently recent. -func VerifyEvidence(stateDB dbm.DB, s State, evidence types.Evidence) error { - height := s.LastBlockHeight +func VerifyEvidence(stateDB dbm.DB, state State, evidence types.Evidence) error { + height := state.LastBlockHeight evidenceAge := height - evidence.Height() - maxAge := s.ConsensusParams.EvidenceParams.MaxAge + maxAge := state.ConsensusParams.EvidenceParams.MaxAge if evidenceAge > maxAge { return fmt.Errorf("Evidence from height %d is too old. Min height is %d", evidence.Height(), height-maxAge) } - if err := evidence.Verify(s.ChainID); err != nil { + if err := evidence.Verify(state.ChainID); err != nil { return err } diff --git a/test/app/grpc_client.go b/test/app/grpc_client.go old mode 100755 new mode 100644 index 9d024b1b..c55713c7 --- a/test/app/grpc_client.go +++ b/test/app/grpc_client.go @@ -2,12 +2,12 @@ package main import ( "encoding/hex" + "encoding/json" "fmt" "os" "context" - "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/rpc/grpc" ) @@ -32,5 +32,11 @@ func main() { fmt.Println(err) os.Exit(1) } - fmt.Println(string(wire.JSONBytes(res))) + + bz, err := json.Marshal(res) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println(string(bz)) } diff --git a/version/version.go b/version/version.go index a6ec6fba..a26ad913 100644 --- a/version/version.go +++ b/version/version.go @@ -4,13 +4,13 @@ package version const ( Maj = "0" Min = "19" - Fix = "8" + Fix = "9" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.19.8" + Version = "0.19.9" // GitCommit is the current HEAD set using ldflags. GitCommit string