Merge pull request #1703 from tendermint/release/v0.20.0

Release/v0.20.0

.circleci/config.yml

@@ -77,6 +77,22 @@ jobs:
           paths:
             - "bin/abci*"
 
+  build_slate:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: slate docs
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make build-slate
+
   lint:
     <<: *defaults
     steps:
@@ -117,18 +133,21 @@ jobs:
           key: v1-pkg-cache
       - restore_cache:
           key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run: mkdir -p /tmp/logs
       - run:
           name: Run tests
           command: |
             for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
               id=$(basename "$pkg")
 
-              go test -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg"
+              GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
             done
       - persist_to_workspace:
           root: /tmp/workspace
           paths:
             - "profiles/*"
+      - store_artifacts:
+          path: /tmp/logs
 
   test_persistence:
     <<: *defaults

.gitignore

@@ -5,7 +5,6 @@
 .DS_Store
 build/*
 rpc/test/.tendermint
-.debora
 .tendermint
 remote_dump
 .revision
@@ -13,7 +12,6 @@ vendor
 .vagrant
 test/p2p/data/
 test/logs
-.glide
 coverage.txt
 docs/_build
 docs/tools
@@ -25,3 +23,5 @@ scripts/cutWALUntil/cutWALUntil
 
 .idea/
 *.iml
+
+libs/pubsub/query/fuzz_test/output

CHANGELOG.md

@@ -1,5 +1,86 @@
 # Changelog
 
+## 0.20.0
+
+*June 6th, 2018*
+
+This is the first in a series of breaking releases coming to Tendermint after
+soliciting developer feedback and conducting security audits.
+
+This release does not break any blockchain data structures or
+protocols other than the ABCI messages between Tendermint and the application.
+
+Applications that upgrade for ABCI v0.11.0 should be able to continue running Tendermint
+v0.20.0 on blockchains created with v0.19.X
+
+BREAKING CHANGES
+
+- [abci] Upgrade to
+  [v0.11.0](https://github.com/tendermint/abci/blob/master/CHANGELOG.md#0110)
+- [abci] Change Query path for filtering peers by node ID from
+  `p2p/filter/pubkey/<id>` to `p2p/filter/id/<id>`
+
+## 0.19.9
+
+*June 5th, 2018*
+
+BREAKING CHANGES
+
+- [types/priv_validator] Moved to top level `privval` package
+
+FEATURES
+
+- [config] Collapse PeerConfig into P2PConfig
+- [docs] Add quick-install script
+- [docs/spec] Add table of Amino prefixes
+
+BUG FIXES
+
+- [rpc] Return 404 for unknown endpoints
+- [consensus] Flush WAL on stop
+- [evidence] Don't send evidence to peers that are behind
+- [p2p] Fix memory leak on peer disconnects
+- [rpc] Fix panic when `per_page=0`
+
+## 0.19.8
+
+*June 4th, 2018*
+
+BREAKING:
+
+- [p2p] Remove `auth_enc` config option, peer connections are always auth
+  encrypted. Technically a breaking change but seems no one was using it and
+  arguably a bug fix :)
+
+BUG FIXES
+
+- [mempool] Fix deadlock under high load when `skip_timeout_commit=true` and
+  `create_empty_blocks=false`
+
+## 0.19.7
+
+*May 31st, 2018*
+
+BREAKING:
+
+- [libs/pubsub] TagMap#Get returns a string value
+- [libs/pubsub] NewTagMap accepts a map of strings
+
+FEATURES
+
+- [rpc] the RPC documentation is now published to https://tendermint.github.io/slate
+- [p2p] AllowDuplicateIP config option to refuse connections from same IP.
+    - true by default for now, false by default in next breaking release
+- [docs] Add docs for query, tx indexing, events, pubsub
+- [docs] Add some notes about running Tendermint in production
+
+IMPROVEMENTS:
+
+- [consensus] Consensus reactor now receives events from a separate synchronous event bus,
+  which is not dependant on external RPC load
+- [consensus/wal] do not look for height in older files if we've seen height - 1
+- [docs] Various cleanup and link fixes
+
 ## 0.19.6
 
 *May 29th, 2018*
@@ -8,6 +89,12 @@ BUG FIXES
 
 - [blockchain] Fix fast-sync deadlock during high peer turnover
 
+BUG FIX:
+
+- [evidence] Dont send peers evidence from heights they haven't synced to yet
+- [p2p] Refuse connections to more than one peer with the same IP
+- [docs] Various fixes
+
 ## 0.19.5
 
 *May 20th, 2018*

DOCKER/README.md

@@ -17,7 +17,7 @@
 # Quick reference
 
 * **Where to get help:**
-  https://tendermint.com/community
+  https://cosmos.network/community
 
 * **Where to file issues:**
   https://github.com/tendermint/tendermint/issues
@@ -37,25 +37,29 @@ To get started developing applications, see the [application developers guide](h
 
 ## Start one instance of the Tendermint core with the `kvstore` app
 
-A very simple example of a built-in app and Tendermint core in one container.
+A quick example of a built-in app and Tendermint core in one container.
 
 ```
 docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init
 docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint node --proxy_app=kvstore
 ```
 
-## mintnet-kubernetes
+# Local cluster
 
-If you want to see many containers talking to each other, consider using [mintnet-kubernetes](https://github.com/tendermint/tools/tree/master/mintnet-kubernetes), which is a tool for running Tendermint-based applications on a Kubernetes cluster.
+To run a 4-node network, see the `Makefile` in the root of [the repo](https://github.com/tendermint/tendermint/master/Makefile) and run:
+
+```
+make build-linux
+make build-docker-localnode
+make localnet-start
+```
+
+Note that this will build and use a different image than the ones provided here.
 
 # License
 
-View [license information](https://raw.githubusercontent.com/tendermint/tendermint/master/LICENSE) for the software contained in this image.
+- Tendermint's license is [Apache 2.0](https://github.com/tendermint/tendermint/master/LICENSE).
 
-# User Feedback
+# Contributing
 
-## Contributing
-
-You are invited to contribute new features, fixes, or updates, large or small; we are always thrilled to receive pull requests, and do our best to process them as fast as we can.
-
-Before you start to code, we recommend discussing your plans through a [GitHub](https://github.com/tendermint/tendermint/issues) issue, especially for more ambitious contributions. This gives other contributors a chance to point you in the right direction, give you feedback on your design, and help you find out if someone else is working on the same thing.
+Contributions are most welcome! See the [contributing file](https://github.com/tendermint/tendermint/blob/master/CONTRIBUTING.md) for more information.

Gopkg.lock

@@ -5,7 +5,7 @@
   branch = "master"
   name = "github.com/btcsuite/btcd"
   packages = ["btcec"]
-  revision = "675abc5df3c5531bc741b56a765e35623459da6d"
+  revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64"
 
 [[projects]]
   name = "github.com/davecgh/go-spew"
@@ -82,7 +82,7 @@
   branch = "master"
   name = "github.com/golang/snappy"
   packages = ["."]
-  revision = "553a641470496b2327abcac10b36396bd98e45c9"
+  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
 
 [[projects]]
   name = "github.com/gorilla/websocket"
@@ -128,20 +128,20 @@
 [[projects]]
   name = "github.com/magiconair/properties"
   packages = ["."]
-  revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6"
-  version = "v1.7.6"
+  revision = "c2353362d570a7bfa228149c62842019201cfb71"
+  version = "v1.8.0"
 
 [[projects]]
   branch = "master"
   name = "github.com/mitchellh/mapstructure"
   packages = ["."]
-  revision = "00c29f56e2386353d58c599509e8dc3801b0d716"
+  revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b"
 
 [[projects]]
   name = "github.com/pelletier/go-toml"
   packages = ["."]
-  revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
-  version = "v1.1.0"
+  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
+  version = "v1.2.0"
 
 [[projects]]
   name = "github.com/pkg/errors"
@@ -159,7 +159,7 @@
   branch = "master"
   name = "github.com/rcrowley/go-metrics"
   packages = ["."]
-  revision = "d932a24a8ccb8fcadc993e5c6c58f93dac168294"
+  revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
 
 [[projects]]
   name = "github.com/spf13/afero"
@@ -179,8 +179,8 @@
 [[projects]]
   name = "github.com/spf13/cobra"
   packages = ["."]
-  revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4"
-  version = "v0.0.2"
+  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+  version = "v0.0.3"
 
 [[projects]]
   branch = "master"
@@ -226,7 +226,7 @@
     "leveldb/table",
     "leveldb/util"
   ]
-  revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad"
+  revision = "5d6fca44a948d2be89a9702de7717f0168403d3d"
 
 [[projects]]
   name = "github.com/tendermint/abci"
@@ -238,8 +238,8 @@
     "server",
     "types"
   ]
-  revision = "78a8905690ef54f9d57e3b2b0ee7ad3a04ef3f1f"
-  version = "v0.10.3"
+  revision = "ebee2fe114020aa49c70bbbae50b7079fc7e7b90"
+  version = "v0.11.0"
 
 [[projects]]
   branch = "master"
@@ -263,12 +263,6 @@
   revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19"
   version = "v0.6.2"
 
-[[projects]]
-  name = "github.com/tendermint/go-wire"
-  packages = ["."]
-  revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c"
-  version = "v0.7.3"
-
 [[projects]]
   name = "github.com/tendermint/tmlibs"
   packages = [
@@ -281,12 +275,10 @@
     "flowrate",
     "log",
     "merkle",
-    "pubsub",
-    "pubsub/query",
     "test"
   ]
-  revision = "cc5f287c4798ffe88c04d02df219ecb6932080fd"
-  version = "v0.8.3-rc0"
+  revision = "692f1d86a6e2c0efa698fd1e4541b68c74ffaf38"
+  version = "v0.8.4"
 
 [[projects]]
   branch = "master"
@@ -301,7 +293,7 @@
     "ripemd160",
     "salsa20/salsa"
   ]
-  revision = "b0697eccbea9adec5b7ba8008f4c33d98d733388"
+  revision = "b47b1587369238182299fe4dad77d05b8b461e06"
 
 [[projects]]
   branch = "master"
@@ -313,16 +305,15 @@
     "http2/hpack",
     "idna",
     "internal/timeseries",
-    "lex/httplex",
     "trace"
   ]
-  revision = "5f9ae10d9af5b1c89ae6904293b14b064d4ada23"
+  revision = "1e491301e022f8f977054da4c2d852decd59571f"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
   packages = ["unix"]
-  revision = "bb9c189858d91f42db229b04d45a4c3d23a7662a"
+  revision = "9527bec2660bd847c050fda93a0f0c6dee0800bb"
 
 [[projects]]
   name = "golang.org/x/text"
@@ -346,7 +337,6 @@
   version = "v0.3.0"
 
 [[projects]]
-  branch = "master"
   name = "google.golang.org/genproto"
   packages = ["googleapis/rpc/status"]
   revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
@@ -384,6 +374,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "52a0dcbebdf8714612444914cfce59a3af8c47c4453a2d43c4ccc5ff1a91d8ea"
+  inputs-digest = "ae6792578b0664708339f4c05e020687d6b799ad6f8282394b919de69e403d1f"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -71,7 +71,7 @@
 
 [[constraint]]
   name = "github.com/tendermint/abci"
-  version = "~0.10.3"
+  version = "~0.11.0"
 
 [[constraint]]
   name = "github.com/tendermint/go-crypto"
@@ -79,16 +79,21 @@
 
 [[constraint]]
   name = "github.com/tendermint/go-amino"
-  version = "0.9.9"
+  version = "=0.9.9"
 
 [[override]]
   name = "github.com/tendermint/tmlibs"
-  version = "~0.8.3-rc0"
+  version = "~0.8.4"
 
 [[constraint]]
   name = "google.golang.org/grpc"
   version = "~1.7.3"
 
+# this got updated and broke, so locked to an old working commit ...
+[[override]]
+  name = "google.golang.org/genproto"
+  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
+
 [prune]
   go-tests = true
   unused-packages = true

Makefile

@@ -226,8 +226,11 @@ sentry-stop:
 	@if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi
 	cd networks/remote/terraform && terraform destroy -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub"
 
+# meant for the CI, inspect script & adapt accordingly
+build-slate:
+	bash scripts/slate.sh
+
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop
-
+.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate

blockchain/reactor_test.go

@@ -1,6 +1,7 @@
 package blockchain
 
 import (
+	"net"
 	"testing"
 
 	cmn "github.com/tendermint/tmlibs/common"
@@ -35,7 +36,7 @@ func newBlockchainReactor(logger log.Logger, maxBlockHeight int64) *BlockchainRe
 	fastSync := true
 	var nilApp proxy.AppConnConsensus
 	blockExec := sm.NewBlockExecutor(dbm.NewMemDB(), log.TestingLogger(), nilApp,
-		types.MockMempool{}, types.MockEvidencePool{})
+		sm.MockMempool{}, sm.MockEvidencePool{})
 
 	bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync)
 	bcReactor.SetLogger(logger.With("module", "blockchain"))
@@ -204,3 +205,4 @@ func (tp *bcrTestPeer) IsOutbound() bool                     { return false }
 func (tp *bcrTestPeer) IsPersistent() bool                   { return true }
 func (tp *bcrTestPeer) Get(s string) interface{}             { return s }
 func (tp *bcrTestPeer) Set(string, interface{})              {}
+func (tp *bcrTestPeer) RemoteIP() net.IP                     { return []byte{127, 0, 0, 1} }

cmd/priv_val_server/main.go

@@ -8,7 +8,7 @@ import (
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
 
-	priv_val "github.com/tendermint/tendermint/types/priv_validator"
+	"github.com/tendermint/tendermint/privval"
 )
 
 func main() {
@@ -30,13 +30,13 @@ func main() {
 		"privPath", *privValPath,
 	)
 
-	privVal := priv_val.LoadFilePV(*privValPath)
+	pv := privval.LoadFilePV(*privValPath)
 
-	rs := priv_val.NewRemoteSigner(
+	rs := privval.NewRemoteSigner(
 		logger,
 		*chainID,
 		*addr,
-		privVal,
+		pv,
 		crypto.GenPrivKeyEd25519(),
 	)
 	err := rs.Start()

cmd/tendermint/commands/gen_validator.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
+	"github.com/tendermint/tendermint/privval"
 )
 
 // GenValidatorCmd allows the generation of a keypair for a
@@ -17,7 +17,7 @@ var GenValidatorCmd = &cobra.Command{
 }
 
 func genValidator(cmd *cobra.Command, args []string) {
-	pv := pvm.GenFilePV("")
+	pv := privval.GenFilePV("")
 	jsbz, err := cdc.MarshalJSON(pv)
 	if err != nil {
 		panic(err)

cmd/tendermint/commands/init.go

@@ -7,8 +7,8 @@ import (
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/types"
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
 	cmn "github.com/tendermint/tmlibs/common"
 )
 
@@ -26,12 +26,12 @@ func initFiles(cmd *cobra.Command, args []string) error {
 func initFilesWithConfig(config *cfg.Config) error {
 	// private validator
 	privValFile := config.PrivValidatorFile()
-	var pv *pvm.FilePV
+	var pv *privval.FilePV
 	if cmn.FileExists(privValFile) {
-		pv = pvm.LoadFilePV(privValFile)
+		pv = privval.LoadFilePV(privValFile)
 		logger.Info("Found private validator", "path", privValFile)
 	} else {
-		pv = pvm.GenFilePV(privValFile)
+		pv = privval.GenFilePV(privValFile)
 		pv.Save()
 		logger.Info("Generated private validator", "path", privValFile)
 	}

cmd/tendermint/commands/reset_priv_validator.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tmlibs/log"
 )
 
@@ -13,14 +13,14 @@ import (
 // instance.
 var ResetAllCmd = &cobra.Command{
 	Use:   "unsafe_reset_all",
-	Short: "(unsafe) Remove all the data and WAL, reset this node's validator",
+	Short: "(unsafe) Remove all the data and WAL, reset this node's validator to genesis state",
 	Run:   resetAll,
 }
 
 // ResetPrivValidatorCmd resets the private validator files.
 var ResetPrivValidatorCmd = &cobra.Command{
 	Use:   "unsafe_reset_priv_validator",
-	Short: "(unsafe) Reset this node's validator",
+	Short: "(unsafe) Reset this node's validator to genesis state",
 	Run:   resetPrivValidator,
 }
 
@@ -32,7 +32,7 @@ func ResetAll(dbDir, privValFile string, logger log.Logger) {
 		logger.Error("Error removing directory", "err", err)
 		return
 	}
-	logger.Info("Removed all data", "dir", dbDir)
+	logger.Info("Removed all blockchain history", "dir", dbDir)
 }
 
 // XXX: this is totally unsafe.
@@ -50,11 +50,11 @@ func resetPrivValidator(cmd *cobra.Command, args []string) {
 func resetFilePV(privValFile string, logger log.Logger) {
 	// Get PrivValidator
 	if _, err := os.Stat(privValFile); err == nil {
-		pv := pvm.LoadFilePV(privValFile)
+		pv := privval.LoadFilePV(privValFile)
 		pv.Reset()
-		logger.Info("Reset PrivValidator", "file", privValFile)
+		logger.Info("Reset PrivValidator to genesis state", "file", privValFile)
 	} else {
-		pv := pvm.GenFilePV(privValFile)
+		pv := privval.GenFilePV(privValFile)
 		pv.Save()
 		logger.Info("Generated PrivValidator", "file", privValFile)
 	}

cmd/tendermint/commands/show_validator.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	privval "github.com/tendermint/tendermint/types/priv_validator"
+	"github.com/tendermint/tendermint/privval"
 )
 
 // ShowValidatorCmd adds capabilities for showing the validator info.

cmd/tendermint/commands/testnet.go

@@ -12,8 +12,8 @@ import (
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/types"
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
 	cmn "github.com/tendermint/tmlibs/common"
 )
 
@@ -89,7 +89,7 @@ func testnetFiles(cmd *cobra.Command, args []string) error {
 		initFilesWithConfig(config)
 
 		pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator)
-		pv := pvm.LoadFilePV(pvFile)
+		pv := privval.LoadFilePV(pvFile)
 		genVals[i] = types.GenesisValidator{
 			PubKey: pv.GetPubKey(),
 			Power:  1,

config/config.go

@@ -7,6 +7,13 @@ import (
 	"time"
 )
 
+const (
+	// FuzzModeDrop is a mode in which we randomly drop reads/writes, connections or sleep
+	FuzzModeDrop = iota
+	// FuzzModeDelay is a mode in which we randomly sleep
+	FuzzModeDelay
+)
+
 // NOTE: Most of the structs & relevant comments + the
 // default configuration options were used to manually
 // generate the config.toml. Please reflect any changes
@@ -287,11 +294,23 @@ type P2PConfig struct {
 	// Does not work if the peer-exchange reactor is disabled.
 	SeedMode bool `mapstructure:"seed_mode"`
 
-	// Authenticated encryption
-	AuthEnc bool `mapstructure:"auth_enc"`
-
-	// Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
+	// Comma separated list of peer IDs to keep private (will not be gossiped to
+	// other peers)
 	PrivatePeerIDs string `mapstructure:"private_peer_ids"`
+
+	// Toggle to disable guard against peers connecting from the same ip.
+	AllowDuplicateIP bool `mapstructure:"allow_duplicate_ip"`
+
+	// Peer connection configuration.
+	HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
+	DialTimeout      time.Duration `mapstructure:"dial_timeout"`
+
+	// Testing params.
+	// Force dial to fail
+	TestDialFail bool `mapstructure:"test_dial_fail"`
+	// FUzz connection
+	TestFuzz       bool            `mapstructure:"test_fuzz"`
+	TestFuzzConfig *FuzzConnConfig `mapstructure:"test_fuzz_config"`
 }
 
 // DefaultP2PConfig returns a default configuration for the peer-to-peer layer
@@ -307,7 +326,12 @@ func DefaultP2PConfig() *P2PConfig {
 		RecvRate:                512000, // 500 kB/s
 		PexReactor:              true,
 		SeedMode:                false,
-		AuthEnc:                 true,
+		AllowDuplicateIP:        true, // so non-breaking yet
+		HandshakeTimeout:        20 * time.Second,
+		DialTimeout:             3 * time.Second,
+		TestDialFail:            false,
+		TestFuzz:                false,
+		TestFuzzConfig:          DefaultFuzzConnConfig(),
 	}
 }
 
@@ -317,6 +341,7 @@ func TestP2PConfig() *P2PConfig {
 	cfg.ListenAddress = "tcp://0.0.0.0:36656"
 	cfg.SkipUPNP = true
 	cfg.FlushThrottleTimeout = 10
+	cfg.AllowDuplicateIP = true
 	return cfg
 }
 
@@ -325,6 +350,26 @@ func (cfg *P2PConfig) AddrBookFile() string {
 	return rootify(cfg.AddrBook, cfg.RootDir)
 }
 
+// FuzzConnConfig is a FuzzedConnection configuration.
+type FuzzConnConfig struct {
+	Mode         int
+	MaxDelay     time.Duration
+	ProbDropRW   float64
+	ProbDropConn float64
+	ProbSleep    float64
+}
+
+// DefaultFuzzConnConfig returns the default config.
+func DefaultFuzzConnConfig() *FuzzConnConfig {
+	return &FuzzConnConfig{
+		Mode:         FuzzModeDrop,
+		MaxDelay:     3 * time.Second,
+		ProbDropRW:   0.2,
+		ProbDropConn: 0.00,
+		ProbSleep:    0.00,
+	}
+}
+
 //-----------------------------------------------------------------------------
 // MempoolConfig
 

config/toml.go

@@ -165,9 +165,6 @@ pex = {{ .P2P.PexReactor }}
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = {{ .P2P.SeedMode }}
 
-# Authenticated encryption
-auth_enc = {{ .P2P.AuthEnc }}
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = "{{ .P2P.PrivatePeerIDs }}"
 

consensus/README.md

@@ -1,18 +1 @@
-# The core consensus algorithm.
-
-* state.go - The state machine as detailed in the whitepaper
-* reactor.go - A reactor that connects the state machine to the gossip network
-
-# Go-routine summary
-
-The reactor runs 2 go-routines for each added peer: gossipDataRoutine and gossipVotesRoutine.
-
-The consensus state runs two persistent go-routines: timeoutRoutine and receiveRoutine.
-Go-routines are also started to trigger timeouts and to avoid blocking when the internalMsgQueue is really backed up.
-
-# Replay/WAL
-
-A write-ahead log is used to record all messages processed by the receiveRoutine,
-which amounts to all inputs to the consensus state machine:
-messages from peers, messages from ourselves, and timeouts.
-They can be played back deterministically at startup or using the replay console. 
+See the [consensus spec](https://github.com/tendermint/tendermint/tree/master/docs/spec/consensus) and the [reactor consensus spec](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/consensus) for more information.

consensus/byzantine_test.go

@@ -27,7 +27,7 @@ func init() {
 // Heal partition and ensure A sees the commit
 func TestByzantine(t *testing.T) {
 	N := 4
-	logger := consensusLogger()
+	logger := consensusLogger().With("test", "byzantine")
 	css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter)
 
 	// give the byzantine validator a normal ticker
@@ -58,14 +58,11 @@ func TestByzantine(t *testing.T) {
 			css[i].doPrevote = func(height int64, round int) {}
 		}
 
-		eventBus := types.NewEventBus()
+		eventBus := css[i].eventBus
 		eventBus.SetLogger(logger.With("module", "events", "validator", i))
-		err := eventBus.Start()
-		require.NoError(t, err)
-		defer eventBus.Stop()
 
 		eventChans[i] = make(chan interface{}, 1)
-		err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		err := eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
 		require.NoError(t, err)
 
 		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
@@ -105,15 +102,18 @@ func TestByzantine(t *testing.T) {
 		p2p.Connect2Switches(sws, i, j)
 	})
 
-	// start the state machines
-	byzR := reactors[0].(*ByzantineReactor)
-	s := byzR.reactor.conS.GetState()
-	byzR.reactor.SwitchToConsensus(s, 0)
+	// start the non-byz state machines.
+	// note these must be started before the byz
 	for i := 1; i < N; i++ {
 		cr := reactors[i].(*ConsensusReactor)
 		cr.SwitchToConsensus(cr.conS.GetState(), 0)
 	}
 
+	// start the byzantine state machine
+	byzR := reactors[0].(*ByzantineReactor)
+	s := byzR.reactor.conS.GetState()
+	byzR.reactor.SwitchToConsensus(s, 0)
+
 	// byz proposer sends one block to peers[0]
 	// and the other block to peers[1] and peers[2].
 	// note peers and switches order don't match.

consensus/common_test.go

@@ -19,9 +19,9 @@ import (
 	cstypes "github.com/tendermint/tendermint/consensus/types"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
 	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
@@ -262,9 +262,9 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S
 	}
 
 	// mock the evidence pool
-	evpool := types.MockEvidencePool{}
+	evpool := sm.MockEvidencePool{}
 
-	// Make ConsensusReactor
+	// Make ConsensusState
 	stateDB := dbm.NewMemDB()
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
 	cs := NewConsensusState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
@@ -278,10 +278,10 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S
 	return cs
 }
 
-func loadPrivValidator(config *cfg.Config) *pvm.FilePV {
+func loadPrivValidator(config *cfg.Config) *privval.FilePV {
 	privValidatorFile := config.PrivValidatorFile()
 	ensureDir(path.Dir(privValidatorFile), 0700)
-	privValidator := pvm.LoadOrGenFilePV(privValidatorFile)
+	privValidator := privval.LoadOrGenFilePV(privValidatorFile)
 	privValidator.Reset()
 	return privValidator
 }
@@ -379,7 +379,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 			privVal = privVals[i]
 		} else {
 			_, tempFilePath := cmn.Tempfile("priv_validator_")
-			privVal = pvm.GenFilePV(tempFilePath)
+			privVal = privval.GenFilePV(tempFilePath)
 		}
 
 		app := appFunc()

consensus/reactor.go

@@ -1,7 +1,6 @@
 package consensus
 
 import (
-	"context"
 	"fmt"
 	"reflect"
 	"sync"
@@ -14,6 +13,7 @@ import (
 	"github.com/tendermint/tmlibs/log"
 
 	cstypes "github.com/tendermint/tendermint/consensus/types"
+	tmevents "github.com/tendermint/tendermint/libs/events"
 	"github.com/tendermint/tendermint/p2p"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -43,7 +43,8 @@ type ConsensusReactor struct {
 	eventBus *types.EventBus
 }
 
-// NewConsensusReactor returns a new ConsensusReactor with the given consensusState.
+// NewConsensusReactor returns a new ConsensusReactor with the given
+// consensusState.
 func NewConsensusReactor(consensusState *ConsensusState, fastSync bool) *ConsensusReactor {
 	conR := &ConsensusReactor{
 		conS:     consensusState,
@@ -53,17 +54,15 @@ func NewConsensusReactor(consensusState *ConsensusState, fastSync bool) *Consens
 	return conR
 }
 
-// OnStart implements BaseService.
+// OnStart implements BaseService by subscribing to events, which later will be
+// broadcasted to other peers and starting state if we're not in fast sync.
 func (conR *ConsensusReactor) OnStart() error {
 	conR.Logger.Info("ConsensusReactor ", "fastSync", conR.FastSync())
 	if err := conR.BaseReactor.OnStart(); err != nil {
 		return err
 	}
 
-	err := conR.startBroadcastRoutine()
-	if err != nil {
-		return err
-	}
+	conR.subscribeToBroadcastEvents()
 
 	if !conR.FastSync() {
 		err := conR.conS.Start()
@@ -75,9 +74,11 @@ func (conR *ConsensusReactor) OnStart() error {
 	return nil
 }
 
-// OnStop implements BaseService
+// OnStop implements BaseService by unsubscribing from events and stopping
+// state.
 func (conR *ConsensusReactor) OnStop() {
 	conR.BaseReactor.OnStop()
+	conR.unsubscribeFromBroadcastEvents()
 	conR.conS.Stop()
 }
 
@@ -101,6 +102,7 @@ func (conR *ConsensusReactor) SwitchToConsensus(state sm.State, blocksSynced int
 	err := conR.conS.Start()
 	if err != nil {
 		conR.Logger.Error("Error starting conS", "err", err)
+		return
 	}
 }
 
@@ -345,77 +347,40 @@ func (conR *ConsensusReactor) FastSync() bool {
 
 //--------------------------------------
 
-// startBroadcastRoutine subscribes for new round steps, votes and proposal
-// heartbeats using the event bus and starts a go routine to broadcasts events
-// to peers upon receiving them.
-func (conR *ConsensusReactor) startBroadcastRoutine() error {
+// subscribeToBroadcastEvents subscribes for new round steps, votes and
+// proposal heartbeats using internal pubsub defined on state to broadcast
+// them to peers upon receiving.
+func (conR *ConsensusReactor) subscribeToBroadcastEvents() {
 	const subscriber = "consensus-reactor"
-	ctx := context.Background()
+	conR.conS.evsw.AddListenerForEvent(subscriber, types.EventNewRoundStep,
+		func(data tmevents.EventData) {
+			conR.broadcastNewRoundStepMessages(data.(*cstypes.RoundState))
+		})
 
-	// new round steps
-	stepsCh := make(chan interface{})
-	err := conR.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, stepsCh)
-	if err != nil {
-		return errors.Wrapf(err, "failed to subscribe %s to %s", subscriber, types.EventQueryNewRoundStep)
-	}
+	conR.conS.evsw.AddListenerForEvent(subscriber, types.EventVote,
+		func(data tmevents.EventData) {
+			conR.broadcastHasVoteMessage(data.(*types.Vote))
+		})
 
-	// votes
-	votesCh := make(chan interface{})
-	err = conR.eventBus.Subscribe(ctx, subscriber, types.EventQueryVote, votesCh)
-	if err != nil {
-		return errors.Wrapf(err, "failed to subscribe %s to %s", subscriber, types.EventQueryVote)
-	}
-
-	// proposal heartbeats
-	heartbeatsCh := make(chan interface{})
-	err = conR.eventBus.Subscribe(ctx, subscriber, types.EventQueryProposalHeartbeat, heartbeatsCh)
-	if err != nil {
-		return errors.Wrapf(err, "failed to subscribe %s to %s", subscriber, types.EventQueryProposalHeartbeat)
-	}
-
-	go func() {
-		var data interface{}
-		var ok bool
-		for {
-			select {
-			case data, ok = <-stepsCh:
-				if ok { // a receive from a closed channel returns the zero value immediately
-					edrs := data.(types.EventDataRoundState)
-					conR.broadcastNewRoundStep(edrs.RoundState.(*cstypes.RoundState))
-				}
-			case data, ok = <-votesCh:
-				if ok {
-					edv := data.(types.EventDataVote)
-					conR.broadcastHasVoteMessage(edv.Vote)
-				}
-			case data, ok = <-heartbeatsCh:
-				if ok {
-					edph := data.(types.EventDataProposalHeartbeat)
-					conR.broadcastProposalHeartbeatMessage(edph)
-				}
-			case <-conR.Quit():
-				conR.eventBus.UnsubscribeAll(ctx, subscriber)
-				return
-			}
-			if !ok {
-				conR.eventBus.UnsubscribeAll(ctx, subscriber)
-				return
-			}
-		}
-	}()
-
-	return nil
+	conR.conS.evsw.AddListenerForEvent(subscriber, types.EventProposalHeartbeat,
+		func(data tmevents.EventData) {
+			conR.broadcastProposalHeartbeatMessage(data.(*types.Heartbeat))
+		})
 }
 
-func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(heartbeat types.EventDataProposalHeartbeat) {
-	hb := heartbeat.Heartbeat
+func (conR *ConsensusReactor) unsubscribeFromBroadcastEvents() {
+	const subscriber = "consensus-reactor"
+	conR.conS.evsw.RemoveListener(subscriber)
+}
+
+func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(hb *types.Heartbeat) {
 	conR.Logger.Debug("Broadcasting proposal heartbeat message",
 		"height", hb.Height, "round", hb.Round, "sequence", hb.Sequence)
 	msg := &ProposalHeartbeatMessage{hb}
 	conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(msg))
 }
 
-func (conR *ConsensusReactor) broadcastNewRoundStep(rs *cstypes.RoundState) {
+func (conR *ConsensusReactor) broadcastNewRoundStepMessages(rs *cstypes.RoundState) {
 	nrsMsg, csMsg := makeRoundStepMessages(rs)
 	if nrsMsg != nil {
 		conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg))

consensus/reactor_test.go

@@ -254,7 +254,8 @@ func TestReactorVotingPowerChange(t *testing.T) {
 	logger.Debug("---------------------------- Testing changing the voting power of one validator a few times")
 
 	val1PubKey := css[0].privValidator.GetPubKey()
-	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKey.Bytes(), 25)
+	val1PubKeyABCI := types.TM2PB.PubKey(val1PubKey)
+	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25)
 	previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
@@ -266,7 +267,7 @@ func TestReactorVotingPowerChange(t *testing.T) {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
 	}
 
-	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKey.Bytes(), 2)
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2)
 	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
@@ -278,7 +279,7 @@ func TestReactorVotingPowerChange(t *testing.T) {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
 	}
 
-	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKey.Bytes(), 26)
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26)
 	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
@@ -316,7 +317,8 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	logger.Info("---------------------------- Testing adding one validator")
 
 	newValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
-	newValidatorTx1 := kvstore.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), testMinPower)
+	valPubKey1ABCI := types.TM2PB.PubKey(newValidatorPubKey1)
+	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
 
 	// wait till everyone makes block 2
 	// ensure the commit includes all validators
@@ -342,7 +344,8 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	logger.Info("---------------------------- Testing changing the voting power of one validator")
 
 	updateValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
-	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25)
+	updatePubKey1ABCI := types.TM2PB.PubKey(updateValidatorPubKey1)
+	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
 	previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
@@ -358,10 +361,12 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	logger.Info("---------------------------- Testing adding two validators at once")
 
 	newValidatorPubKey2 := css[nVals+1].privValidator.GetPubKey()
-	newValidatorTx2 := kvstore.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), testMinPower)
+	newVal2ABCI := types.TM2PB.PubKey(newValidatorPubKey2)
+	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
 
 	newValidatorPubKey3 := css[nVals+2].privValidator.GetPubKey()
-	newValidatorTx3 := kvstore.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), testMinPower)
+	newVal3ABCI := types.TM2PB.PubKey(newValidatorPubKey3)
+	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
 	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
@@ -373,8 +378,8 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	//---------------------------------------------------------------------------
 	logger.Info("---------------------------- Testing removing two validators at once")
 
-	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0)
-	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0)
+	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
+	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)
 	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)

consensus/replay.go

@@ -2,7 +2,6 @@ package consensus
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -196,21 +195,21 @@ func makeHeightSearchFunc(height int64) auto.SearchFunc {
 type Handshaker struct {
 	stateDB      dbm.DB
 	initialState sm.State
-	store        types.BlockStore
-	appState     json.RawMessage
+	store        sm.BlockStore
+	genDoc       *types.GenesisDoc
 	logger       log.Logger
 
 	nBlocks int // number of blocks applied to the state
 }
 
 func NewHandshaker(stateDB dbm.DB, state sm.State,
-	store types.BlockStore, appState json.RawMessage) *Handshaker {
+	store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker {
 
 	return &Handshaker{
 		stateDB:      stateDB,
 		initialState: state,
 		store:        store,
-		appState:     appState,
+		genDoc:       genDoc,
 		logger:       log.NewNopLogger(),
 		nBlocks:      0,
 	}
@@ -268,14 +267,33 @@ func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight
 	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain
 	if appBlockHeight == 0 {
 		validators := types.TM2PB.Validators(state.Validators)
+		csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams)
 		req := abci.RequestInitChain{
-			Validators:    validators,
-			AppStateBytes: h.appState,
+			Time:            h.genDoc.GenesisTime.Unix(), // TODO
+			ChainId:         h.genDoc.ChainID,
+			ConsensusParams: csParams,
+			Validators:      validators,
+			AppStateBytes:   h.genDoc.AppStateJSON,
 		}
-		_, err := proxyApp.Consensus().InitChainSync(req)
+		res, err := proxyApp.Consensus().InitChainSync(req)
 		if err != nil {
 			return nil, err
 		}
+
+		// if the app returned validators
+		// or consensus params, update the state
+		// with the them
+		if len(res.Validators) > 0 {
+			vals, err := types.PB2TM.Validators(res.Validators)
+			if err != nil {
+				return nil, err
+			}
+			state.Validators = types.NewValidatorSet(vals)
+		}
+		if res.ConsensusParams != nil {
+			state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams)
+		}
+		sm.SaveState(h.stateDB, state)
 	}
 
 	// First handle edge cases and constraints on the storeBlockHeight
@@ -365,7 +383,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl
 	for i := appBlockHeight + 1; i <= finalBlock; i++ {
 		h.logger.Info("Applying block", "height", i)
 		block := h.store.LoadBlock(i)
-		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger)
+		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, state.LastValidators, h.stateDB)
 		if err != nil {
 			return nil, err
 		}
@@ -390,7 +408,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap
 	block := h.store.LoadBlock(height)
 	meta := h.store.LoadBlockMeta(height)
 
-	blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, types.MockMempool{}, types.MockEvidencePool{})
+	blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, sm.MockMempool{}, sm.MockEvidencePool{})
 
 	var err error
 	state, err = blockExec.ApplyBlock(state, meta.BlockID, block)

consensus/replay_file.go

@@ -299,7 +299,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 	// Create proxyAppConn connection (consensus, mempool, query)
 	clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
 	proxyApp := proxy.NewAppConns(clientCreator,
-		NewHandshaker(stateDB, state, blockStore, gdoc.AppState()))
+		NewHandshaker(stateDB, state, blockStore, gdoc))
 	err = proxyApp.Start()
 	if err != nil {
 		cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err))
@@ -310,7 +310,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 		cmn.Exit(cmn.Fmt("Failed to start event bus: %v", err))
 	}
 
-	mempool, evpool := types.MockMempool{}, types.MockEvidencePool{}
+	mempool, evpool := sm.MockMempool{}, sm.MockEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 
 	consensusState := NewConsensusState(csConfig, state.Copy(), blockExec,

consensus/replay_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/tendermint/abci/example/kvstore"
@@ -23,10 +24,10 @@ import (
 	dbm "github.com/tendermint/tmlibs/db"
 
 	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
 	"github.com/tendermint/tmlibs/log"
 )
 
@@ -263,8 +264,8 @@ const (
 )
 
 var (
-	mempool = types.MockMempool{}
-	evpool  = types.MockEvidencePool{}
+	mempool = sm.MockMempool{}
+	evpool  = sm.MockEvidencePool{}
 )
 
 //---------------------------------------
@@ -329,7 +330,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	walFile := tempWALWithData(walBody)
 	config.Consensus.SetWalFile(walFile)
 
-	privVal := pvm.LoadFilePV(config.PrivValidatorFile())
+	privVal := privval.LoadFilePV(config.PrivValidatorFile())
 
 	wal, err := NewWAL(walFile)
 	if err != nil {
@@ -366,7 +367,8 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	}
 
 	// now start the app using the handshake - it should sync
-	handshaker := NewHandshaker(stateDB, state, store, nil)
+	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	handshaker := NewHandshaker(stateDB, state, store, genDoc)
 	proxyApp := proxy.NewAppConns(clientCreator2, handshaker)
 	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
@@ -416,10 +418,10 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB,
 	}
 	defer proxyApp.Stop()
 
-	// TODO: get the genesis bytes (https://github.com/tendermint/tendermint/issues/1224)
-	var genesisBytes []byte
 	validators := types.TM2PB.Validators(state.Validators)
-	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators, genesisBytes}); err != nil {
+	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{
+		Validators: validators,
+	}); err != nil {
 		panic(err)
 	}
 
@@ -453,10 +455,10 @@ func buildTMStateFromChain(config *cfg.Config, stateDB dbm.DB, state sm.State, c
 	}
 	defer proxyApp.Stop()
 
-	// TODO: get the genesis bytes (https://github.com/tendermint/tendermint/issues/1224)
-	var genesisBytes []byte
 	validators := types.TM2PB.Validators(state.Validators)
-	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators, genesisBytes}); err != nil {
+	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{
+		Validators: validators,
+	}); err != nil {
 		panic(err)
 	}
 
@@ -633,3 +635,53 @@ func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit {
 func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit {
 	return bs.commits[height-1]
 }
+
+//----------------------------------------
+
+func TestInitChainUpdateValidators(t *testing.T) {
+	val, _ := types.RandValidator(true, 10)
+	vals := types.NewValidatorSet([]*types.Validator{val})
+	app := &initChainApp{vals: types.TM2PB.Validators(vals)}
+	clientCreator := proxy.NewLocalClientCreator(app)
+
+	config := ResetConfig("proxy_test_")
+	privVal := privval.LoadFilePV(config.PrivValidatorFile())
+	stateDB, state, store := stateAndStore(config, privVal.GetPubKey())
+
+	oldValAddr := state.Validators.Validators[0].Address
+
+	// now start the app using the handshake - it should sync
+	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	handshaker := NewHandshaker(stateDB, state, store, genDoc)
+	proxyApp := proxy.NewAppConns(clientCreator, handshaker)
+	if err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+	defer proxyApp.Stop()
+
+	// reload the state, check the validator set was updated
+	state = sm.LoadState(stateDB)
+
+	newValAddr := state.Validators.Validators[0].Address
+	expectValAddr := val.Address
+	assert.NotEqual(t, oldValAddr, newValAddr)
+	assert.Equal(t, newValAddr, expectValAddr)
+}
+
+func newInitChainApp(vals []abci.Validator) *initChainApp {
+	return &initChainApp{
+		vals: vals,
+	}
+}
+
+// returns the vals on InitChain
+type initChainApp struct {
+	abci.BaseApplication
+	vals []abci.Validator
+}
+
+func (ica *initChainApp) InitChain(req abci.RequestInitChain) abci.ResponseInitChain {
+	return abci.ResponseInitChain{
+		Validators: ica.vals,
+	}
+}

consensus/state.go

@@ -15,6 +15,7 @@ import (
 
 	cfg "github.com/tendermint/tendermint/config"
 	cstypes "github.com/tendermint/tendermint/consensus/types"
+	tmevents "github.com/tendermint/tendermint/libs/events"
 	"github.com/tendermint/tendermint/p2p"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -75,9 +76,9 @@ type ConsensusState struct {
 	// services for creating and executing blocks
 	// TODO: encapsulate all of this in one "BlockManager"
 	blockExec  *sm.BlockExecutor
-	blockStore types.BlockStore
-	mempool    types.Mempool
-	evpool     types.EvidencePool
+	blockStore sm.BlockStore
+	mempool    sm.Mempool
+	evpool     sm.EvidencePool
 
 	// internal state
 	mtx sync.Mutex
@@ -110,10 +111,14 @@ type ConsensusState struct {
 
 	// closed when we finish shutting down
 	done chan struct{}
+
+	// synchronous pubsub between consensus state and reactor.
+	// state only emits EventNewRoundStep, EventVote and EventProposalHeartbeat
+	evsw tmevents.EventSwitch
 }
 
 // NewConsensusState returns a new ConsensusState.
-func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *sm.BlockExecutor, blockStore types.BlockStore, mempool types.Mempool, evpool types.EvidencePool) *ConsensusState {
+func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *sm.BlockExecutor, blockStore sm.BlockStore, mempool sm.Mempool, evpool sm.EvidencePool) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
 		blockExec:        blockExec,
@@ -126,6 +131,7 @@ func NewConsensusState(config *cfg.ConsensusConfig, state sm.State, blockExec *s
 		doWALCatchup:     true,
 		wal:              nilWAL{},
 		evpool:           evpool,
+		evsw:             tmevents.NewEventSwitch(),
 	}
 	// set function defaults (may be overwritten before calling Start)
 	cs.decideProposal = cs.defaultDecideProposal
@@ -227,6 +233,10 @@ func (cs *ConsensusState) LoadCommit(height int64) *types.Commit {
 // OnStart implements cmn.Service.
 // It loads the latest state via the WAL, and starts the timeout and receive routines.
 func (cs *ConsensusState) OnStart() error {
+	if err := cs.evsw.Start(); err != nil {
+		return err
+	}
+
 	// we may set the WAL in testing before calling Start,
 	// so only OpenWAL if its still the nilWAL
 	if _, ok := cs.wal.(nilWAL); ok {
@@ -244,8 +254,7 @@ func (cs *ConsensusState) OnStart() error {
 	// NOTE: we will get a build up of garbage go routines
 	// firing on the tockChan until the receiveRoutine is started
 	// to deal with them (by that point, at most one will be valid)
-	err := cs.timeoutTicker.Start()
-	if err != nil {
+	if err := cs.timeoutTicker.Start(); err != nil {
 		return err
 	}
 
@@ -284,6 +293,8 @@ func (cs *ConsensusState) startRoutines(maxSteps int) {
 func (cs *ConsensusState) OnStop() {
 	cs.BaseService.OnStop()
 
+	cs.evsw.Stop()
+
 	cs.timeoutTicker.Stop()
 
 	// Make BaseService.Wait() wait until cs.wal.Wait()
@@ -509,6 +520,7 @@ func (cs *ConsensusState) newStep() {
 	// newStep is called by updateToStep in NewConsensusState before the eventBus is set!
 	if cs.eventBus != nil {
 		cs.eventBus.PublishEventNewRoundStep(rs)
+		cs.evsw.FireEvent(types.EventNewRoundStep, &cs.RoundState)
 	}
 }
 
@@ -752,6 +764,7 @@ func (cs *ConsensusState) proposalHeartbeat(height int64, round int) {
 		}
 		cs.privValidator.SignHeartbeat(chainID, heartbeat)
 		cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat})
+		cs.evsw.FireEvent(types.EventProposalHeartbeat, heartbeat)
 		counter++
 		time.Sleep(proposalHeartbeatIntervalSeconds * time.Second)
 	}
@@ -1418,6 +1431,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool,
 
 		cs.Logger.Info(cmn.Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
 		cs.eventBus.PublishEventVote(types.EventDataVote{vote})
+		cs.evsw.FireEvent(types.EventVote, vote)
 
 		// if we can skip timeoutCommit and have all the votes now,
 		if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() {
@@ -1445,6 +1459,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool,
 	}
 
 	cs.eventBus.PublishEventVote(types.EventDataVote{vote})
+	cs.evsw.FireEvent(types.EventVote, vote)
 
 	switch vote.Type {
 	case types.VoteTypePrevote:

consensus/state_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
-	tmpubsub "github.com/tendermint/tmlibs/pubsub"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 )
 
 func init() {

consensus/wal.go

@@ -106,12 +106,12 @@ func (wal *baseWAL) OnStart() error {
 }
 
 func (wal *baseWAL) OnStop() {
-	wal.BaseService.OnStop()
 	wal.group.Stop()
+	wal.group.Close()
 }
 
 // Write is called in newStep and for each receive on the
-// peerMsgQueue and the timoutTicker.
+// peerMsgQueue and the timeoutTicker.
 // NOTE: does not call fsync()
 func (wal *baseWAL) Write(msg WALMessage) {
 	if wal == nil {
@@ -144,13 +144,14 @@ type WALSearchOptions struct {
 	IgnoreDataCorruptionErrors bool
 }
 
-// SearchForEndHeight searches for the EndHeightMessage with the height and
-// returns an auto.GroupReader, whenever it was found or not and an error.
+// SearchForEndHeight searches for the EndHeightMessage with the given height
+// and returns an auto.GroupReader, whenever it was found or not and an error.
 // Group reader will be nil if found equals false.
 //
 // CONTRACT: caller must close group reader.
 func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
 	var msg *TimedWALMessage
+	lastHeightFound := int64(-1)
 
 	// NOTE: starting from the last file in the group because we're usually
 	// searching for the last height. See replay.go
@@ -166,17 +167,25 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions)
 		for {
 			msg, err = dec.Decode()
 			if err == io.EOF {
+				// OPTIMISATION: no need to look for height in older files if we've seen h < height
+				if lastHeightFound > 0 && lastHeightFound < height {
+					gr.Close()
+					return nil, false, nil
+				}
 				// check next file
 				break
 			}
 			if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) {
+				wal.Logger.Debug("Corrupted entry. Skipping...", "err", err)
 				// do nothing
+				continue
 			} else if err != nil {
 				gr.Close()
 				return nil, false, err
 			}
 
 			if m, ok := msg.Msg.(EndHeightMessage); ok {
+				lastHeightFound = m.Height
 				if m.Height == height { // found
 					wal.Logger.Debug("Found", "height", height, "index", index)
 					return gr, true, nil
@@ -271,23 +280,17 @@ func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
 
 	b = make([]byte, 4)
 	_, err = dec.rd.Read(b)
-	if err == io.EOF {
-		return nil, err
-	}
 	if err != nil {
 		return nil, fmt.Errorf("failed to read length: %v", err)
 	}
 	length := binary.BigEndian.Uint32(b)
 
 	if length > maxMsgSizeBytes {
-		return nil, DataCorruptionError{fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes)}
+		return nil, fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes)
 	}
 
 	data := make([]byte, length)
 	_, err = dec.rd.Read(data)
-	if err == io.EOF {
-		return nil, err
-	}
 	if err != nil {
 		return nil, fmt.Errorf("failed to read data: %v", err)
 	}

consensus/wal_generator.go

@@ -13,10 +13,10 @@ import (
 	"github.com/tendermint/abci/example/kvstore"
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
 	auto "github.com/tendermint/tmlibs/autofile"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/db"
@@ -40,7 +40,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) {
 	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
 	// NOTE: we can't import node package because of circular dependency
 	privValidatorFile := config.PrivValidatorFile()
-	privValidator := pvm.LoadOrGenFilePV(privValidatorFile)
+	privValidator := privval.LoadOrGenFilePV(privValidatorFile)
 	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read genesis file")
@@ -52,7 +52,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) {
 		return nil, errors.Wrap(err, "failed to make genesis state")
 	}
 	blockStore := bc.NewBlockStore(blockStoreDB)
-	handshaker := NewHandshaker(stateDB, state, blockStore, genDoc.AppState())
+	handshaker := NewHandshaker(stateDB, state, blockStore, genDoc)
 	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), handshaker)
 	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if err := proxyApp.Start(); err != nil {
@@ -65,8 +65,8 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) {
 		return nil, errors.Wrap(err, "failed to start event bus")
 	}
 	defer eventBus.Stop()
-	mempool := types.MockMempool{}
-	evpool := types.MockEvidencePool{}
+	mempool := sm.MockMempool{}
+	evpool := sm.MockEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 	consensusState := NewConsensusState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
 	consensusState.SetLogger(logger)

docs/abci-cli.rst

@@ -183,6 +183,7 @@ Try running these commands:
     
     > commit
     -> code: OK
+    -> data.hex: 0x0000000000000000
     
     > deliver_tx "abc"
     -> code: OK
@@ -194,7 +195,7 @@ Try running these commands:
     
     > commit
     -> code: OK
-    -> data.hex: 0x49DFD15CCDACDEAE9728CB01FBB5E8688CA58B91
+    -> data.hex: 0x0200000000000000
     
     > query "abc"
     -> code: OK
@@ -208,7 +209,7 @@ Try running these commands:
     
     > commit
     -> code: OK
-    -> data.hex: 0x70102DB32280373FBF3F9F89DA2A20CE2CD62B0B
+    -> data.hex: 0x0400000000000000
     
     > query "def"
     -> code: OK
@@ -301,6 +302,7 @@ In another window, start the ``abci-cli console``:
 
     > set_option serial on
     -> code: OK
+    -> log: OK (SetOption doesn't return anything.)
     
     > check_tx 0x00
     -> code: OK

docs/app-architecture.rst

@@ -1,133 +1,42 @@
 Application Architecture Guide
 ==============================
 
-Overview
---------
+Here we provide a brief guide on the recommended architecture of a Tendermint blockchain
+application.
 
-A blockchain application is more than the consensus engine and the
-transaction logic (eg. smart contracts, business logic) as implemented
-in the ABCI app. There are also (mobile, web, desktop) clients that will
-need to connect and make use of the app. We will assume for now that you
-have a well designed transactions and database model, but maybe this
-will be the topic of another article. This article is more interested in
-various ways of setting up the "plumbing" and connecting these pieces,
-and demonstrating some evolving best practices.
+The following diagram provides a superb example:
 
-Security
---------
+https://drive.google.com/open?id=1yR2XpRi9YCY9H9uMfcw8-RMJpvDyvjz9
 
-A very important aspect when constructing a blockchain is security. The
-consensus model can be DoSed (no consensus possible) by corrupting 1/3
-of the validators and exploited (writing arbitrary blocks) by corrupting
-2/3 of the validators. So, while the security is not that of the
-"weakest link", you should take care that the "average link" is
-sufficiently hardened.
+The end-user application here is the Cosmos Voyager, at the bottom left.
+Voyager communicates with a REST API exposed by a local Light-Client Daemon.
+The Light-Client Daemon is an application specific program that communicates with
+Tendermint nodes and verifies Tendermint light-client proofs through the Tendermint Core RPC.
+The Tendermint Core process communicates with a local ABCI application, where the
+user query or transaction is actually processed.
 
-One big attack surface on the validators is the communication between
-the ABCI app and the tendermint core. This should be highly protected.
-Ideally, the app and the core are running on the same machine, so no
-external agent can target the communication channel. You can use unix
-sockets (with permissions preventing access from other users), or even
-compile the two apps into one binary if the ABCI app is also writen in
-go. If you are unable to do that due to language support, then the ABCI
-app should bind a TCP connection to localhost (127.0.0.1), which is less
-efficient and secure, but still not reachable from outside. If you must
-run the ABCI app and tendermint core on separate machines, make sure you
-have a secure communication channel (ssh tunnel?)
+The ABCI application must be a deterministic result of the Tendermint consensus - any external influence
+on the application state that didn't come through Tendermint could cause a
+consensus failure. Thus *nothing* should communicate with the application except Tendermint via ABCI.
 
-Now assuming, you have linked together your app and the core securely,
-you must also make sure no one can get on the machine it is hosted on.
-At this point it is basic network security. Run on a secure operating
-system (SELinux?). Limit who has access to the machine (user accounts,
-but also where the physical machine is hosted). Turn off all services
-except for ssh, which should only be accessible by some well-guarded
-public/private key pairs (no password). And maybe even firewall off
-access to the ports used by the validators, so only known validators can
-connect.
+If the application is written in Go, it can be compiled into the Tendermint binary.
+Otherwise, it should use a unix socket to communicate with Tendermint.
+If it's necessary to use TCP, extra care must be taken to encrypt and authenticate the connection.
 
-There was also a suggestion on slack from @jhon about compiling
-everything together with a unikernel for more security, such as
-`Mirage <https://mirage.io>`__ or
-`UNIK <https://github.com/emc-advanced-dev/unik>`__.
+All reads from the app happen through the Tendermint `/abci_query` endpoint.
+All writes to the app happen through the Tendermint `/broadcast_tx_*` endpoints.
 
-Connecting your client to the blockchain
-----------------------------------------
+The Light-Client Daemon is what provides light clients (end users) with nearly all the security of a full node.
+It formats and broadcasts transactions, and verifies proofs of queries and transaction results.
+Note that it need not be a daemon - the Light-Client logic could instead be implemented in the same process as the end-user application.
 
-Tendermint Core RPC
-~~~~~~~~~~~~~~~~~~~
+Note for those ABCI applications with weaker security requirements, the functionality of the Light-Client Daemon can be moved
+into the ABCI application process itself. That said, exposing the application process to anything besides Tendermint over ABCI
+requires extreme caution, as all transactions, and possibly all queries, should still pass through Tendermint.
 
-The concept is that the ABCI app is completely hidden from the outside
-world and only communicated through a tested and secured `interface
-exposed by the tendermint core <./specification/rpc.html>`__. This interface
-exposes a lot of data on the block header and consensus process, which
-is quite useful for externally verifying the system. It also includes
-3(!) methods to broadcast a transaction (propose it for the blockchain,
-and possibly await a response). And one method to query app-specific
-data from the ABCI application.
-
-Pros:
-
-- Server code already written
-- Access to block headers to validate merkle proofs (nice for light clients)
-- Basic read/write functionality is supported
-
-Cons:
-
-- Limited interface to app. All queries must be serialized into []byte (less expressive than JSON over HTTP) and there is no way to push data from ABCI app to the client (eg. notify me if account X receives a transaction)
-
-Custom ABCI server
-~~~~~~~~~~~~~~~~~~
-
-This was proposed by @wolfposd on slack and demonstrated by
-`TMChat <https://github.com/wolfposd/TMChat>`__, a sample app. The
-concept is to write a custom server for your app (with typical REST
-API/websockets/etc for easy use by a mobile app). This custom server is
-in the same binary as the ABCI app and data store, so can easily react
-to complex events there that involve understanding the data format (send
-a message if my balance drops below 500). All "writes" sent to this
-server are proxied via websocket/JSON-RPC to tendermint core. When they
-come back as deliver\_tx over ABCI, they will be written to the data
-store. For "reads", we can do any queries we wish that are supported by
-our architecture, using any web technology that is useful. The general
-architecture is shown in the following diagram:
-
-.. figure:: assets/tm-application-example.png
-
-Pros:
-
-- Separates application logic from blockchain logic
-- Allows much richer, more flexible client-facing API
-- Allows pub-sub, watching certain fields, etc.
-
-Cons:
-
-- Access to ABCI app can be dangerous (be VERY careful not to write unless it comes from the validator node)
-- No direct access to the blockchain headers to verify tx
-- You must write your own API (but maybe that's a pro...)
-
-Hybrid solutions
-~~~~~~~~~~~~~~~~
-
-Likely the least secure but most versatile. The client can access both
-the tendermint node for all blockchain info, as well as a custom app
-server, for complex queries and pub-sub on the abci app.
-
-Pros:
-
-- All from both above solutions
-
-Cons:
-
-- Even more complexity; even more attack vectors (less
-security)
-
-Scalability
------------
-
-Read replica using non-validating nodes? They could forward transactions
-to the validators (fewer connections, more security), and locally allow
-all queries in any of the above configurations. Thus, while
-transaction-processing speed is limited by the speed of the abci app and
-the number of validators, one should be able to scale our read
-performance to quite an extent (until the replication process drains too
-many resources from the validator nodes).
+See the following for more extensive documentation:
+- [Interchain Standard for the Light-Client REST API](https://github.com/cosmos/cosmos-sdk/pull/1028)
+- [Tendermint RPC Docs](https://tendermint.github.io/slate/)
+- [Tendermint in Production](https://github.com/tendermint/tendermint/pull/1618)
+- [Tendermint Basics](https://tendermint.readthedocs.io/en/master/using-tendermint.html)
+- [ABCI spec](https://github.com/tendermint/abci/blob/master/specification.rst)

docs/examples/node0/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/examples/node1/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/examples/node2/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/examples/node3/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/index.rst

@@ -55,7 +55,10 @@ Tendermint 102
    abci-spec.rst
    app-architecture.rst
    app-development.rst
+   subscribing-to-events-via-websocket.rst
+   indexing-transactions.rst
    how-to-read-logs.rst
+   running-in-production.rst
 
 Tendermint 201
 --------------

docs/indexing-transactions.rst

@@ -0,0 +1,100 @@
+Indexing Transactions
+=====================
+
+Tendermint allows you to index transactions and later query or subscribe to
+their results.
+
+Let's take a look at the ``[tx_index]`` config section:
+
+::
+
+    ##### transactions indexer configuration options #####
+    [tx_index]
+
+    # What indexer to use for transactions
+    #
+    # Options:
+    #   1) "null" (default)
+    #   2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
+    indexer = "kv"
+
+    # Comma-separated list of tags to index (by default the only tag is tx hash)
+    #
+    # It's recommended to index only a subset of tags due to possible memory
+    # bloat. This is, of course, depends on the indexer's DB and the volume of
+    # transactions.
+    index_tags = ""
+
+    # When set to true, tells indexer to index all tags. Note this may be not
+    # desirable (see the comment above). IndexTags has a precedence over
+    # IndexAllTags (i.e. when given both, IndexTags will be indexed).
+    index_all_tags = false
+
+By default, Tendermint will index all transactions by their respective hashes
+using an embedded simple indexer. Note, we are planning to add more options in
+the future (e.g., Postgresql indexer).
+
+Adding tags
+-----------
+
+In your application's ``DeliverTx`` method, add the ``Tags`` field with the
+pairs of UTF-8 encoded strings (e.g. "account.owner": "Bob", "balance":
+"100.0", "date": "2018-01-02").
+
+Example:
+
+::
+
+    func (app *KVStoreApplication) DeliverTx(tx []byte) types.Result {
+        ...
+        tags := []cmn.KVPair{
+          {[]byte("account.name"), []byte("igor")},
+          {[]byte("account.address"), []byte("0xdeadbeef")},
+          {[]byte("tx.amount"), []byte("7")},
+        }
+        return types.ResponseDeliverTx{Code: code.CodeTypeOK, Tags: tags}
+    }
+
+If you want Tendermint to only index transactions by "account.name" tag, in the
+config set ``tx_index.index_tags="account.name"``. If you to index all tags,
+set ``index_all_tags=true``
+
+Note, there are a few predefined tags:
+
+- ``tm.event`` (event type)
+- ``tx.hash`` (transaction's hash)
+- ``tx.height`` (height of the block transaction was committed in)
+
+Tendermint will throw a warning if you try to use any of the above keys.
+
+Quering transactions
+--------------------
+
+You can query the transaction results by calling ``/tx_search`` RPC endpoint:
+
+::
+
+    curl "localhost:46657/tx_search?query=\"account.name='igor'\"&prove=true"
+
+Check out `API docs <https://tendermint.github.io/slate/?shell#txsearch>`__ for more
+information on query syntax and other options.
+
+Subscribing to transactions
+---------------------------
+
+Clients can subscribe to transactions with the given tags via Websocket by
+providing a query to ``/subscribe`` RPC endpoint.
+
+::
+
+    {
+        "jsonrpc": "2.0",
+        "method": "subscribe",
+        "id": "0",
+        "params": {
+            "query": "account.name='igor'"
+        }
+    }
+
+Check out `API docs <https://tendermint.github.io/slate/#subscribe>`__ for more
+information on query syntax and other options.

docs/install.rst

@@ -1,6 +1,13 @@
 Install Tendermint
 ==================
 
+The fastest and easiest way to install the ``tendermint`` binary
+is to run `this script <https://github.com/tendermint/tendermint/blob/develop/scripts/install/install_tendermint_ubuntu.sh>`__ on
+a fresh Ubuntu instance,
+or `this script <https://github.com/tendermint/tendermint/blob/develop/scripts/install/install_tendermint_bsd.sh>`__
+on a fresh FreeBSD instance. Read the comments / instructions carefully (i.e., reset your terminal after running the script,
+make sure your okay with the network connections being made).
+
 From Binary
 -----------
 

docs/running-in-production.rst

@@ -0,0 +1,203 @@
+Running in production
+=====================
+
+Logging
+-------
+
+Default logging level (``main:info,state:info,*:``) should suffice for normal
+operation mode. Read `this post
+<https://blog.cosmos.network/one-of-the-exciting-new-features-in-0-10-0-release-is-smart-log-level-flag-e2506b4ab756>`__
+for details on how to configure ``log_level`` config variable. Some of the
+modules can be found `here <./how-to-read-logs.html#list-of-modules>`__. If
+you're trying to debug Tendermint or asked to provide logs with debug logging
+level, you can do so by running tendermint with ``--log_level="*:debug"``.
+
+DOS Exposure and Mitigation
+---------------------------
+
+Validators are supposed to setup `Sentry Node Architecture
+<https://blog.cosmos.network/tendermint-explained-bringing-bft-based-pos-to-the-public-blockchain-domain-f22e274a0fdb>`__
+to prevent Denial-of-service attacks. You can read more about it `here
+<https://github.com/tendermint/aib-data/blob/develop/medium/TendermintBFT.md>`__.
+
+P2P
+~~~
+
+The core of the Tendermint peer-to-peer system is ``MConnection``. Each
+connection has ``MaxPacketMsgPayloadSize``, which is the maximum packet size
+and bounded send & receive queues. One can impose restrictions on send &
+receive rate per connection (``SendRate``, ``RecvRate``).
+
+RPC
+~~~
+
+Endpoints returning multiple entries are limited by default to return 30
+elements (100 max).
+
+Rate-limiting and authentication are another key aspects to help protect
+against DOS attacks. While in the future we may implement these features, for
+now, validators are supposed to use external tools like `NGINX
+<https://www.nginx.com/blog/rate-limiting-nginx/>`__ or `traefik
+<https://docs.traefik.io/configuration/commons/#rate-limiting>`__ to achieve
+the same things.
+
+Debugging Tendermint
+--------------------
+
+If you ever have to debug Tendermint, the first thing you should probably do is
+to check out the logs. See `"How to read logs" <./how-to-read-logs.html>`__,
+where we explain what certain log statements mean.
+
+If, after skimming through the logs, things are not clear still, the second
+TODO is to query the `/status` RPC endpoint. It provides the necessary info:
+whenever the node is syncing or not, what height it is on, etc.
+
+```
+$ curl http(s)://{ip}:{rpcPort}/status
+```
+
+`/dump_consensus_state` will give you a detailed overview of the consensus
+state (proposer, lastest validators, peers states). From it, you should be able
+to figure out why, for example, the network had halted.
+
+```
+$ curl http(s)://{ip}:{rpcPort}/dump_consensus_state
+```
+
+There is a reduced version of this endpoint - `/consensus_state`, which
+returns just the votes seen at the current height.
+
+- `Github Issues <https://github.com/tendermint/tendermint/issues>`__
+- `StackOverflow questions <https://stackoverflow.com/questions/tagged/tendermint>`__
+
+Monitoring Tendermint
+---------------------
+
+Each Tendermint instance has a standard `/health` RPC endpoint, which responds
+with 200 (OK) if everything is fine and 500 (or no response) - if something is
+wrong.
+
+Other useful endpoints include mentioned earlier `/status`, `/net_info` and
+`/validators`.
+
+We have a small tool, called tm-monitor, which outputs information from the
+endpoints above plus some statistics. The tool can be found `here
+<https://github.com/tendermint/tools/tree/master/tm-monitor>`__.
+
+What happens when my app dies?
+------------------------------
+
+You are supposed to run Tendermint under a `process supervisor
+<https://en.wikipedia.org/wiki/Process_supervision>`__ (like systemd or runit).
+It will ensure Tendermint is always running (despite possible errors).
+
+Getting back to the original question, if your application dies, Tendermint
+will panic. After a process supervisor restarts your application, Tendermint
+should be able to reconnect successfully. The order of restart does not matter
+for it.
+
+Signal handling
+---------------
+
+We catch SIGINT and SIGTERM and try to clean up nicely. For other signals we
+use the default behaviour in Go: `Default behavior of signals in Go programs
+<https://golang.org/pkg/os/signal/#hdr-Default_behavior_of_signals_in_Go_programs>`__.
+
+Hardware
+--------
+
+Processor and Memory
+~~~~~~~~~~~~~~~~~~~~
+
+While actual specs vary depending on the load and validators count, minimal requirements are:
+
+- 1GB RAM
+- 25GB of disk space
+- 1.4 GHz CPU
+
+SSD disks are preferable for applications with high transaction throughput.
+
+Recommended:
+
+- 2GB RAM
+- 100GB SSD
+- x64 2.0 GHz 2v CPU
+
+While for now, Tendermint stores all the history and it may require significant
+disk space over time, we are planning to implement state syncing (See `#828
+<https://github.com/tendermint/tendermint/issues/828>`__). So, storing all the
+past blocks will not be necessary.
+
+Operating Systems
+~~~~~~~~~~~~~~~~~
+
+Tendermint can be compiled for a wide range of operating systems thanks to Go
+language (the list of $OS/$ARCH pairs can be found `here
+<https://golang.org/doc/install/source#environment>`__).
+
+While we do not favor any operation system, more secure and stable Linux server
+distributions (like Centos) should be preferred over desktop operation systems
+(like Mac OS).
+
+Misc.
+~~~~~
+
+NOTE: if you are going to use Tendermint in a public domain, make sure you read
+`hardware recommendations (see "4. Hardware")
+<https://cosmos.network/validators>`__ for a validator in the Cosmos network.
+
+Configuration parameters
+------------------------
+
+- ``p2p.flush_throttle_timeout``
+  ``p2p.max_packet_msg_payload_size``
+  ``p2p.send_rate``
+  ``p2p.recv_rate``
+
+If you are going to use Tendermint in a private domain and you have a private
+high-speed network among your peers, it makes sense to lower flush throttle
+timeout and increase other params.
+
+::
+
+    [p2p]
+
+    send_rate=20000000 # 2MB/s
+    recv_rate=20000000 # 2MB/s
+    flush_throttle_timeout=10
+    max_packet_msg_payload_size=10240 # 10KB
+
+- ``mempool.recheck``
+
+After every block, Tendermint rechecks every transaction left in the mempool to
+see if transactions committed in that block affected the application state, so
+some of the transactions left may become invalid. If that does not apply to
+your application, you can disable it by setting ``mempool.recheck=false``.
+
+- ``mempool.broadcast``
+
+Setting this to false will stop the mempool from relaying transactions to other
+peers until they are included in a block. It means only the peer you send the
+tx to will see it until it is included in a block.
+
+- ``consensus.skip_timeout_commit``
+
+We want skip_timeout_commit=false when there is economics on the line because
+proposers should wait to hear for more votes. But if you don't care about that
+and want the fastest consensus, you can skip it. It will be kept false by
+default for public deployments (e.g. `Cosmos Hub
+<https://cosmos.network/intro/hub>`__) while for enterprise applications,
+setting it to true is not a problem.
+
+- ``consensus.peer_gossip_sleep_duration``
+
+You can try to reduce the time your node sleeps before checking if theres something to send its peers.
+
+- ``consensus.timeout_commit``
+
+You can also try lowering ``timeout_commit`` (time we sleep before proposing the next block).
+
+- ``consensus.max_block_size_txs``
+
+By default, the maximum number of transactions per a block is 10_000. Feel free
+to change it to suit your needs.

docs/spec/README.md

@@ -14,9 +14,9 @@ please submit them to our [bug bounty](https://tendermint.com/security)!
 
 ### Data Structures
 
-- [Encoding and Digests](./blockchain/encoding.md)
-- [Blockchain](./blockchain/blockchain.md)
-- [State](./blockchain/state.md)
+- [Encoding and Digests](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/encoding.md)
+- [Blockchain](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/blockchain.md)
+- [State](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/state.md)
 
 ### Consensus Protocol
 
@@ -24,11 +24,11 @@ please submit them to our [bug bounty](https://tendermint.com/security)!
 
 ### P2P and Network Protocols
 
-- [The Base P2P Layer](p2p): multiplex the protocols ("reactors") on authenticated and encrypted TCP connections
-- [Peer Exchange (PEX)](reactors/pex): gossip known peer addresses so peers can find each other
-- [Block Sync](reactors/block_sync): gossip blocks so peers can catch up quickly
-- [Consensus](reactors/consensus): gossip votes and block parts so new blocks can be committed
-- [Mempool](reactors/mempool): gossip transactions so they get included in blocks
+- [The Base P2P Layer](https://github.com/tendermint/tendermint/tree/master/docs/spec/p2p): multiplex the protocols ("reactors") on authenticated and encrypted TCP connections
+- [Peer Exchange (PEX)](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/pex): gossip known peer addresses so peers can find each other
+- [Block Sync](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/block_sync): gossip blocks so peers can catch up quickly
+- [Consensus](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/consensus): gossip votes and block parts so new blocks can be committed
+- [Mempool](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/mempool): gossip transactions so they get included in blocks
 - Evidence: TODO
 
 ### More

docs/spec/blockchain/blockchain.md

@@ -162,7 +162,7 @@ We refer to certain globally available objects:
 and `state` keeps track of the validator set, the consensus parameters
 and other results from the application.
 Elements of an object are accessed as expected,
-ie. `block.Header`. See [here](state.md) for the definition of `state`.
+ie. `block.Header`. See [here](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/state.md) for the definition of `state`.
 
 ### Header
 

docs/spec/blockchain/encoding.md

@@ -2,7 +2,7 @@
 
 ## Amino
 
-Tendermint uses the Protobuf3 derrivative [Amino]() for all data structures.
+Tendermint uses the Protobuf3 derivative [Amino](https://github.com/tendermint/go-amino) for all data structures.
 Think of Amino as an object-oriented Protobuf3 with native JSON support.
 The goal of the Amino encoding protocol is to bring parity between application
 logic objects and persistence objects.
@@ -39,7 +39,7 @@ place of the public key. Here we list the concrete types, their names,
 and prefix bytes for public keys and signatures, as well as the address schemes
 for each PubKey. Note for brevity we don't
 include details of the private keys beyond their type and name, as they can be
-derrived the same way as the others using Amino.
+derived the same way as the others using Amino.
 
 All registered objects are encoded by Amino using a 4-byte PrefixBytes that
 uniquely identifies the object and includes information about its underlying
@@ -49,107 +49,60 @@ spec](https://github.com/tendermint/go-amino#computing-the-prefix-and-disambigua
 In what follows, we provide the type names and prefix bytes directly.
 Notice that when encoding byte-arrays, the length of the byte-array is appended
 to the PrefixBytes. Thus the encoding of a byte array becomes `<PrefixBytes>
-<Length> <ByteArray>`
+<Length> <ByteArray>`. In other words, to encode any type listed below you do not need to be
+familiar with amino encoding.
+You can simply use below table and concatenate Prefix || Length (of raw bytes) || raw bytes
+( while || stands for byte concatenation here).
 
-(NOTE: the remainder of this section on Public Key Cryptography can be generated
-from [this script](./scripts/crypto.go))
+| Type | Name | Prefix | Length |
+| ---- | ---- | ------ | ----- |
+| PubKeyEd25519 | tendermint/PubKeyEd25519 | 0x1624DE62 | 0x20 |
+| PubKeyLedgerEd25519 | tendermint/PubKeyLedgerEd25519 | 0x5C3453B2 | 0x20 |
+| PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE982 | 0x21 |
+| PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288912 | 0x40 |
+| PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79A | 0x20 |
+| PrivKeyLedgerSecp256k1 | tendermint/PrivKeyLedgerSecp256k1 | 0x10CAB393 | variable |
+| PrivKeyLedgerEd25519 | tendermint/PrivKeyLedgerEd25519 | 0x0CFEEF9B | variable |
+| SignatureEd25519 | tendermint/SignatureKeyEd25519 | 0x3DA1DB2A | 0x40 |
+| SignatureSecp256k1 | tendermint/SignatureKeySecp256k1 | 0x16E1FEEA | variable |
 
-### PubKeyEd25519
+### Examples
 
-```
-// Name: tendermint/PubKeyEd25519
-// PrefixBytes: 0x1624DE62
-// Length: 0x20
-// Notes: raw 32-byte Ed25519 pubkey
-type PubKeyEd25519 [32]byte
-
-func (pubkey PubKeyEd25519) Address() []byte {
-      // NOTE: hash of the Amino encoded bytes!
-        return RIPEMD160(AminoEncode(pubkey))
-}
-```
-
-For example, the 32-byte Ed25519 pubkey
-`CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E` would be
-encoded as
-`1624DE6220CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E`.
-
-The address would then be
-`RIPEMD160(0x1624DE6220CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E)`
-or `430FF75BAF1EC4B0D51BB3EEC2955479D0071605`
-
-### SignatureEd25519
-
-```
-// Name: tendermint/SignatureKeyEd25519
-// PrefixBytes: 0x3DA1DB2A
-// Length: 0x40
-// Notes: raw 64-byte Ed25519 signature
-type SignatureEd25519 [64]byte
-```
-
-For example, the 64-byte Ed25519 signature
-`1B6034A8ED149D3C94FDA13EC03B26CC0FB264D9B0E47D3FA3DEF9FCDE658E49C80B35F9BE74949356401B15B18FB817D6E54495AD1C4A8401B248466CB0DB0B`
+1. For example, the 33-byte (or 0x21-byte in hex) Secp256k1 pubkey
+`020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9`
 would be encoded as
-`3DA1DB2A401B6034A8ED149D3C94FDA13EC03B26CC0FB264D9B0E47D3FA3DEF9FCDE658E49C80B35F9BE74949356401B15B18FB817D6E54495AD1C4A8401B248466CB0DB0B`
-
-### PrivKeyEd25519
-
-```
-// Name: tendermint/PrivKeyEd25519
-// Notes: raw 32-byte priv key concatenated to raw 32-byte pub key
-type PrivKeyEd25519 [64]byte
-```
-
-### PubKeySecp256k1
-
-```
-// Name: tendermint/PubKeySecp256k1
-// PrefixBytes: 0xEB5AE982
-// Length: 0x21
-// Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03
-type PubKeySecp256k1 [33]byte
-
-func (pubkey PubKeySecp256k1) Address() []byte {
-      // NOTE: hash of the raw pubkey bytes (not Amino encoded!).
-        // Compatible with Bitcoin addresses.
-          return RIPEMD160(SHA256(pubkey[:]))
-}
-```
-
-For example, the 33-byte Secp256k1 pubkey
-`020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` would be
-encoded as
 `EB5AE98221020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9`
 
-The address would then be
-`RIPEMD160(SHA256(0x020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9))`
-or `0AE5BEE929ABE51BAD345DB925EEA652680783FC`
-
-### SignatureSecp256k1
-
-```
-// Name: tendermint/SignatureKeySecp256k1
-// PrefixBytes: 0x16E1FEEA
-// Length: Variable
-// Encoding prefix: Variable
-// Notes: raw bytes of the Secp256k1 signature
-type SignatureSecp256k1 []byte
-```
-
-For example, the Secp256k1 signature
+2. For example, the variable size Secp256k1 signature (in this particular example 70 or 0x46 bytes)
 `304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7`
 would be encoded as
 `16E1FEEA46304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7`
 
-### PrivKeySecp256k1
+### Addresses
+
+Addresses for each public key types are computed as follows:
+
+#### Ed25519
+
+RIPEMD160 hash of the Amino encoded public key:
 
 ```
-// Name: tendermint/PrivKeySecp256k1
-// Notes: raw 32-byte priv key
-type PrivKeySecp256k1 [32]byte
+address = RIPEMD160(AMINO(pubkey))
 ```
 
+NOTE: this will soon change to the truncated 20-bytes of the SHA256 of the raw
+public key
+
+#### Secp256k1
+
+RIPEMD160 hash of the SHA256 hash of the OpenSSL compressed public key:
+
+```
+address = RIPEMD160(SHA256(pubkey))
+```
+
+This is the same as Bitcoin.
+
 ## Other Common Types
 
 ### BitArray
@@ -290,6 +243,7 @@ Amino also supports JSON encoding - registered types are simply encoded as:
   "type": "<DisfixBytes>",
   "value": <JSON>
 }
+```
 
 For instance, an ED25519 PubKey would look like:
 

docs/spec/blockchain/state.md

@@ -77,5 +77,4 @@ func TotalVotingPower(vals []Validators) int64{
 
 ### ConsensusParams
 
-TODO:
-
+TODO

docs/spec/consensus/abci.md

@@ -52,20 +52,33 @@ objects in the `ResponseBeginBlock`:
 
 ```
 message Validator {
-    bytes pub_key = 1;
-    int64 power = 2;
+  bytes address = 1;
+  PubKey pub_key = 2;
+  int64 power = 3;
 }
+
+message PubKey {
+  string type = 1;
+  bytes  data = 2;
+}
+
 ```
 
-The `pub_key` is the Amino encoded public key for the validator. For details on
-Amino encoded public keys, see the [section of the encoding spec](./encoding.md#public-key-cryptography).
+The `pub_key` currently supports two types:
+    - `type = "ed25519" and `data = <raw 32-byte public key>`
+    - `type = "secp256k1" and `data = <33-byte OpenSSL compressed public key>`
 
+If the address is provided, it must match the address of the pubkey, as
+specified [here](/docs/spec/blockchain/encoding.md#Addresses)
+
+(Note: In the v0.19 series, the `pub_key` is the [Amino encoded public
+key](/docs/spec/blockchain/encoding.md#public-key-cryptography).
 For Ed25519 pubkeys, the Amino prefix is always "1624DE6220". For example, the 32-byte Ed25519 pubkey
 `76852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85` would be
 Amino encoded as
-`1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85`
+`1624DE622076852933A4686A721442E931A8415F62F5F1AEDF4910F1F252FB393F74C40C85`)
 
-(Note: in old versions of Tendermint (pre-v0.19.0), the pubkey is just prefixed with a
+(Note: In old versions of Tendermint (pre-v0.19.0), the pubkey is just prefixed with a
 single type byte, so for ED25519 we'd have `pub_key = 0x1 | pub`)
 
 The `power` is the new voting power for the validator, with the
@@ -79,6 +92,19 @@ following rules:
       set with the given power
     - if the validator does already exist, its power will be adjusted to the given power
 
+## InitChain Validator Updates
+
+ResponseInitChain has the option to return a list of validators.
+If the list is not empty, Tendermint will adopt it for the validator set.
+This way the application can determine the initial validator set for the
+blockchain.
+
+Note that if addressses are included in the returned validators, they must match
+the address of the public key.
+
+ResponseInitChain also includes ConsensusParams, but these are presently
+ignored.
+
 ## Query
 
 Query is a generic message type with lots of flexibility to enable diverse sets
@@ -94,7 +120,7 @@ using the following paths, with no additional data:
 
  - `/p2p/filter/addr/<IP:PORT>`, where `<IP:PORT>` denote the IP address and
    the port of the connection
- - `p2p/filter/pubkey/<ID>`, where `<ID>` is the peer node ID (ie. the
+ - `p2p/filter/id/<ID>`, where `<ID>` is the peer node ID (ie. the
    pubkey.Address() for the peer's PubKey)
 
 If either of these queries return a non-zero ABCI code, Tendermint will refuse
@@ -121,7 +147,6 @@ stateBlockHeight = height of the last block for which Tendermint completed all
     block processing and saved all ABCI results to disk
 appBlockHeight = height of the last block for which ABCI app succesfully
     completely Commit
-
 ```
 
 Note we always have `storeBlockHeight >= stateBlockHeight` and `storeBlockHeight >= appBlockHeight`
@@ -165,4 +190,3 @@ If `storeBlockHeight == stateBlockHeight+1`
 	If appBlockHeight == storeBlockHeight {
 		update the state using the saved ABCI responses but dont run the block against the real app.
 	This happens if we crashed after the app finished Commit but before Tendermint saved the state.
-

docs/spec/consensus/wal.md

@@ -0,0 +1,33 @@
+# WAL
+
+Consensus module writes every message to the WAL (write-ahead log).
+
+It also issues fsync syscall through
+[File#Sync](https://golang.org/pkg/os/#File.Sync) for messages signed by this
+node (to prevent double signing).
+
+Under the hood, it uses
+[autofile.Group](https://godoc.org/github.com/tendermint/tmlibs/autofile#Group),
+which rotates files when those get too big (> 10MB).
+
+The total maximum size is 1GB. We only need the latest block and the block before it,
+but if the former is dragging on across many rounds, we want all those rounds.
+
+## Replay
+
+Consensus module will replay all the messages of the last height written to WAL
+before a crash (if such occurs).
+
+The private validator may try to sign messages during replay because it runs
+somewhat autonomously and does not know about replay process.
+
+For example, if we got all the way to precommit in the WAL and then crash,
+after we replay the proposal message, the private validator will try to sign a
+prevote. But it will fail. That's ok because we’ll see the prevote later in the
+WAL. Then it will go to precommit, and that time it will work because the
+private validator contains the `LastSignBytes` and then we’ll replay the
+precommit from the WAL.
+
+Make sure to read about [WAL
+corruption](https://tendermint.readthedocs.io/projects/tools/en/master/specification/corruption.html#wal-corruption)
+and recovery strategies.

docs/spec/p2p/node.md

@@ -12,7 +12,7 @@ Seeds should operate full nodes with the PEX reactor in a "crawler" mode
 that continuously explores to validate the availability of peers.
 
 Seeds should only respond with some top percentile of the best peers it knows about.
-See [the peer-exchange docs](/docs/specification/new-spec/reactors/pex/pex.md)for details on peer quality.
+See [the peer-exchange docs](https://github.com/tendermint/tendermint/blob/master/docs/spec/reactors/pex/pex.md)for details on peer quality.
 
 ## New Full Node
 

docs/spec/p2p/peer.md

@@ -2,7 +2,7 @@
 
 This document explains how Tendermint Peers are identified and how they connect to one another.
 
-For details on peer discovery, see the [peer exchange (PEX) reactor doc](/docs/specification/new-spec/reactors/pex/pex.md).
+For details on peer discovery, see the [peer exchange (PEX) reactor doc](https://github.com/tendermint/tendermint/blob/master/docs/spec/reactors/pex/pex.md).
 
 ## Peer Identity
 
@@ -17,9 +17,6 @@ We will attempt to connect to the peer at IP:PORT, and verify,
 via authenticated encryption, that it is in possession of the private key
 corresponding to `<ID>`. This prevents man-in-the-middle attacks on the peer layer.
 
-If `auth_enc = false`, peers can use an arbitrary ID, but they must always use
-one. Authentication can then happen out-of-band of Tendermint, for instance via VPN.
-
 ## Connections
 
 All p2p connections use TCP.

docs/spec/reactors/block_sync/reactor.md

@@ -47,3 +47,13 @@ type bcStatusResponseMessage struct {
 ## Protocol
 
 TODO
+
+## Channels
+
+Defines `maxMsgSize` for the maximum size of incoming messages,
+`SendQueueCapacity` and `RecvBufferCapacity` for maximum sending and
+receiving buffers respectively. These are supposed to prevent amplification
+attacks by setting up the upper limit on how much data we can receive & send to
+a peer.
+
+Sending incorrectly encoded data will result in stopping the peer.

docs/spec/reactors/consensus/consensus-reactor.md

@@ -342,3 +342,11 @@ It broadcasts `NewRoundStepMessage` or `CommitStepMessage` upon new round state
 broadcasting these messages does not depend on the PeerRoundState; it is sent on the StateChannel. 
 Upon receiving VoteMessage it broadcasts `HasVoteMessage` message to its peers on the StateChannel. 
 `ProposalHeartbeatMessage` is sent the same way on the StateChannel.
+
+## Channels
+
+Defines 4 channels: state, data, vote and vote_set_bits. Each channel
+has  `SendQueueCapacity` and `RecvBufferCapacity` and
+`RecvMessageCapacity` set to `maxMsgSize`.
+
+Sending incorrectly encoded data will result in stopping the peer.

docs/spec/reactors/consensus/consensus.md

@@ -16,7 +16,7 @@ explained in a forthcoming document.
 For efficiency reasons, validators in Tendermint consensus protocol do not agree directly on the
 block as the block size is big, i.e., they don't embed the block inside `Proposal` and
 `VoteMessage`. Instead, they reach agreement on the `BlockID` (see `BlockID` definition in
-[Blockchain](blockchain.md) section) that uniquely identifies each block. The block itself is
+[Blockchain](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/blockchain.md#blockid) section) that uniquely identifies each block. The block itself is
 disseminated to validator processes using peer-to-peer gossiping protocol. It starts by having a
 proposer first splitting a block into a number of block parts, that are then gossiped between
 processes using `BlockPartMessage`.
@@ -69,7 +69,7 @@ BlockID contains PartSetHeader.
 ## VoteMessage
 
 VoteMessage is sent to vote for some block (or to inform others that a process does not vote in the
-current round). Vote is defined in [Blockchain](blockchain.md) section and contains validator's
+current round). Vote is defined in the [Blockchain](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/blockchain.md#blockid) section and contains validator's
 information (validator address and index), height and round for which the vote is sent, vote type,
 blockID if process vote for some block (`nil` otherwise) and a timestamp when the vote is sent. The
 message is signed by the validator private key.

docs/spec/reactors/consensus/proposer-selection.md

@@ -44,4 +44,3 @@ p0, p0, p0, p0, p0, p0, p0, p0, p0, p0, p0, p0, p0, p1, p0, p0, p0, p0, p0, etc
 
 This basically means that almost all rounds have the same proposer. But in this case, the process p0 has anyway enough 
 voting power to decide whatever he wants, so the fact that he coordinates almost all rounds seems correct.
-

docs/spec/reactors/evidence/reactor.md

@@ -0,0 +1,10 @@
+# Evidence Reactor
+
+## Channels
+
+[#1503](https://github.com/tendermint/tendermint/issues/1503)
+
+Sending invalid evidence will result in stopping the peer.
+
+Sending incorrectly encoded data or data exceeding `maxMsgSize` will result
+in stopping the peer.

docs/spec/reactors/mempool/README.md

@@ -1,11 +0,0 @@
-# Mempool Specification
-
-This package contains documents specifying the functionality
-of the mempool module.
-
-Components:
-
-* [Config](./config.md) - how to configure it
-* [External Messages](./messages.md) - The messages we accept over p2p and rpc interfaces
-* [Functionality](./functionality.md) - high-level description of the functionality it provides
-* [Concurrency Model](./concurrency.md) - What guarantees we provide, what locks we require.

docs/spec/reactors/mempool/messages.md

@@ -32,6 +32,7 @@ wait before returning (sync makes sure CheckTx passes, commit
 makes sure it was included in a signed block).
 
 Request (`POST http://gaia.zone:46657/`):
+
 ```json
 {
  "id": "",
@@ -43,8 +44,8 @@ Request (`POST http://gaia.zone:46657/`):
 }
 ```
 
-
 Response:
+
 ```json
 {
  "error": "",

docs/spec/reactors/mempool/reactor.md

@@ -0,0 +1,14 @@
+# Mempool Reactor
+
+## Channels
+
+[#1503](https://github.com/tendermint/tendermint/issues/1503)
+
+Mempool maintains a cache of the last 10000 transactions to prevent
+replaying old transactions (plus transactions coming from other
+validators, who are continually exchanging transactions). Read [Replay
+Protection](https://tendermint.readthedocs.io/projects/tools/en/master/app-development.html?#replay-protection)
+for details.
+
+Sending incorrectly encoded data or data exceeding `maxMsgSize` will result
+in stopping the peer.

docs/spec/reactors/pex/pex.md

@@ -117,7 +117,7 @@ current, past, and rate-of-change data to inform peer quality.
 While a PID trust metric has been implemented, it remains for future work
 to use it in the PEX.
 
-See the [trustmetric](../../../architecture/adr-006-trust-metric.md )
-and [trustmetric useage](../../../architecture/adr-007-trust-metric-usage.md )
+See the [trustmetric](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-006-trust-metric.md)
+and [trustmetric useage](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-007-trust-metric-usage.md)
 architecture docs for more details.
 

docs/spec/reactors/pex/reactor.md

@@ -0,0 +1,12 @@
+# PEX Reactor
+
+## Channels
+
+Defines only `SendQueueCapacity`. [#1503](https://github.com/tendermint/tendermint/issues/1503)
+
+Implements rate-limiting by enforcing minimal time between two consecutive
+`pexRequestMessage` requests. If the peer sends us addresses we did not ask,
+it is stopped.
+
+Sending incorrectly encoded data or data exceeding `maxMsgSize` will result
+in stopping the peer.

docs/specification/configuration.rst

@@ -122,9 +122,6 @@ like the file below, however, double check by inspecting the
     # Does not work if the peer-exchange reactor is disabled.
     seed_mode = false
 
-    # Authenticated encryption
-    auth_enc = true
-
     # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
     private_peer_ids = ""
 

docs/specification/new-spec/README.md

@@ -1 +1 @@
-Spec moved to [docs/spec](/docs/spec).
+Spec moved to [docs/spec](https://github.com/tendermint/tendermint/tree/master/docs/spec).

docs/specification/rpc.rst

@@ -1,190 +1,4 @@
 RPC
 ===
 
-Coming soon: RPC docs powered by `slate <https://github.com/lord/slate>`__. Until then, read on.
-
-Tendermint supports the following RPC protocols:
-
--  URI over HTTP
--  JSONRPC over HTTP
--  JSONRPC over websockets
-
-Tendermint RPC is build using `our own RPC
-library <https://github.com/tendermint/tendermint/tree/master/rpc/lib>`__.
-Documentation and tests for that library could be found at
-``tendermint/rpc/lib`` directory.
-
-Configuration
-~~~~~~~~~~~~~
-
-Set the ``laddr`` config parameter under ``[rpc]`` table in the
-$TMHOME/config/config.toml file or the ``--rpc.laddr`` command-line flag to the
-desired protocol://host:port setting. Default: ``tcp://0.0.0.0:46657``.
-
-Arguments
-~~~~~~~~~
-
-Arguments which expect strings or byte arrays may be passed as quoted
-strings, like ``"abc"`` or as ``0x``-prefixed strings, like
-``0x616263``.
-
-URI/HTTP
-~~~~~~~~
-
-Example request:
-
-.. code:: bash
-
-    curl -s 'http://localhost:46657/broadcast_tx_sync?tx="abc"' | jq .
-
-Response:
-
-.. code:: json
-
-    {
-      "error": "",
-      "result": {
-        "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF",
-        "log": "",
-        "data": "",
-        "code": 0
-      },
-      "id": "",
-      "jsonrpc": "2.0"
-    }
-
-The first entry in the result-array (``96``) is the method this response
-correlates with. ``96`` refers to "ResultTypeBroadcastTx", see
-`responses.go <https://github.com/tendermint/tendermint/blob/master/rpc/core/types/responses.go>`__
-for a complete overview.
-
-JSONRPC/HTTP
-~~~~~~~~~~~~
-
-JSONRPC requests can be POST'd to the root RPC endpoint via HTTP (e.g.
-``http://localhost:46657/``).
-
-Example request:
-
-.. code:: json
-
-    {
-      "method": "broadcast_tx_sync",
-      "jsonrpc": "2.0",
-      "params": [ "abc" ],
-      "id": "dontcare"
-    }
-
-JSONRPC/websockets
-~~~~~~~~~~~~~~~~~~
-
-JSONRPC requests can be made via websocket. The websocket endpoint is at
-``/websocket``, e.g. ``http://localhost:46657/websocket``. Asynchronous
-RPC functions like event ``subscribe`` and ``unsubscribe`` are only
-available via websockets.
-
-Endpoints
-~~~~~~~~~
-
-An HTTP Get request to the root RPC endpoint (e.g.
-``http://localhost:46657``) shows a list of available endpoints.
-
-::
-
-    Available endpoints:
-    http://localhost:46657/abci_info
-    http://localhost:46657/dump_consensus_state
-    http://localhost:46657/genesis
-    http://localhost:46657/net_info
-    http://localhost:46657/num_unconfirmed_txs
-    http://localhost:46657/health
-    http://localhost:46657/status
-    http://localhost:46657/unconfirmed_txs
-    http://localhost:46657/unsafe_flush_mempool
-    http://localhost:46657/unsafe_stop_cpu_profiler
-    http://localhost:46657/validators
-
-    Endpoints that require arguments:
-    http://localhost:46657/abci_query?path=_&data=_&prove=_
-    http://localhost:46657/block?height=_
-    http://localhost:46657/blockchain?minHeight=_&maxHeight=_
-    http://localhost:46657/broadcast_tx_async?tx=_
-    http://localhost:46657/broadcast_tx_commit?tx=_
-    http://localhost:46657/broadcast_tx_sync?tx=_
-    http://localhost:46657/commit?height=_
-    http://localhost:46657/dial_seeds?seeds=_
-    http://localhost:46657/dial_peers?peers=_&persistent=_
-    http://localhost:46657/subscribe?event=_
-    http://localhost:46657/tx?hash=_&prove=_
-    http://localhost:46657/unsafe_start_cpu_profiler?filename=_
-    http://localhost:46657/unsafe_write_heap_profile?filename=_
-    http://localhost:46657/unsubscribe?event=_
-
-tx
-~~
-
-Returns a transaction matching the given transaction hash.
-
-**Parameters**
-
-1. hash - the transaction hash
-2. prove - include a proof of the transaction inclusion in the block in
-   the result (optional, default: false)
-
-**Returns**
-
--  ``proof``: the ``types.TxProof`` object
--  ``tx``: ``[]byte`` - the transaction
--  ``tx_result``: the ``abci.Result`` object
--  ``index``: ``int`` - index of the transaction
--  ``height``: ``int`` - height of the block where this transaction was
-   in
-
-**Example**
-
-.. code:: bash
-
-    curl -s 'http://localhost:46657/broadcast_tx_commit?tx="abc"' | jq .
-    # {
-    #   "error": "",
-    #   "result": {
-    #     "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF",
-    #     "log": "",
-    #     "data": "",
-    #     "code": 0
-    #   },
-    #   "id": "",
-    #   "jsonrpc": "2.0"
-    # }
-
-    curl -s 'http://localhost:46657/tx?hash=0x2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF' | jq .
-    # {
-    #   "error": "",
-    #   "result": {
-    #     "proof": {
-    #       "Proof": {
-    #         "aunts": []
-    #       },
-    #       "Data": "YWJjZA==",
-    #       "RootHash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF",
-    #       "Total": 1,
-    #       "Index": 0
-    #     },
-    #     "tx": "YWJjZA==",
-    #     "tx_result": {
-    #       "log": "",
-    #       "data": "",
-    #       "code": 0
-    #     },
-    #     "index": 0,
-    #     "height": 52
-    #   },
-    #   "id": "",
-    #   "jsonrpc": "2.0"
-    # }
-
-More Examples
-~~~~~~~~~~~~~
-
-See the various bash tests using curl in ``test/``, and examples using
-the ``Go`` API in ``rpc/client/``.
+The RPC documentation is hosted `here <https://tendermint.github.io/slate>`__ and is generated by the CI from our `Slate repo <https://github.com/tendermint/slate>`__. To update the documentation, edit the relevant ``godoc`` comments in the `rpc/core directory <https://github.com/tendermint/tendermint/tree/develop/rpc/core>`__.

docs/specification/secure-p2p.rst

@@ -65,9 +65,7 @@ are connected to at least one validator.
 Config
 ------
 
-Authenticated encryption is enabled by default. If you wish to use another
-authentication scheme or your peers are connected via VPN, you can turn it off
-by setting ``auth_enc`` to ``false`` in the config file.
+Authenticated encryption is enabled by default.
 
 Additional Reading
 ------------------

docs/subscribing-to-events-via-websocket.rst

@@ -0,0 +1,28 @@
+Subscribing to events via Websocket
+===================================
+
+Tendermint emits different events, to which you can subscribe via `Websocket
+<https://en.wikipedia.org/wiki/WebSocket>`__. This can be useful for
+third-party applications (for analysys) or inspecting state.
+
+`List of events <https://godoc.org/github.com/tendermint/tendermint/types#pkg-constants>`__
+
+You can subscribe to any of the events above by calling ``subscribe`` RPC method via Websocket.
+
+::
+
+    {
+        "jsonrpc": "2.0",
+        "method": "subscribe",
+        "id": "0",
+        "params": {
+            "query": "tm.event='NewBlock'"
+        }
+    }
+
+Check out `API docs <https://tendermint.github.io/slate/#subscribe>`__ for more
+information on query syntax and other options.
+
+You can also use tags, given you had included them into DeliverTx response, to
+query transaction results. See `Indexing transactions
+<./indexing-transactions.html>`__ for details.

evidence/pool.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sync"
 
+	clist "github.com/tendermint/tmlibs/clist"
 	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
 
@@ -17,6 +18,7 @@ type EvidencePool struct {
 	logger log.Logger
 
 	evidenceStore *EvidenceStore
+	evidenceList  *clist.CList // concurrent linked-list of evidence
 
 	// needed to load validators to verify evidence
 	stateDB dbm.DB
@@ -24,9 +26,6 @@ type EvidencePool struct {
 	// latest state
 	mtx   sync.Mutex
 	state sm.State
-
-	// never close
-	evidenceChan chan types.Evidence
 }
 
 func NewEvidencePool(stateDB dbm.DB, evidenceStore *EvidenceStore) *EvidencePool {
@@ -35,21 +34,24 @@ func NewEvidencePool(stateDB dbm.DB, evidenceStore *EvidenceStore) *EvidencePool
 		state:         sm.LoadState(stateDB),
 		logger:        log.NewNopLogger(),
 		evidenceStore: evidenceStore,
-		evidenceChan:  make(chan types.Evidence),
+		evidenceList:  clist.New(),
 	}
 	return evpool
 }
 
+func (evpool *EvidencePool) EvidenceFront() *clist.CElement {
+	return evpool.evidenceList.Front()
+}
+
+func (evpool *EvidencePool) EvidenceWaitChan() <-chan struct{} {
+	return evpool.evidenceList.WaitChan()
+}
+
 // SetLogger sets the Logger.
 func (evpool *EvidencePool) SetLogger(l log.Logger) {
 	evpool.logger = l
 }
 
-// EvidenceChan returns an unbuffered channel on which new evidence can be received.
-func (evpool *EvidencePool) EvidenceChan() <-chan types.Evidence {
-	return evpool.evidenceChan
-}
-
 // PriorityEvidence returns the priority evidence.
 func (evpool *EvidencePool) PriorityEvidence() []types.Evidence {
 	return evpool.evidenceStore.PriorityEvidence()
@@ -68,22 +70,23 @@ func (evpool *EvidencePool) State() sm.State {
 }
 
 // Update loads the latest
-func (evpool *EvidencePool) Update(block *types.Block) {
-	evpool.mtx.Lock()
-	defer evpool.mtx.Unlock()
+func (evpool *EvidencePool) Update(block *types.Block, state sm.State) {
 
-	state := sm.LoadState(evpool.stateDB)
+	// sanity check
 	if state.LastBlockHeight != block.Height {
-		panic(fmt.Sprintf("EvidencePool.Update: loaded state with height %d when block.Height=%d", state.LastBlockHeight, block.Height))
+		panic(fmt.Sprintf("Failed EvidencePool.Update sanity check: got state.Height=%d with block.Height=%d", state.LastBlockHeight, block.Height))
 	}
-	evpool.state = state
 
-	// NOTE: shouldn't need the mutex
-	evpool.MarkEvidenceAsCommitted(block.Evidence.Evidence)
+	// update the state
+	evpool.mtx.Lock()
+	evpool.state = state
+	evpool.mtx.Unlock()
+
+	// remove evidence from pending and mark committed
+	evpool.MarkEvidenceAsCommitted(block.Height, block.Evidence.Evidence)
 }
 
 // AddEvidence checks the evidence is valid and adds it to the pool.
-// Blocks on the EvidenceChan.
 func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) {
 
 	// TODO: check if we already have evidence for this
@@ -107,14 +110,43 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) {
 
 	evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence)
 
-	// never closes. always safe to send on
-	evpool.evidenceChan <- evidence
+	// add evidence to clist
+	evpool.evidenceList.PushBack(evidence)
+
 	return nil
 }
 
-// MarkEvidenceAsCommitted marks all the evidence as committed.
-func (evpool *EvidencePool) MarkEvidenceAsCommitted(evidence []types.Evidence) {
+// MarkEvidenceAsCommitted marks all the evidence as committed and removes it from the queue.
+func (evpool *EvidencePool) MarkEvidenceAsCommitted(height int64, evidence []types.Evidence) {
+	// make a map of committed evidence to remove from the clist
+	blockEvidenceMap := make(map[string]struct{})
 	for _, ev := range evidence {
 		evpool.evidenceStore.MarkEvidenceAsCommitted(ev)
+		blockEvidenceMap[evMapKey(ev)] = struct{}{}
+	}
+
+	// remove committed evidence from the clist
+	maxAge := evpool.State().ConsensusParams.EvidenceParams.MaxAge
+	evpool.removeEvidence(height, maxAge, blockEvidenceMap)
+
+}
+
+func (evpool *EvidencePool) removeEvidence(height, maxAge int64, blockEvidenceMap map[string]struct{}) {
+	for e := evpool.evidenceList.Front(); e != nil; e = e.Next() {
+		ev := e.Value.(types.Evidence)
+
+		// Remove the evidence if it's already in a block
+		// or if it's now too old.
+		if _, ok := blockEvidenceMap[evMapKey(ev)]; ok ||
+			ev.Height() < height-maxAge {
+
+			// remove from clist
+			evpool.evidenceList.Remove(e)
+			e.DetachPrev()
+		}
 	}
 }
+
+func evMapKey(ev types.Evidence) string {
+	return string(ev.Hash())
+}

evidence/pool_test.go

@@ -45,7 +45,6 @@ func initializeValidatorState(valAddr []byte, height int64) dbm.DB {
 }
 
 func TestEvidencePool(t *testing.T) {
-	assert := assert.New(t)
 
 	valAddr := []byte("val1")
 	height := int64(5)
@@ -56,26 +55,25 @@ func TestEvidencePool(t *testing.T) {
 	goodEvidence := types.NewMockGoodEvidence(height, 0, valAddr)
 	badEvidence := types.MockBadEvidence{goodEvidence}
 
+	// bad evidence
 	err := pool.AddEvidence(badEvidence)
-	assert.NotNil(err)
+	assert.NotNil(t, err)
 
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		<-pool.EvidenceChan()
+		<-pool.EvidenceWaitChan()
 		wg.Done()
 	}()
 
 	err = pool.AddEvidence(goodEvidence)
-	assert.Nil(err)
+	assert.Nil(t, err)
 	wg.Wait()
 
-	// if we send it again it wont fire on the chan
+	assert.Equal(t, 1, pool.evidenceList.Len())
+
+	// if we send it again, it shouldnt change the size
 	err = pool.AddEvidence(goodEvidence)
-	assert.Nil(err)
-	select {
-	case <-pool.EvidenceChan():
-		t.Fatal("unexpected read on EvidenceChan")
-	default:
-	}
+	assert.Nil(t, err)
+	assert.Equal(t, 1, pool.evidenceList.Len())
 }

evidence/reactor.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/tendermint/go-amino"
+	clist "github.com/tendermint/tmlibs/clist"
 	"github.com/tendermint/tmlibs/log"
 
 	"github.com/tendermint/tendermint/p2p"
@@ -15,8 +16,10 @@ import (
 const (
 	EvidenceChannel = byte(0x38)
 
-	maxMsgSize                 = 1048576 // 1MB TODO make it configurable
-	broadcastEvidenceIntervalS = 60      // broadcast uncommitted evidence this often
+	maxMsgSize = 1048576 // 1MB TODO make it configurable
+
+	broadcastEvidenceIntervalS = 60  // broadcast uncommitted evidence this often
+	peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount
 )
 
 // EvidenceReactor handles evpool evidence broadcasting amongst peers.
@@ -43,11 +46,7 @@ func (evR *EvidenceReactor) SetLogger(l log.Logger) {
 
 // OnStart implements cmn.Service
 func (evR *EvidenceReactor) OnStart() error {
-	if err := evR.BaseReactor.OnStart(); err != nil {
-		return err
-	}
-	go evR.broadcastRoutine()
-	return nil
+	return evR.BaseReactor.OnStart()
 }
 
 // GetChannels implements Reactor.
@@ -63,14 +62,7 @@ func (evR *EvidenceReactor) GetChannels() []*p2p.ChannelDescriptor {
 
 // AddPeer implements Reactor.
 func (evR *EvidenceReactor) AddPeer(peer p2p.Peer) {
-	// send the peer our high-priority evidence.
-	// the rest will be sent by the broadcastRoutine
-	evidences := evR.evpool.PriorityEvidence()
-	msg := &EvidenceListMessage{evidences}
-	success := peer.Send(EvidenceChannel, cdc.MustMarshalBinaryBare(msg))
-	if !success {
-		// TODO: remove peer ?
-	}
+	go evR.broadcastEvidenceRoutine(peer)
 }
 
 // RemovePeer implements Reactor.
@@ -109,30 +101,97 @@ func (evR *EvidenceReactor) SetEventBus(b *types.EventBus) {
 	evR.eventBus = b
 }
 
-// Broadcast new evidence to all peers.
-// Broadcasts must be non-blocking so routine is always available to read off EvidenceChan.
-func (evR *EvidenceReactor) broadcastRoutine() {
-	ticker := time.NewTicker(time.Second * broadcastEvidenceIntervalS)
+// Modeled after the mempool routine.
+// - Evidence accumulates in a clist.
+// - Each peer has a routien that iterates through the clist,
+// sending available evidence to the peer.
+// - If we're waiting for new evidence and the list is not empty,
+// start iterating from the beginning again.
+func (evR *EvidenceReactor) broadcastEvidenceRoutine(peer p2p.Peer) {
+	var next *clist.CElement
 	for {
-		select {
-		case evidence := <-evR.evpool.EvidenceChan():
-			// broadcast some new evidence
-			msg := &EvidenceListMessage{[]types.Evidence{evidence}}
-			evR.Switch.Broadcast(EvidenceChannel, cdc.MustMarshalBinaryBare(msg))
+		// This happens because the CElement we were looking at got garbage
+		// collected (removed). That is, .NextWait() returned nil. Go ahead and
+		// start from the beginning.
+		if next == nil {
+			select {
+			case <-evR.evpool.EvidenceWaitChan(): // Wait until evidence is available
+				if next = evR.evpool.EvidenceFront(); next == nil {
+					continue
+				}
+			case <-peer.Quit():
+				return
+			case <-evR.Quit():
+				return
+			}
+		}
 
-			// TODO: Broadcast runs asynchronously, so this should wait on the successChan
-			// in another routine before marking to be proper.
-			evR.evpool.evidenceStore.MarkEvidenceAsBroadcasted(evidence)
-		case <-ticker.C:
-			// broadcast all pending evidence
-			msg := &EvidenceListMessage{evR.evpool.PendingEvidence()}
-			evR.Switch.Broadcast(EvidenceChannel, cdc.MustMarshalBinaryBare(msg))
+		ev := next.Value.(types.Evidence)
+		msg, retry := evR.checkSendEvidenceMessage(peer, ev)
+		if msg != nil {
+			success := peer.Send(EvidenceChannel, cdc.MustMarshalBinaryBare(msg))
+			retry = !success
+		}
+
+		if retry {
+			time.Sleep(peerCatchupSleepIntervalMS * time.Millisecond)
+			continue
+		}
+
+		afterCh := time.After(time.Second * broadcastEvidenceIntervalS)
+		select {
+		case <-afterCh:
+			// start from the beginning every tick.
+			// TODO: only do this if we're at the end of the list!
+			next = nil
+		case <-next.NextWaitChan():
+			// see the start of the for loop for nil check
+			next = next.Next()
+		case <-peer.Quit():
+			return
 		case <-evR.Quit():
 			return
 		}
 	}
 }
 
+// Returns the message to send the peer, or nil if the evidence is invalid for the peer.
+// If message is nil, return true if we should sleep and try again.
+func (evR EvidenceReactor) checkSendEvidenceMessage(peer p2p.Peer, ev types.Evidence) (msg EvidenceMessage, retry bool) {
+
+	// make sure the peer is up to date
+	evHeight := ev.Height()
+	peerState, ok := peer.Get(types.PeerStateKey).(PeerState)
+	if !ok {
+		evR.Logger.Info("Found peer without PeerState", "peer", peer)
+		return nil, true
+	}
+
+	// NOTE: We only send evidence to peers where
+	// peerHeight - maxAge < evidenceHeight < peerHeight
+	maxAge := evR.evpool.State().ConsensusParams.EvidenceParams.MaxAge
+	peerHeight := peerState.GetHeight()
+	if peerHeight < evHeight {
+		// peer is behind. sleep while he catches up
+		return nil, true
+	} else if peerHeight > evHeight+maxAge {
+		// evidence is too old, skip
+		// NOTE: if evidence is too old for an honest peer,
+		// then we're behind and either it already got committed or it never will!
+		evR.Logger.Info("Not sending peer old evidence", "peerHeight", peerHeight, "evHeight", evHeight, "maxAge", maxAge, "peer", peer)
+		return nil, false
+	}
+
+	// send evidence
+	msg = &EvidenceListMessage{[]types.Evidence{ev}}
+	return msg, false
+}
+
+// PeerState describes the state of a peer.
+type PeerState interface {
+	GetHeight() int64
+}
+
 //-----------------------------------------------------------------------------
 // Messages
 

evidence/reactor_test.go

@@ -84,7 +84,7 @@ func _waitForEvidence(t *testing.T, wg *sync.WaitGroup, evs types.EvidenceList,
 	}
 
 	reapedEv := evpool.PendingEvidence()
-	// put the reaped evidence is a map so we can quickly check we got everything
+	// put the reaped evidence in a map so we can quickly check we got everything
 	evMap := make(map[string]types.Evidence)
 	for _, e := range reapedEv {
 		evMap[string(e.Hash())] = e
@@ -95,6 +95,7 @@ func _waitForEvidence(t *testing.T, wg *sync.WaitGroup, evs types.EvidenceList,
 			fmt.Sprintf("evidence at index %d on reactor %d don't match: %v vs %v",
 				i, reactorIdx, expectedEv, gotEv))
 	}
+
 	wg.Done()
 }
 
@@ -110,7 +111,7 @@ func sendEvidence(t *testing.T, evpool *EvidencePool, valAddr []byte, n int) typ
 }
 
 var (
-	NUM_EVIDENCE = 1
+	NUM_EVIDENCE = 10
 	TIMEOUT      = 120 * time.Second // ridiculously high because CircleCI is slow
 )
 
@@ -130,8 +131,52 @@ func TestReactorBroadcastEvidence(t *testing.T) {
 	// make reactors from statedb
 	reactors := makeAndConnectEvidenceReactors(config, stateDBs)
 
+	// set the peer height on each reactor
+	for _, r := range reactors {
+		for _, peer := range r.Switch.Peers().List() {
+			ps := peerState{height}
+			peer.Set(types.PeerStateKey, ps)
+		}
+	}
+
 	// send a bunch of valid evidence to the first reactor's evpool
 	// and wait for them all to be received in the others
 	evList := sendEvidence(t, reactors[0].evpool, valAddr, NUM_EVIDENCE)
 	waitForEvidence(t, evList, reactors)
 }
+
+type peerState struct {
+	height int64
+}
+
+func (ps peerState) GetHeight() int64 {
+	return ps.height
+}
+
+func TestReactorSelectiveBroadcast(t *testing.T) {
+	config := cfg.TestConfig()
+
+	valAddr := []byte("myval")
+	height1 := int64(NUM_EVIDENCE) + 10
+	height2 := int64(NUM_EVIDENCE) / 2
+
+	// DB1 is ahead of DB2
+	stateDB1 := initializeValidatorState(valAddr, height1)
+	stateDB2 := initializeValidatorState(valAddr, height2)
+
+	// make reactors from statedb
+	reactors := makeAndConnectEvidenceReactors(config, []dbm.DB{stateDB1, stateDB2})
+	peer := reactors[0].Switch.Peers().List()[0]
+	ps := peerState{height2}
+	peer.Set(types.PeerStateKey, ps)
+
+	// send a bunch of valid evidence to the first reactor's evpool
+	evList := sendEvidence(t, reactors[0].evpool, valAddr, NUM_EVIDENCE)
+
+	// only ones less than the peers height should make it through
+	waitForEvidence(t, evList[:NUM_EVIDENCE/2], reactors[1:2])
+
+	// peers should still be connected
+	peers := reactors[1].Switch.Peers().List()
+	assert.Equal(t, 1, len(peers))
+}

evidence/store.go

@@ -17,10 +17,6 @@ Impl:
 	- First commit atomically in outqueue, pending, lookup.
 	- Once broadcast, remove from outqueue. No need to sync
 	- Once committed, atomically remove from pending and update lookup.
-		- TODO: If we crash after committed but before removing/updating,
-			we'll be stuck broadcasting evidence we never know we committed.
-			so either share the state db and atomically MarkCommitted
-			with ApplyBlock, or check all outqueue/pending on Start to see if its committed
 
 Schema for indexing evidence (note you need both height and hash to find a piece of evidence):
 
@@ -164,7 +160,7 @@ func (store *EvidenceStore) MarkEvidenceAsBroadcasted(evidence types.Evidence) {
 	store.db.Delete(key)
 }
 
-// MarkEvidenceAsPending removes evidence from pending and outqueue and sets the state to committed.
+// MarkEvidenceAsCommitted removes evidence from pending and outqueue and sets the state to committed.
 func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) {
 	// if its committed, its been broadcast
 	store.MarkEvidenceAsBroadcasted(evidence)

libs/events/Makefile

@@ -0,0 +1,9 @@
+.PHONY: docs
+REPO:=github.com/tendermint/tendermint/libs/events
+
+docs:
+	@go get github.com/davecheney/godoc2md
+	godoc2md $(REPO) > README.md
+
+test:
+	go test -v ./...

libs/events/README.md

@@ -0,0 +1,175 @@
+
+
+# events
+`import "github.com/tendermint/tendermint/libs/events"`
+
+* [Overview](#pkg-overview)
+* [Index](#pkg-index)
+
+## <a name="pkg-overview">Overview</a>
+Pub-Sub in go with event caching
+
+
+
+
+## <a name="pkg-index">Index</a>
+* [type EventCache](#EventCache)
+  * [func NewEventCache(evsw Fireable) *EventCache](#NewEventCache)
+  * [func (evc *EventCache) FireEvent(event string, data EventData)](#EventCache.FireEvent)
+  * [func (evc *EventCache) Flush()](#EventCache.Flush)
+* [type EventCallback](#EventCallback)
+* [type EventData](#EventData)
+* [type EventSwitch](#EventSwitch)
+  * [func NewEventSwitch() EventSwitch](#NewEventSwitch)
+* [type Eventable](#Eventable)
+* [type Fireable](#Fireable)
+
+
+#### <a name="pkg-files">Package files</a>
+[event_cache.go](/src/github.com/tendermint/tendermint/libs/events/event_cache.go) [events.go](/src/github.com/tendermint/tendermint/libs/events/events.go) 
+
+
+
+
+
+
+## <a name="EventCache">type</a> [EventCache](/src/target/event_cache.go?s=116:179#L5)
+``` go
+type EventCache struct {
+    // contains filtered or unexported fields
+}
+```
+An EventCache buffers events for a Fireable
+All events are cached. Filtering happens on Flush
+
+
+
+
+
+
+
+### <a name="NewEventCache">func</a> [NewEventCache](/src/target/event_cache.go?s=239:284#L11)
+``` go
+func NewEventCache(evsw Fireable) *EventCache
+```
+Create a new EventCache with an EventSwitch as backend
+
+
+
+
+
+### <a name="EventCache.FireEvent">func</a> (\*EventCache) [FireEvent](/src/target/event_cache.go?s=449:511#L24)
+``` go
+func (evc *EventCache) FireEvent(event string, data EventData)
+```
+Cache an event to be fired upon finality.
+
+
+
+
+### <a name="EventCache.Flush">func</a> (\*EventCache) [Flush](/src/target/event_cache.go?s=735:765#L31)
+``` go
+func (evc *EventCache) Flush()
+```
+Fire events by running evsw.FireEvent on all cached events. Blocks.
+Clears cached events
+
+
+
+
+## <a name="EventCallback">type</a> [EventCallback](/src/target/events.go?s=4201:4240#L185)
+``` go
+type EventCallback func(data EventData)
+```
+
+
+
+
+
+
+
+
+
+## <a name="EventData">type</a> [EventData](/src/target/events.go?s=243:294#L14)
+``` go
+type EventData interface {
+}
+```
+Generic event data can be typed and registered with tendermint/go-amino
+via concrete implementation of this interface
+
+
+
+
+
+
+
+
+
+
+## <a name="EventSwitch">type</a> [EventSwitch](/src/target/events.go?s=560:771#L29)
+``` go
+type EventSwitch interface {
+    cmn.Service
+    Fireable
+
+    AddListenerForEvent(listenerID, event string, cb EventCallback)
+    RemoveListenerForEvent(event string, listenerID string)
+    RemoveListener(listenerID string)
+}
+```
+
+
+
+
+
+
+### <a name="NewEventSwitch">func</a> [NewEventSwitch](/src/target/events.go?s=917:950#L46)
+``` go
+func NewEventSwitch() EventSwitch
+```
+
+
+
+
+## <a name="Eventable">type</a> [Eventable](/src/target/events.go?s=378:440#L20)
+``` go
+type Eventable interface {
+    SetEventSwitch(evsw EventSwitch)
+}
+```
+reactors and other modules should export
+this interface to become eventable
+
+
+
+
+
+
+
+
+
+
+## <a name="Fireable">type</a> [Fireable](/src/target/events.go?s=490:558#L25)
+``` go
+type Fireable interface {
+    FireEvent(event string, data EventData)
+}
+```
+an event switch or cache implements fireable
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+- - -
+Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md)

libs/events/event_cache.go

@@ -0,0 +1,37 @@
+package events
+
+// An EventCache buffers events for a Fireable
+// All events are cached. Filtering happens on Flush
+type EventCache struct {
+	evsw   Fireable
+	events []eventInfo
+}
+
+// Create a new EventCache with an EventSwitch as backend
+func NewEventCache(evsw Fireable) *EventCache {
+	return &EventCache{
+		evsw: evsw,
+	}
+}
+
+// a cached event
+type eventInfo struct {
+	event string
+	data  EventData
+}
+
+// Cache an event to be fired upon finality.
+func (evc *EventCache) FireEvent(event string, data EventData) {
+	// append to list (go will grow our backing array exponentially)
+	evc.events = append(evc.events, eventInfo{event, data})
+}
+
+// Fire events by running evsw.FireEvent on all cached events. Blocks.
+// Clears cached events
+func (evc *EventCache) Flush() {
+	for _, ei := range evc.events {
+		evc.evsw.FireEvent(ei.event, ei.data)
+	}
+	// Clear the buffer, since we only add to it with append it's safe to just set it to nil and maybe safe an allocation
+	evc.events = nil
+}

libs/events/event_cache_test.go

@@ -0,0 +1,35 @@
+package events
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEventCache_Flush(t *testing.T) {
+	evsw := NewEventSwitch()
+	evsw.Start()
+	evsw.AddListenerForEvent("nothingness", "", func(data EventData) {
+		// Check we are not initialising an empty buffer full of zeroed eventInfos in the EventCache
+		require.FailNow(t, "We should never receive a message on this switch since none are fired")
+	})
+	evc := NewEventCache(evsw)
+	evc.Flush()
+	// Check after reset
+	evc.Flush()
+	fail := true
+	pass := false
+	evsw.AddListenerForEvent("somethingness", "something", func(data EventData) {
+		if fail {
+			require.FailNow(t, "Shouldn't see a message until flushed")
+		}
+		pass = true
+	})
+	evc.FireEvent("something", struct{ int }{1})
+	evc.FireEvent("something", struct{ int }{2})
+	evc.FireEvent("something", struct{ int }{3})
+	fail = false
+	evc.Flush()
+	assert.True(t, pass)
+}

libs/events/events.go

@@ -0,0 +1,220 @@
+/*
+Pub-Sub in go with event caching
+*/
+package events
+
+import (
+	"sync"
+
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+// Generic event data can be typed and registered with tendermint/go-amino
+// via concrete implementation of this interface
+type EventData interface {
+	//AssertIsEventData()
+}
+
+// reactors and other modules should export
+// this interface to become eventable
+type Eventable interface {
+	SetEventSwitch(evsw EventSwitch)
+}
+
+// an event switch or cache implements fireable
+type Fireable interface {
+	FireEvent(event string, data EventData)
+}
+
+type EventSwitch interface {
+	cmn.Service
+	Fireable
+
+	AddListenerForEvent(listenerID, event string, cb EventCallback)
+	RemoveListenerForEvent(event string, listenerID string)
+	RemoveListener(listenerID string)
+}
+
+type eventSwitch struct {
+	cmn.BaseService
+
+	mtx        sync.RWMutex
+	eventCells map[string]*eventCell
+	listeners  map[string]*eventListener
+}
+
+func NewEventSwitch() EventSwitch {
+	evsw := &eventSwitch{
+		eventCells: make(map[string]*eventCell),
+		listeners:  make(map[string]*eventListener),
+	}
+	evsw.BaseService = *cmn.NewBaseService(nil, "EventSwitch", evsw)
+	return evsw
+}
+
+func (evsw *eventSwitch) OnStart() error {
+	return nil
+}
+
+func (evsw *eventSwitch) OnStop() {}
+
+func (evsw *eventSwitch) AddListenerForEvent(listenerID, event string, cb EventCallback) {
+	// Get/Create eventCell and listener
+	evsw.mtx.Lock()
+	eventCell := evsw.eventCells[event]
+	if eventCell == nil {
+		eventCell = newEventCell()
+		evsw.eventCells[event] = eventCell
+	}
+	listener := evsw.listeners[listenerID]
+	if listener == nil {
+		listener = newEventListener(listenerID)
+		evsw.listeners[listenerID] = listener
+	}
+	evsw.mtx.Unlock()
+
+	// Add event and listener
+	eventCell.AddListener(listenerID, cb)
+	listener.AddEvent(event)
+}
+
+func (evsw *eventSwitch) RemoveListener(listenerID string) {
+	// Get and remove listener
+	evsw.mtx.RLock()
+	listener := evsw.listeners[listenerID]
+	evsw.mtx.RUnlock()
+	if listener == nil {
+		return
+	}
+
+	evsw.mtx.Lock()
+	delete(evsw.listeners, listenerID)
+	evsw.mtx.Unlock()
+
+	// Remove callback for each event.
+	listener.SetRemoved()
+	for _, event := range listener.GetEvents() {
+		evsw.RemoveListenerForEvent(event, listenerID)
+	}
+}
+
+func (evsw *eventSwitch) RemoveListenerForEvent(event string, listenerID string) {
+	// Get eventCell
+	evsw.mtx.Lock()
+	eventCell := evsw.eventCells[event]
+	evsw.mtx.Unlock()
+
+	if eventCell == nil {
+		return
+	}
+
+	// Remove listenerID from eventCell
+	numListeners := eventCell.RemoveListener(listenerID)
+
+	// Maybe garbage collect eventCell.
+	if numListeners == 0 {
+		// Lock again and double check.
+		evsw.mtx.Lock()      // OUTER LOCK
+		eventCell.mtx.Lock() // INNER LOCK
+		if len(eventCell.listeners) == 0 {
+			delete(evsw.eventCells, event)
+		}
+		eventCell.mtx.Unlock() // INNER LOCK
+		evsw.mtx.Unlock()      // OUTER LOCK
+	}
+}
+
+func (evsw *eventSwitch) FireEvent(event string, data EventData) {
+	// Get the eventCell
+	evsw.mtx.RLock()
+	eventCell := evsw.eventCells[event]
+	evsw.mtx.RUnlock()
+
+	if eventCell == nil {
+		return
+	}
+
+	// Fire event for all listeners in eventCell
+	eventCell.FireEvent(data)
+}
+
+//-----------------------------------------------------------------------------
+
+// eventCell handles keeping track of listener callbacks for a given event.
+type eventCell struct {
+	mtx       sync.RWMutex
+	listeners map[string]EventCallback
+}
+
+func newEventCell() *eventCell {
+	return &eventCell{
+		listeners: make(map[string]EventCallback),
+	}
+}
+
+func (cell *eventCell) AddListener(listenerID string, cb EventCallback) {
+	cell.mtx.Lock()
+	cell.listeners[listenerID] = cb
+	cell.mtx.Unlock()
+}
+
+func (cell *eventCell) RemoveListener(listenerID string) int {
+	cell.mtx.Lock()
+	delete(cell.listeners, listenerID)
+	numListeners := len(cell.listeners)
+	cell.mtx.Unlock()
+	return numListeners
+}
+
+func (cell *eventCell) FireEvent(data EventData) {
+	cell.mtx.RLock()
+	for _, listener := range cell.listeners {
+		listener(data)
+	}
+	cell.mtx.RUnlock()
+}
+
+//-----------------------------------------------------------------------------
+
+type EventCallback func(data EventData)
+
+type eventListener struct {
+	id string
+
+	mtx     sync.RWMutex
+	removed bool
+	events  []string
+}
+
+func newEventListener(id string) *eventListener {
+	return &eventListener{
+		id:      id,
+		removed: false,
+		events:  nil,
+	}
+}
+
+func (evl *eventListener) AddEvent(event string) {
+	evl.mtx.Lock()
+	defer evl.mtx.Unlock()
+
+	if evl.removed {
+		return
+	}
+	evl.events = append(evl.events, event)
+}
+
+func (evl *eventListener) GetEvents() []string {
+	evl.mtx.RLock()
+	defer evl.mtx.RUnlock()
+
+	events := make([]string, len(evl.events))
+	copy(events, evl.events)
+	return events
+}
+
+func (evl *eventListener) SetRemoved() {
+	evl.mtx.Lock()
+	defer evl.mtx.Unlock()
+	evl.removed = true
+}

libs/events/events_test.go

@@ -0,0 +1,380 @@
+package events
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestAddListenerForEventFireOnce sets up an EventSwitch, subscribes a single
+// listener to an event, and sends a string "data".
+func TestAddListenerForEventFireOnce(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	messages := make(chan EventData)
+	evsw.AddListenerForEvent("listener", "event",
+		func(data EventData) {
+			messages <- data
+		})
+	go evsw.FireEvent("event", "data")
+	received := <-messages
+	if received != "data" {
+		t.Errorf("Message received does not match: %v", received)
+	}
+}
+
+// TestAddListenerForEventFireMany sets up an EventSwitch, subscribes a single
+// listener to an event, and sends a thousand integers.
+func TestAddListenerForEventFireMany(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	doneSum := make(chan uint64)
+	doneSending := make(chan uint64)
+	numbers := make(chan uint64, 4)
+	// subscribe one listener for one event
+	evsw.AddListenerForEvent("listener", "event",
+		func(data EventData) {
+			numbers <- data.(uint64)
+		})
+	// collect received events
+	go sumReceivedNumbers(numbers, doneSum)
+	// go fire events
+	go fireEvents(evsw, "event", doneSending, uint64(1))
+	checkSum := <-doneSending
+	close(numbers)
+	eventSum := <-doneSum
+	if checkSum != eventSum {
+		t.Errorf("Not all messages sent were received.\n")
+	}
+}
+
+// TestAddListenerForDifferentEvents sets up an EventSwitch, subscribes a single
+// listener to three different events and sends a thousand integers for each
+// of the three events.
+func TestAddListenerForDifferentEvents(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	doneSum := make(chan uint64)
+	doneSending1 := make(chan uint64)
+	doneSending2 := make(chan uint64)
+	doneSending3 := make(chan uint64)
+	numbers := make(chan uint64, 4)
+	// subscribe one listener to three events
+	evsw.AddListenerForEvent("listener", "event1",
+		func(data EventData) {
+			numbers <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener", "event2",
+		func(data EventData) {
+			numbers <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener", "event3",
+		func(data EventData) {
+			numbers <- data.(uint64)
+		})
+	// collect received events
+	go sumReceivedNumbers(numbers, doneSum)
+	// go fire events
+	go fireEvents(evsw, "event1", doneSending1, uint64(1))
+	go fireEvents(evsw, "event2", doneSending2, uint64(1))
+	go fireEvents(evsw, "event3", doneSending3, uint64(1))
+	var checkSum uint64 = 0
+	checkSum += <-doneSending1
+	checkSum += <-doneSending2
+	checkSum += <-doneSending3
+	close(numbers)
+	eventSum := <-doneSum
+	if checkSum != eventSum {
+		t.Errorf("Not all messages sent were received.\n")
+	}
+}
+
+// TestAddDifferentListenerForDifferentEvents sets up an EventSwitch,
+// subscribes a first listener to three events, and subscribes a second
+// listener to two of those three events, and then sends a thousand integers
+// for each of the three events.
+func TestAddDifferentListenerForDifferentEvents(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	doneSum1 := make(chan uint64)
+	doneSum2 := make(chan uint64)
+	doneSending1 := make(chan uint64)
+	doneSending2 := make(chan uint64)
+	doneSending3 := make(chan uint64)
+	numbers1 := make(chan uint64, 4)
+	numbers2 := make(chan uint64, 4)
+	// subscribe two listener to three events
+	evsw.AddListenerForEvent("listener1", "event1",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener1", "event2",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener1", "event3",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener2", "event2",
+		func(data EventData) {
+			numbers2 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener2", "event3",
+		func(data EventData) {
+			numbers2 <- data.(uint64)
+		})
+	// collect received events for listener1
+	go sumReceivedNumbers(numbers1, doneSum1)
+	// collect received events for listener2
+	go sumReceivedNumbers(numbers2, doneSum2)
+	// go fire events
+	go fireEvents(evsw, "event1", doneSending1, uint64(1))
+	go fireEvents(evsw, "event2", doneSending2, uint64(1001))
+	go fireEvents(evsw, "event3", doneSending3, uint64(2001))
+	checkSumEvent1 := <-doneSending1
+	checkSumEvent2 := <-doneSending2
+	checkSumEvent3 := <-doneSending3
+	checkSum1 := checkSumEvent1 + checkSumEvent2 + checkSumEvent3
+	checkSum2 := checkSumEvent2 + checkSumEvent3
+	close(numbers1)
+	close(numbers2)
+	eventSum1 := <-doneSum1
+	eventSum2 := <-doneSum2
+	if checkSum1 != eventSum1 ||
+		checkSum2 != eventSum2 {
+		t.Errorf("Not all messages sent were received for different listeners to different events.\n")
+	}
+}
+
+// TestAddAndRemoveListener sets up an EventSwitch, subscribes a listener to
+// two events, fires a thousand integers for the first event, then unsubscribes
+// the listener and fires a thousand integers for the second event.
+func TestAddAndRemoveListener(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	doneSum1 := make(chan uint64)
+	doneSum2 := make(chan uint64)
+	doneSending1 := make(chan uint64)
+	doneSending2 := make(chan uint64)
+	numbers1 := make(chan uint64, 4)
+	numbers2 := make(chan uint64, 4)
+	// subscribe two listener to three events
+	evsw.AddListenerForEvent("listener", "event1",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener", "event2",
+		func(data EventData) {
+			numbers2 <- data.(uint64)
+		})
+	// collect received events for event1
+	go sumReceivedNumbers(numbers1, doneSum1)
+	// collect received events for event2
+	go sumReceivedNumbers(numbers2, doneSum2)
+	// go fire events
+	go fireEvents(evsw, "event1", doneSending1, uint64(1))
+	checkSumEvent1 := <-doneSending1
+	// after sending all event1, unsubscribe for all events
+	evsw.RemoveListener("listener")
+	go fireEvents(evsw, "event2", doneSending2, uint64(1001))
+	checkSumEvent2 := <-doneSending2
+	close(numbers1)
+	close(numbers2)
+	eventSum1 := <-doneSum1
+	eventSum2 := <-doneSum2
+	if checkSumEvent1 != eventSum1 ||
+		// correct value asserted by preceding tests, suffices to be non-zero
+		checkSumEvent2 == uint64(0) ||
+		eventSum2 != uint64(0) {
+		t.Errorf("Not all messages sent were received or unsubscription did not register.\n")
+	}
+}
+
+// TestRemoveListener does basic tests on adding and removing
+func TestRemoveListener(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	count := 10
+	sum1, sum2 := 0, 0
+	// add some listeners and make sure they work
+	evsw.AddListenerForEvent("listener", "event1",
+		func(data EventData) {
+			sum1++
+		})
+	evsw.AddListenerForEvent("listener", "event2",
+		func(data EventData) {
+			sum2++
+		})
+	for i := 0; i < count; i++ {
+		evsw.FireEvent("event1", true)
+		evsw.FireEvent("event2", true)
+	}
+	assert.Equal(t, count, sum1)
+	assert.Equal(t, count, sum2)
+
+	// remove one by event and make sure it is gone
+	evsw.RemoveListenerForEvent("event2", "listener")
+	for i := 0; i < count; i++ {
+		evsw.FireEvent("event1", true)
+		evsw.FireEvent("event2", true)
+	}
+	assert.Equal(t, count*2, sum1)
+	assert.Equal(t, count, sum2)
+
+	// remove the listener entirely and make sure both gone
+	evsw.RemoveListener("listener")
+	for i := 0; i < count; i++ {
+		evsw.FireEvent("event1", true)
+		evsw.FireEvent("event2", true)
+	}
+	assert.Equal(t, count*2, sum1)
+	assert.Equal(t, count, sum2)
+}
+
+// TestAddAndRemoveListenersAsync sets up an EventSwitch, subscribes two
+// listeners to three events, and fires a thousand integers for each event.
+// These two listeners serve as the baseline validation while other listeners
+// are randomly subscribed and unsubscribed.
+// More precisely it randomly subscribes new listeners (different from the first
+// two listeners) to one of these three events. At the same time it starts
+// randomly unsubscribing these additional listeners from all events they are
+// at that point subscribed to.
+// NOTE: it is important to run this test with race conditions tracking on,
+// `go test -race`, to examine for possible race conditions.
+func TestRemoveListenersAsync(t *testing.T) {
+	evsw := NewEventSwitch()
+	err := evsw.Start()
+	if err != nil {
+		t.Errorf("Failed to start EventSwitch, error: %v", err)
+	}
+	doneSum1 := make(chan uint64)
+	doneSum2 := make(chan uint64)
+	doneSending1 := make(chan uint64)
+	doneSending2 := make(chan uint64)
+	doneSending3 := make(chan uint64)
+	numbers1 := make(chan uint64, 4)
+	numbers2 := make(chan uint64, 4)
+	// subscribe two listener to three events
+	evsw.AddListenerForEvent("listener1", "event1",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener1", "event2",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener1", "event3",
+		func(data EventData) {
+			numbers1 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener2", "event1",
+		func(data EventData) {
+			numbers2 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener2", "event2",
+		func(data EventData) {
+			numbers2 <- data.(uint64)
+		})
+	evsw.AddListenerForEvent("listener2", "event3",
+		func(data EventData) {
+			numbers2 <- data.(uint64)
+		})
+	// collect received events for event1
+	go sumReceivedNumbers(numbers1, doneSum1)
+	// collect received events for event2
+	go sumReceivedNumbers(numbers2, doneSum2)
+	addListenersStress := func() {
+		s1 := rand.NewSource(time.Now().UnixNano())
+		r1 := rand.New(s1)
+		for k := uint16(0); k < 400; k++ {
+			listenerNumber := r1.Intn(100) + 3
+			eventNumber := r1.Intn(3) + 1
+			go evsw.AddListenerForEvent(fmt.Sprintf("listener%v", listenerNumber),
+				fmt.Sprintf("event%v", eventNumber),
+				func(_ EventData) {})
+		}
+	}
+	removeListenersStress := func() {
+		s2 := rand.NewSource(time.Now().UnixNano())
+		r2 := rand.New(s2)
+		for k := uint16(0); k < 80; k++ {
+			listenerNumber := r2.Intn(100) + 3
+			go evsw.RemoveListener(fmt.Sprintf("listener%v", listenerNumber))
+		}
+	}
+	addListenersStress()
+	// go fire events
+	go fireEvents(evsw, "event1", doneSending1, uint64(1))
+	removeListenersStress()
+	go fireEvents(evsw, "event2", doneSending2, uint64(1001))
+	go fireEvents(evsw, "event3", doneSending3, uint64(2001))
+	checkSumEvent1 := <-doneSending1
+	checkSumEvent2 := <-doneSending2
+	checkSumEvent3 := <-doneSending3
+	checkSum := checkSumEvent1 + checkSumEvent2 + checkSumEvent3
+	close(numbers1)
+	close(numbers2)
+	eventSum1 := <-doneSum1
+	eventSum2 := <-doneSum2
+	if checkSum != eventSum1 ||
+		checkSum != eventSum2 {
+		t.Errorf("Not all messages sent were received.\n")
+	}
+}
+
+//------------------------------------------------------------------------------
+// Helper functions
+
+// sumReceivedNumbers takes two channels and adds all numbers received
+// until the receiving channel `numbers` is closed; it then sends the sum
+// on `doneSum` and closes that channel.  Expected to be run in a go-routine.
+func sumReceivedNumbers(numbers, doneSum chan uint64) {
+	var sum uint64 = 0
+	for {
+		j, more := <-numbers
+		sum += j
+		if !more {
+			doneSum <- sum
+			close(doneSum)
+			return
+		}
+	}
+}
+
+// fireEvents takes an EventSwitch and fires a thousand integers under
+// a given `event` with the integers mootonically increasing from `offset`
+// to `offset` + 999.  It additionally returns the addition of all integers
+// sent on `doneChan` for assertion that all events have been sent, and enabling
+// the test to assert all events have also been received.
+func fireEvents(evsw EventSwitch, event string, doneChan chan uint64,
+	offset uint64) {
+	var sentSum uint64 = 0
+	for i := offset; i <= offset+uint64(999); i++ {
+		sentSum += i
+		evsw.FireEvent(event, i)
+	}
+	doneChan <- sentSum
+	close(doneChan)
+}

libs/pubsub/example_test.go

@@ -0,0 +1,28 @@
+package pubsub_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+)
+
+func TestExample(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	ch := make(chan interface{}, 1)
+	err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name='John'"), ch)
+	require.NoError(t, err)
+	err = s.PublishWithTags(ctx, "Tombstone", pubsub.NewTagMap(map[string]string{"abci.account.name": "John"}))
+	require.NoError(t, err)
+	assertReceive(t, "Tombstone", ch)
+}

libs/pubsub/pubsub.go

@@ -0,0 +1,344 @@
+// Package pubsub implements a pub-sub model with a single publisher (Server)
+// and multiple subscribers (clients).
+//
+// Though you can have multiple publishers by sharing a pointer to a server or
+// by giving the same channel to each publisher and publishing messages from
+// that channel (fan-in).
+//
+// Clients subscribe for messages, which could be of any type, using a query.
+// When some message is published, we match it with all queries. If there is a
+// match, this message will be pushed to all clients, subscribed to that query.
+// See query subpackage for our implementation.
+package pubsub
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+type operation int
+
+const (
+	sub operation = iota
+	pub
+	unsub
+	shutdown
+)
+
+var (
+	// ErrSubscriptionNotFound is returned when a client tries to unsubscribe
+	// from not existing subscription.
+	ErrSubscriptionNotFound = errors.New("subscription not found")
+
+	// ErrAlreadySubscribed is returned when a client tries to subscribe twice or
+	// more using the same query.
+	ErrAlreadySubscribed = errors.New("already subscribed")
+)
+
+type cmd struct {
+	op       operation
+	query    Query
+	ch       chan<- interface{}
+	clientID string
+	msg      interface{}
+	tags     TagMap
+}
+
+// Query defines an interface for a query to be used for subscribing.
+type Query interface {
+	Matches(tags TagMap) bool
+	String() string
+}
+
+// Server allows clients to subscribe/unsubscribe for messages, publishing
+// messages with or without tags, and manages internal state.
+type Server struct {
+	cmn.BaseService
+
+	cmds    chan cmd
+	cmdsCap int
+
+	mtx           sync.RWMutex
+	subscriptions map[string]map[string]Query // subscriber -> query (string) -> Query
+}
+
+// Option sets a parameter for the server.
+type Option func(*Server)
+
+// TagMap is used to associate tags to a message.
+// They can be queried by subscribers to choose messages they will received.
+type TagMap interface {
+	// Get returns the value for a key, or nil if no value is present.
+	// The ok result indicates whether value was found in the tags.
+	Get(key string) (value string, ok bool)
+	// Len returns the number of tags.
+	Len() int
+}
+
+type tagMap map[string]string
+
+var _ TagMap = (*tagMap)(nil)
+
+// NewTagMap constructs a new immutable tag set from a map.
+func NewTagMap(data map[string]string) TagMap {
+	return tagMap(data)
+}
+
+// Get returns the value for a key, or nil if no value is present.
+// The ok result indicates whether value was found in the tags.
+func (ts tagMap) Get(key string) (value string, ok bool) {
+	value, ok = ts[key]
+	return
+}
+
+// Len returns the number of tags.
+func (ts tagMap) Len() int {
+	return len(ts)
+}
+
+// NewServer returns a new server. See the commentary on the Option functions
+// for a detailed description of how to configure buffering. If no options are
+// provided, the resulting server's queue is unbuffered.
+func NewServer(options ...Option) *Server {
+	s := &Server{
+		subscriptions: make(map[string]map[string]Query),
+	}
+	s.BaseService = *cmn.NewBaseService(nil, "PubSub", s)
+
+	for _, option := range options {
+		option(s)
+	}
+
+	// if BufferCapacity option was not set, the channel is unbuffered
+	s.cmds = make(chan cmd, s.cmdsCap)
+
+	return s
+}
+
+// BufferCapacity allows you to specify capacity for the internal server's
+// queue. Since the server, given Y subscribers, could only process X messages,
+// this option could be used to survive spikes (e.g. high amount of
+// transactions during peak hours).
+func BufferCapacity(cap int) Option {
+	return func(s *Server) {
+		if cap > 0 {
+			s.cmdsCap = cap
+		}
+	}
+}
+
+// BufferCapacity returns capacity of the internal server's queue.
+func (s *Server) BufferCapacity() int {
+	return s.cmdsCap
+}
+
+// Subscribe creates a subscription for the given client. It accepts a channel
+// on which messages matching the given query can be received. An error will be
+// returned to the caller if the context is canceled or if subscription already
+// exist for pair clientID and query.
+func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, out chan<- interface{}) error {
+	s.mtx.RLock()
+	clientSubscriptions, ok := s.subscriptions[clientID]
+	if ok {
+		_, ok = clientSubscriptions[query.String()]
+	}
+	s.mtx.RUnlock()
+	if ok {
+		return ErrAlreadySubscribed
+	}
+
+	select {
+	case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}:
+		s.mtx.Lock()
+		if _, ok = s.subscriptions[clientID]; !ok {
+			s.subscriptions[clientID] = make(map[string]Query)
+		}
+		s.subscriptions[clientID][query.String()] = query
+		s.mtx.Unlock()
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// Unsubscribe removes the subscription on the given query. An error will be
+// returned to the caller if the context is canceled or if subscription does
+// not exist.
+func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error {
+	var origQuery Query
+	s.mtx.RLock()
+	clientSubscriptions, ok := s.subscriptions[clientID]
+	if ok {
+		origQuery, ok = clientSubscriptions[query.String()]
+	}
+	s.mtx.RUnlock()
+	if !ok {
+		return ErrSubscriptionNotFound
+	}
+
+	// original query is used here because we're using pointers as map keys
+	select {
+	case s.cmds <- cmd{op: unsub, clientID: clientID, query: origQuery}:
+		s.mtx.Lock()
+		delete(clientSubscriptions, query.String())
+		s.mtx.Unlock()
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// UnsubscribeAll removes all client subscriptions. An error will be returned
+// to the caller if the context is canceled or if subscription does not exist.
+func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error {
+	s.mtx.RLock()
+	_, ok := s.subscriptions[clientID]
+	s.mtx.RUnlock()
+	if !ok {
+		return ErrSubscriptionNotFound
+	}
+
+	select {
+	case s.cmds <- cmd{op: unsub, clientID: clientID}:
+		s.mtx.Lock()
+		delete(s.subscriptions, clientID)
+		s.mtx.Unlock()
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// Publish publishes the given message. An error will be returned to the caller
+// if the context is canceled.
+func (s *Server) Publish(ctx context.Context, msg interface{}) error {
+	return s.PublishWithTags(ctx, msg, NewTagMap(make(map[string]string)))
+}
+
+// PublishWithTags publishes the given message with the set of tags. The set is
+// matched with clients queries. If there is a match, the message is sent to
+// the client.
+func (s *Server) PublishWithTags(ctx context.Context, msg interface{}, tags TagMap) error {
+	select {
+	case s.cmds <- cmd{op: pub, msg: msg, tags: tags}:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// OnStop implements Service.OnStop by shutting down the server.
+func (s *Server) OnStop() {
+	s.cmds <- cmd{op: shutdown}
+}
+
+// NOTE: not goroutine safe
+type state struct {
+	// query -> client -> ch
+	queries map[Query]map[string]chan<- interface{}
+	// client -> query -> struct{}
+	clients map[string]map[Query]struct{}
+}
+
+// OnStart implements Service.OnStart by starting the server.
+func (s *Server) OnStart() error {
+	go s.loop(state{
+		queries: make(map[Query]map[string]chan<- interface{}),
+		clients: make(map[string]map[Query]struct{}),
+	})
+	return nil
+}
+
+// OnReset implements Service.OnReset
+func (s *Server) OnReset() error {
+	return nil
+}
+
+func (s *Server) loop(state state) {
+loop:
+	for cmd := range s.cmds {
+		switch cmd.op {
+		case unsub:
+			if cmd.query != nil {
+				state.remove(cmd.clientID, cmd.query)
+			} else {
+				state.removeAll(cmd.clientID)
+			}
+		case shutdown:
+			for clientID := range state.clients {
+				state.removeAll(clientID)
+			}
+			break loop
+		case sub:
+			state.add(cmd.clientID, cmd.query, cmd.ch)
+		case pub:
+			state.send(cmd.msg, cmd.tags)
+		}
+	}
+}
+
+func (state *state) add(clientID string, q Query, ch chan<- interface{}) {
+	// add query if needed
+	if _, ok := state.queries[q]; !ok {
+		state.queries[q] = make(map[string]chan<- interface{})
+	}
+
+	// create subscription
+	state.queries[q][clientID] = ch
+
+	// add client if needed
+	if _, ok := state.clients[clientID]; !ok {
+		state.clients[clientID] = make(map[Query]struct{})
+	}
+	state.clients[clientID][q] = struct{}{}
+}
+
+func (state *state) remove(clientID string, q Query) {
+	clientToChannelMap, ok := state.queries[q]
+	if !ok {
+		return
+	}
+
+	ch, ok := clientToChannelMap[clientID]
+	if ok {
+		close(ch)
+
+		delete(state.clients[clientID], q)
+
+		// if it not subscribed to anything else, remove the client
+		if len(state.clients[clientID]) == 0 {
+			delete(state.clients, clientID)
+		}
+
+		delete(state.queries[q], clientID)
+	}
+}
+
+func (state *state) removeAll(clientID string) {
+	queryMap, ok := state.clients[clientID]
+	if !ok {
+		return
+	}
+
+	for q := range queryMap {
+		ch := state.queries[q][clientID]
+		close(ch)
+
+		delete(state.queries[q], clientID)
+	}
+
+	delete(state.clients, clientID)
+}
+
+func (state *state) send(msg interface{}, tags TagMap) {
+	for q, clientToChannelMap := range state.queries {
+		if q.Matches(tags) {
+			for _, ch := range clientToChannelMap {
+				ch <- msg
+			}
+		}
+	}
+}

libs/pubsub/pubsub_test.go

@@ -0,0 +1,253 @@
+package pubsub_test
+
+import (
+	"context"
+	"fmt"
+	"runtime/debug"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+)
+
+const (
+	clientID = "test-client"
+)
+
+func TestSubscribe(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	ch := make(chan interface{}, 1)
+	err := s.Subscribe(ctx, clientID, query.Empty{}, ch)
+	require.NoError(t, err)
+	err = s.Publish(ctx, "Ka-Zar")
+	require.NoError(t, err)
+	assertReceive(t, "Ka-Zar", ch)
+
+	err = s.Publish(ctx, "Quicksilver")
+	require.NoError(t, err)
+	assertReceive(t, "Quicksilver", ch)
+}
+
+func TestDifferentClients(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	ch1 := make(chan interface{}, 1)
+	err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type='NewBlock'"), ch1)
+	require.NoError(t, err)
+	err = s.PublishWithTags(ctx, "Iceman", pubsub.NewTagMap(map[string]string{"tm.events.type": "NewBlock"}))
+	require.NoError(t, err)
+	assertReceive(t, "Iceman", ch1)
+
+	ch2 := make(chan interface{}, 1)
+	err = s.Subscribe(ctx, "client-2", query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), ch2)
+	require.NoError(t, err)
+	err = s.PublishWithTags(ctx, "Ultimo", pubsub.NewTagMap(map[string]string{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}))
+	require.NoError(t, err)
+	assertReceive(t, "Ultimo", ch1)
+	assertReceive(t, "Ultimo", ch2)
+
+	ch3 := make(chan interface{}, 1)
+	err = s.Subscribe(ctx, "client-3", query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), ch3)
+	require.NoError(t, err)
+	err = s.PublishWithTags(ctx, "Valeria Richards", pubsub.NewTagMap(map[string]string{"tm.events.type": "NewRoundStep"}))
+	require.NoError(t, err)
+	assert.Zero(t, len(ch3))
+}
+
+func TestClientSubscribesTwice(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	q := query.MustParse("tm.events.type='NewBlock'")
+
+	ch1 := make(chan interface{}, 1)
+	err := s.Subscribe(ctx, clientID, q, ch1)
+	require.NoError(t, err)
+	err = s.PublishWithTags(ctx, "Goblin Queen", pubsub.NewTagMap(map[string]string{"tm.events.type": "NewBlock"}))
+	require.NoError(t, err)
+	assertReceive(t, "Goblin Queen", ch1)
+
+	ch2 := make(chan interface{}, 1)
+	err = s.Subscribe(ctx, clientID, q, ch2)
+	require.Error(t, err)
+
+	err = s.PublishWithTags(ctx, "Spider-Man", pubsub.NewTagMap(map[string]string{"tm.events.type": "NewBlock"}))
+	require.NoError(t, err)
+	assertReceive(t, "Spider-Man", ch1)
+}
+
+func TestUnsubscribe(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	ch := make(chan interface{})
+	err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'"), ch)
+	require.NoError(t, err)
+	err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'"))
+	require.NoError(t, err)
+
+	err = s.Publish(ctx, "Nick Fury")
+	require.NoError(t, err)
+	assert.Zero(t, len(ch), "Should not receive anything after Unsubscribe")
+
+	_, ok := <-ch
+	assert.False(t, ok)
+}
+
+func TestResubscribe(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	ch := make(chan interface{})
+	err := s.Subscribe(ctx, clientID, query.Empty{}, ch)
+	require.NoError(t, err)
+	err = s.Unsubscribe(ctx, clientID, query.Empty{})
+	require.NoError(t, err)
+	ch = make(chan interface{})
+	err = s.Subscribe(ctx, clientID, query.Empty{}, ch)
+	require.NoError(t, err)
+
+	err = s.Publish(ctx, "Cable")
+	require.NoError(t, err)
+	assertReceive(t, "Cable", ch)
+}
+
+func TestUnsubscribeAll(t *testing.T) {
+	s := pubsub.NewServer()
+	s.SetLogger(log.TestingLogger())
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	ch1, ch2 := make(chan interface{}, 1), make(chan interface{}, 1)
+	err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'"), ch1)
+	require.NoError(t, err)
+	err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlockHeader'"), ch2)
+	require.NoError(t, err)
+
+	err = s.UnsubscribeAll(ctx, clientID)
+	require.NoError(t, err)
+
+	err = s.Publish(ctx, "Nick Fury")
+	require.NoError(t, err)
+	assert.Zero(t, len(ch1), "Should not receive anything after UnsubscribeAll")
+	assert.Zero(t, len(ch2), "Should not receive anything after UnsubscribeAll")
+
+	_, ok := <-ch1
+	assert.False(t, ok)
+	_, ok = <-ch2
+	assert.False(t, ok)
+}
+
+func TestBufferCapacity(t *testing.T) {
+	s := pubsub.NewServer(pubsub.BufferCapacity(2))
+	s.SetLogger(log.TestingLogger())
+
+	assert.Equal(t, 2, s.BufferCapacity())
+
+	ctx := context.Background()
+	err := s.Publish(ctx, "Nighthawk")
+	require.NoError(t, err)
+	err = s.Publish(ctx, "Sage")
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Millisecond)
+	defer cancel()
+	err = s.Publish(ctx, "Ironclad")
+	if assert.Error(t, err) {
+		assert.Equal(t, context.DeadlineExceeded, err)
+	}
+}
+
+func Benchmark10Clients(b *testing.B)   { benchmarkNClients(10, b) }
+func Benchmark100Clients(b *testing.B)  { benchmarkNClients(100, b) }
+func Benchmark1000Clients(b *testing.B) { benchmarkNClients(1000, b) }
+
+func Benchmark10ClientsOneQuery(b *testing.B)   { benchmarkNClientsOneQuery(10, b) }
+func Benchmark100ClientsOneQuery(b *testing.B)  { benchmarkNClientsOneQuery(100, b) }
+func Benchmark1000ClientsOneQuery(b *testing.B) { benchmarkNClientsOneQuery(1000, b) }
+
+func benchmarkNClients(n int, b *testing.B) {
+	s := pubsub.NewServer()
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	for i := 0; i < n; i++ {
+		ch := make(chan interface{})
+		go func() {
+			for range ch {
+			}
+		}()
+		s.Subscribe(ctx, clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), ch)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		s.PublishWithTags(ctx, "Gamora", pubsub.NewTagMap(map[string]string{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": string(i)}))
+	}
+}
+
+func benchmarkNClientsOneQuery(n int, b *testing.B) {
+	s := pubsub.NewServer()
+	s.Start()
+	defer s.Stop()
+
+	ctx := context.Background()
+	q := query.MustParse("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1")
+	for i := 0; i < n; i++ {
+		ch := make(chan interface{})
+		go func() {
+			for range ch {
+			}
+		}()
+		s.Subscribe(ctx, clientID, q, ch)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		s.PublishWithTags(ctx, "Gamora", pubsub.NewTagMap(map[string]string{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": "1"}))
+	}
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// HELPERS
+///////////////////////////////////////////////////////////////////////////////
+
+func assertReceive(t *testing.T, expected interface{}, ch <-chan interface{}, msgAndArgs ...interface{}) {
+	select {
+	case actual := <-ch:
+		if actual != nil {
+			assert.Equal(t, expected, actual, msgAndArgs...)
+		}
+	case <-time.After(1 * time.Second):
+		t.Errorf("Expected to receive %v from the channel, got nothing after 1s", expected)
+		debug.PrintStack()
+	}
+}

libs/pubsub/query/Makefile

@@ -0,0 +1,11 @@
+gen_query_parser:
+	@go get github.com/pointlander/peg
+	peg -inline -switch query.peg
+
+fuzzy_test:
+	@go get github.com/dvyukov/go-fuzz/go-fuzz
+	@go get github.com/dvyukov/go-fuzz/go-fuzz-build
+	go-fuzz-build github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test
+	go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output
+
+.PHONY: gen_query_parser fuzzy_test

libs/pubsub/query/empty.go

@@ -0,0 +1,16 @@
+package query
+
+import "github.com/tendermint/tendermint/libs/pubsub"
+
+// Empty query matches any set of tags.
+type Empty struct {
+}
+
+// Matches always returns true.
+func (Empty) Matches(tags pubsub.TagMap) bool {
+	return true
+}
+
+func (Empty) String() string {
+	return "empty"
+}

libs/pubsub/query/empty_test.go

@@ -0,0 +1,18 @@
+package query_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+)
+
+func TestEmptyQueryMatchesAnything(t *testing.T) {
+	q := query.Empty{}
+	assert.True(t, q.Matches(pubsub.NewTagMap(map[string]string{})))
+	assert.True(t, q.Matches(pubsub.NewTagMap(map[string]string{"Asher": "Roth"})))
+	assert.True(t, q.Matches(pubsub.NewTagMap(map[string]string{"Route": "66"})))
+	assert.True(t, q.Matches(pubsub.NewTagMap(map[string]string{"Route": "66", "Billy": "Blue"})))
+}

libs/pubsub/query/fuzz_test/main.go

@@ -0,0 +1,30 @@
+package fuzz_test
+
+import (
+	"fmt"
+
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+)
+
+func Fuzz(data []byte) int {
+	sdata := string(data)
+	q0, err := query.New(sdata)
+	if err != nil {
+		return 0
+	}
+
+	sdata1 := q0.String()
+	q1, err := query.New(sdata1)
+	if err != nil {
+		panic(err)
+	}
+
+	sdata2 := q1.String()
+	if sdata1 != sdata2 {
+		fmt.Printf("q0: %q\n", sdata1)
+		fmt.Printf("q1: %q\n", sdata2)
+		panic("query changed")
+	}
+
+	return 1
+}

libs/pubsub/query/parser_test.go

@@ -0,0 +1,92 @@
+package query_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+)
+
+// TODO: fuzzy testing?
+func TestParser(t *testing.T) {
+	cases := []struct {
+		query string
+		valid bool
+	}{
+		{"tm.events.type='NewBlock'", true},
+		{"tm.events.type = 'NewBlock'", true},
+		{"tm.events.name = ''", true},
+		{"tm.events.type='TIME'", true},
+		{"tm.events.type='DATE'", true},
+		{"tm.events.type='='", true},
+		{"tm.events.type='TIME", false},
+		{"tm.events.type=TIME'", false},
+		{"tm.events.type==", false},
+		{"tm.events.type=NewBlock", false},
+		{">==", false},
+		{"tm.events.type 'NewBlock' =", false},
+		{"tm.events.type>'NewBlock'", false},
+		{"", false},
+		{"=", false},
+		{"='NewBlock'", false},
+		{"tm.events.type=", false},
+
+		{"tm.events.typeNewBlock", false},
+		{"tm.events.type'NewBlock'", false},
+		{"'NewBlock'", false},
+		{"NewBlock", false},
+		{"", false},
+
+		{"tm.events.type='NewBlock' AND abci.account.name='Igor'", true},
+		{"tm.events.type='NewBlock' AND", false},
+		{"tm.events.type='NewBlock' AN", false},
+		{"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false},
+		{"AND tm.events.type='NewBlock' ", false},
+
+		{"abci.account.name CONTAINS 'Igor'", true},
+
+		{"tx.date > DATE 2013-05-03", true},
+		{"tx.date < DATE 2013-05-03", true},
+		{"tx.date <= DATE 2013-05-03", true},
+		{"tx.date >= DATE 2013-05-03", true},
+		{"tx.date >= DAT 2013-05-03", false},
+		{"tx.date <= DATE2013-05-03", false},
+		{"tx.date <= DATE -05-03", false},
+		{"tx.date >= DATE 20130503", false},
+		{"tx.date >= DATE 2013+01-03", false},
+		// incorrect year, month, day
+		{"tx.date >= DATE 0013-01-03", false},
+		{"tx.date >= DATE 2013-31-03", false},
+		{"tx.date >= DATE 2013-01-83", false},
+
+		{"tx.date > TIME 2013-05-03T14:45:00+07:00", true},
+		{"tx.date < TIME 2013-05-03T14:45:00-02:00", true},
+		{"tx.date <= TIME 2013-05-03T14:45:00Z", true},
+		{"tx.date >= TIME 2013-05-03T14:45:00Z", true},
+		{"tx.date >= TIME2013-05-03T14:45:00Z", false},
+		{"tx.date = IME 2013-05-03T14:45:00Z", false},
+		{"tx.date = TIME 2013-05-:45:00Z", false},
+		{"tx.date >= TIME 2013-05-03T14:45:00", false},
+		{"tx.date >= TIME 0013-00-00T14:45:00Z", false},
+		{"tx.date >= TIME 2013+05=03T14:45:00Z", false},
+
+		{"account.balance=100", true},
+		{"account.balance >= 200", true},
+		{"account.balance >= -300", false},
+		{"account.balance >>= 400", false},
+		{"account.balance=33.22.1", false},
+
+		{"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true},
+		{"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false},
+	}
+
+	for _, c := range cases {
+		_, err := query.New(c.query)
+		if c.valid {
+			assert.NoErrorf(t, err, "Query was '%s'", c.query)
+		} else {
+			assert.Errorf(t, err, "Query was '%s'", c.query)
+		}
+	}
+}

libs/pubsub/query/query.go

@@ -0,0 +1,339 @@
+// Package query provides a parser for a custom query format:
+//
+//		abci.invoice.number=22 AND abci.invoice.owner=Ivan
+//
+// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar.
+// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics
+//
+// It has a support for numbers (integer and floating point), dates and times.
+package query
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/pubsub"
+)
+
+// Query holds the query string and the query parser.
+type Query struct {
+	str    string
+	parser *QueryParser
+}
+
+// Condition represents a single condition within a query and consists of tag
+// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7").
+type Condition struct {
+	Tag     string
+	Op      Operator
+	Operand interface{}
+}
+
+// New parses the given string and returns a query or error if the string is
+// invalid.
+func New(s string) (*Query, error) {
+	p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)}
+	p.Init()
+	if err := p.Parse(); err != nil {
+		return nil, err
+	}
+	return &Query{str: s, parser: p}, nil
+}
+
+// MustParse turns the given string into a query or panics; for tests or others
+// cases where you know the string is valid.
+func MustParse(s string) *Query {
+	q, err := New(s)
+	if err != nil {
+		panic(fmt.Sprintf("failed to parse %s: %v", s, err))
+	}
+	return q
+}
+
+// String returns the original string.
+func (q *Query) String() string {
+	return q.str
+}
+
+// Operator is an operator that defines some kind of relation between tag and
+// operand (equality, etc.).
+type Operator uint8
+
+const (
+	// "<="
+	OpLessEqual Operator = iota
+	// ">="
+	OpGreaterEqual
+	// "<"
+	OpLess
+	// ">"
+	OpGreater
+	// "="
+	OpEqual
+	// "CONTAINS"; used to check if a string contains a certain sub string.
+	OpContains
+)
+
+const (
+	// DateLayout defines a layout for all dates (`DATE date`)
+	DateLayout = "2006-01-02"
+	// TimeLayout defines a layout for all times (`TIME time`)
+	TimeLayout = time.RFC3339
+)
+
+// Conditions returns a list of conditions.
+func (q *Query) Conditions() []Condition {
+	conditions := make([]Condition, 0)
+
+	buffer, begin, end := q.parser.Buffer, 0, 0
+
+	var tag string
+	var op Operator
+
+	// tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7")
+	for _, token := range q.parser.Tokens() {
+		switch token.pegRule {
+
+		case rulePegText:
+			begin, end = int(token.begin), int(token.end)
+		case ruletag:
+			tag = buffer[begin:end]
+		case rulele:
+			op = OpLessEqual
+		case rulege:
+			op = OpGreaterEqual
+		case rulel:
+			op = OpLess
+		case ruleg:
+			op = OpGreater
+		case ruleequal:
+			op = OpEqual
+		case rulecontains:
+			op = OpContains
+		case rulevalue:
+			// strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock")
+			valueWithoutSingleQuotes := buffer[begin+1 : end-1]
+			conditions = append(conditions, Condition{tag, op, valueWithoutSingleQuotes})
+		case rulenumber:
+			number := buffer[begin:end]
+			if strings.ContainsAny(number, ".") { // if it looks like a floating-point number
+				value, err := strconv.ParseFloat(number, 64)
+				if err != nil {
+					panic(fmt.Sprintf("got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", err, number))
+				}
+				conditions = append(conditions, Condition{tag, op, value})
+			} else {
+				value, err := strconv.ParseInt(number, 10, 64)
+				if err != nil {
+					panic(fmt.Sprintf("got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", err, number))
+				}
+				conditions = append(conditions, Condition{tag, op, value})
+			}
+		case ruletime:
+			value, err := time.Parse(TimeLayout, buffer[begin:end])
+			if err != nil {
+				panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", err, buffer[begin:end]))
+			}
+			conditions = append(conditions, Condition{tag, op, value})
+		case ruledate:
+			value, err := time.Parse("2006-01-02", buffer[begin:end])
+			if err != nil {
+				panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", err, buffer[begin:end]))
+			}
+			conditions = append(conditions, Condition{tag, op, value})
+		}
+	}
+
+	return conditions
+}
+
+// Matches returns true if the query matches the given set of tags, false otherwise.
+//
+// For example, query "name=John" matches tags = {"name": "John"}. More
+// examples could be found in parser_test.go and query_test.go.
+func (q *Query) Matches(tags pubsub.TagMap) bool {
+	if tags.Len() == 0 {
+		return false
+	}
+
+	buffer, begin, end := q.parser.Buffer, 0, 0
+
+	var tag string
+	var op Operator
+
+	// tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7")
+	for _, token := range q.parser.Tokens() {
+		switch token.pegRule {
+
+		case rulePegText:
+			begin, end = int(token.begin), int(token.end)
+		case ruletag:
+			tag = buffer[begin:end]
+		case rulele:
+			op = OpLessEqual
+		case rulege:
+			op = OpGreaterEqual
+		case rulel:
+			op = OpLess
+		case ruleg:
+			op = OpGreater
+		case ruleequal:
+			op = OpEqual
+		case rulecontains:
+			op = OpContains
+		case rulevalue:
+			// strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock")
+			valueWithoutSingleQuotes := buffer[begin+1 : end-1]
+
+			// see if the triplet (tag, operator, operand) matches any tag
+			// "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" }
+			if !match(tag, op, reflect.ValueOf(valueWithoutSingleQuotes), tags) {
+				return false
+			}
+		case rulenumber:
+			number := buffer[begin:end]
+			if strings.ContainsAny(number, ".") { // if it looks like a floating-point number
+				value, err := strconv.ParseFloat(number, 64)
+				if err != nil {
+					panic(fmt.Sprintf("got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", err, number))
+				}
+				if !match(tag, op, reflect.ValueOf(value), tags) {
+					return false
+				}
+			} else {
+				value, err := strconv.ParseInt(number, 10, 64)
+				if err != nil {
+					panic(fmt.Sprintf("got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", err, number))
+				}
+				if !match(tag, op, reflect.ValueOf(value), tags) {
+					return false
+				}
+			}
+		case ruletime:
+			value, err := time.Parse(TimeLayout, buffer[begin:end])
+			if err != nil {
+				panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", err, buffer[begin:end]))
+			}
+			if !match(tag, op, reflect.ValueOf(value), tags) {
+				return false
+			}
+		case ruledate:
+			value, err := time.Parse("2006-01-02", buffer[begin:end])
+			if err != nil {
+				panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", err, buffer[begin:end]))
+			}
+			if !match(tag, op, reflect.ValueOf(value), tags) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+// match returns true if the given triplet (tag, operator, operand) matches any tag.
+//
+// First, it looks up the tag in tags and if it finds one, tries to compare the
+// value from it to the operand using the operator.
+//
+// "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" }
+func match(tag string, op Operator, operand reflect.Value, tags pubsub.TagMap) bool {
+	// look up the tag from the query in tags
+	value, ok := tags.Get(tag)
+	if !ok {
+		return false
+	}
+	switch operand.Kind() {
+	case reflect.Struct: // time
+		operandAsTime := operand.Interface().(time.Time)
+		// try our best to convert value from tags to time.Time
+		var (
+			v   time.Time
+			err error
+		)
+		if strings.ContainsAny(value, "T") {
+			v, err = time.Parse(TimeLayout, value)
+		} else {
+			v, err = time.Parse(DateLayout, value)
+		}
+		if err != nil {
+			panic(fmt.Sprintf("Failed to convert value %v from tag to time.Time: %v", value, err))
+		}
+		switch op {
+		case OpLessEqual:
+			return v.Before(operandAsTime) || v.Equal(operandAsTime)
+		case OpGreaterEqual:
+			return v.Equal(operandAsTime) || v.After(operandAsTime)
+		case OpLess:
+			return v.Before(operandAsTime)
+		case OpGreater:
+			return v.After(operandAsTime)
+		case OpEqual:
+			return v.Equal(operandAsTime)
+		}
+	case reflect.Float64:
+		operandFloat64 := operand.Interface().(float64)
+		var v float64
+		// try our best to convert value from tags to float64
+		v, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to convert value %v from tag to float64: %v", value, err))
+		}
+		switch op {
+		case OpLessEqual:
+			return v <= operandFloat64
+		case OpGreaterEqual:
+			return v >= operandFloat64
+		case OpLess:
+			return v < operandFloat64
+		case OpGreater:
+			return v > operandFloat64
+		case OpEqual:
+			return v == operandFloat64
+		}
+	case reflect.Int64:
+		operandInt := operand.Interface().(int64)
+		var v int64
+		// if value looks like float, we try to parse it as float
+		if strings.ContainsAny(value, ".") {
+			v1, err := strconv.ParseFloat(value, 64)
+			if err != nil {
+				panic(fmt.Sprintf("Failed to convert value %v from tag to float64: %v", value, err))
+			}
+			v = int64(v1)
+		} else {
+			var err error
+			// try our best to convert value from tags to int64
+			v, err = strconv.ParseInt(value, 10, 64)
+			if err != nil {
+				panic(fmt.Sprintf("Failed to convert value %v from tag to int64: %v", value, err))
+			}
+		}
+		switch op {
+		case OpLessEqual:
+			return v <= operandInt
+		case OpGreaterEqual:
+			return v >= operandInt
+		case OpLess:
+			return v < operandInt
+		case OpGreater:
+			return v > operandInt
+		case OpEqual:
+			return v == operandInt
+		}
+	case reflect.String:
+		switch op {
+		case OpEqual:
+			return value == operand.String()
+		case OpContains:
+			return strings.Contains(value, operand.String())
+		}
+	default:
+		panic(fmt.Sprintf("Unknown kind of operand %v", operand.Kind()))
+	}
+
+	return false
+}

libs/pubsub/query/query.peg

@@ -0,0 +1,33 @@
+package query
+
+type QueryParser Peg {
+}
+
+e <- '\"' condition ( ' '+ and ' '+ condition )* '\"' !.
+
+condition <- tag ' '* (le ' '* (number / time / date)
+                      / ge ' '* (number / time / date)
+                      / l ' '* (number / time / date)
+                      / g ' '* (number / time / date)
+                      / equal ' '* (number / time / date / value)
+                      / contains ' '* value
+                      )
+
+tag <- < (![ \t\n\r\\()"'=><] .)+ >
+value <- < '\'' (!["'] .)* '\''>
+number <- < ('0'
+           / [1-9] digit* ('.' digit*)?) >
+digit <- [0-9]
+time <- "TIME " < year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit (('-' / '+') digit digit ':' digit digit / 'Z') >
+date <- "DATE " < year '-' month '-' day >
+year <- ('1' / '2') digit digit digit
+month <- ('0' / '1') digit
+day <- ('0' / '1' / '2' / '3') digit
+and <- "AND"
+
+equal <- "="
+contains <- "CONTAINS"
+le <- "<="
+ge <- ">="
+l <- "<"
+g <- ">"

libs/pubsub/query/query_test.go

@@ -0,0 +1,87 @@
+package query_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+)
+
+func TestMatches(t *testing.T) {
+	var (
+		txDate = "2017-01-01"
+		txTime = "2018-05-03T14:45:00Z"
+	)
+
+	testCases := []struct {
+		s       string
+		tags    map[string]string
+		err     bool
+		matches bool
+	}{
+		{"tm.events.type='NewBlock'", map[string]string{"tm.events.type": "NewBlock"}, false, true},
+
+		{"tx.gas > 7", map[string]string{"tx.gas": "8"}, false, true},
+		{"tx.gas > 7 AND tx.gas < 9", map[string]string{"tx.gas": "8"}, false, true},
+		{"body.weight >= 3.5", map[string]string{"body.weight": "3.5"}, false, true},
+		{"account.balance < 1000.0", map[string]string{"account.balance": "900"}, false, true},
+		{"apples.kg <= 4", map[string]string{"apples.kg": "4.0"}, false, true},
+		{"body.weight >= 4.5", map[string]string{"body.weight": fmt.Sprintf("%v", float32(4.5))}, false, true},
+		{"oranges.kg < 4 AND watermellons.kg > 10", map[string]string{"oranges.kg": "3", "watermellons.kg": "12"}, false, true},
+		{"peaches.kg < 4", map[string]string{"peaches.kg": "5"}, false, false},
+
+		{"tx.date > DATE 2017-01-01", map[string]string{"tx.date": time.Now().Format(query.DateLayout)}, false, true},
+		{"tx.date = DATE 2017-01-01", map[string]string{"tx.date": txDate}, false, true},
+		{"tx.date = DATE 2018-01-01", map[string]string{"tx.date": txDate}, false, false},
+
+		{"tx.time >= TIME 2013-05-03T14:45:00Z", map[string]string{"tx.time": time.Now().Format(query.TimeLayout)}, false, true},
+		{"tx.time = TIME 2013-05-03T14:45:00Z", map[string]string{"tx.time": txTime}, false, false},
+
+		{"abci.owner.name CONTAINS 'Igor'", map[string]string{"abci.owner.name": "Igor,Ivan"}, false, true},
+		{"abci.owner.name CONTAINS 'Igor'", map[string]string{"abci.owner.name": "Pavel,Ivan"}, false, false},
+	}
+
+	for _, tc := range testCases {
+		q, err := query.New(tc.s)
+		if !tc.err {
+			require.Nil(t, err)
+		}
+
+		if tc.matches {
+			assert.True(t, q.Matches(pubsub.NewTagMap(tc.tags)), "Query '%s' should match %v", tc.s, tc.tags)
+		} else {
+			assert.False(t, q.Matches(pubsub.NewTagMap(tc.tags)), "Query '%s' should not match %v", tc.s, tc.tags)
+		}
+	}
+}
+
+func TestMustParse(t *testing.T) {
+	assert.Panics(t, func() { query.MustParse("=") })
+	assert.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") })
+}
+
+func TestConditions(t *testing.T) {
+	txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z")
+	require.NoError(t, err)
+
+	testCases := []struct {
+		s          string
+		conditions []query.Condition
+	}{
+		{s: "tm.events.type='NewBlock'", conditions: []query.Condition{query.Condition{Tag: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}}},
+		{s: "tx.gas > 7 AND tx.gas < 9", conditions: []query.Condition{query.Condition{Tag: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, query.Condition{Tag: "tx.gas", Op: query.OpLess, Operand: int64(9)}}},
+		{s: "tx.time >= TIME 2013-05-03T14:45:00Z", conditions: []query.Condition{query.Condition{Tag: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}}},
+	}
+
+	for _, tc := range testCases {
+		q, err := query.New(tc.s)
+		require.Nil(t, err)
+
+		assert.Equal(t, tc.conditions, q.Conditions())
+	}
+}

mempool/mempool.go

@@ -72,8 +72,8 @@ type Mempool struct {
 	rechecking           int32           // for re-checking filtered txs on Update()
 	recheckCursor        *clist.CElement // next expected response
 	recheckEnd           *clist.CElement // re-checking stops here
-	notifiedTxsAvailable bool            // true if fired on txsAvailable for this height
-	txsAvailable         chan int64      // fires the next height once for each height, when the mempool is not empty
+	notifiedTxsAvailable bool
+	txsAvailable         chan int64 // fires the next height once for each height, when the mempool is not empty
 
 	// Keep a cache of already-seen txs.
 	// This reduces the pressure on the proxyApp.
@@ -328,8 +328,12 @@ func (mem *Mempool) notifyTxsAvailable() {
 		panic("notified txs available but mempool is empty!")
 	}
 	if mem.txsAvailable != nil && !mem.notifiedTxsAvailable {
+		select {
+		case mem.txsAvailable <- mem.height + 1:
+		default:
+		}
+
 		mem.notifiedTxsAvailable = true
-		mem.txsAvailable <- mem.height + 1
 	}
 }
 
@@ -382,7 +386,7 @@ func (mem *Mempool) Update(height int64, txs types.Txs) error {
 	// Recheck mempool txs if any txs were committed in the block
 	// NOTE/XXX: in some apps a tx could be invalidated due to EndBlock,
 	//	so we really still do need to recheck, but this is for debugging
-	if mem.config.Recheck && (mem.config.RecheckEmpty || len(txs) > 0) {
+	if mem.config.Recheck && (mem.config.RecheckEmpty || len(goodTxs) > 0) {
 		mem.logger.Info("Recheck txs", "numtxs", len(goodTxs), "height", height)
 		mem.recheckTxs(goodTxs)
 		// At this point, mem.txs are being rechecked.

node/node.go

@@ -21,6 +21,7 @@ import (
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/p2p/pex"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	rpccore "github.com/tendermint/tendermint/rpc/core"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
@@ -32,7 +33,6 @@ import (
 	"github.com/tendermint/tendermint/state/txindex/kv"
 	"github.com/tendermint/tendermint/state/txindex/null"
 	"github.com/tendermint/tendermint/types"
-	pvm "github.com/tendermint/tendermint/types/priv_validator"
 	"github.com/tendermint/tendermint/version"
 
 	_ "net/http/pprof"
@@ -77,7 +77,7 @@ type NodeProvider func(*cfg.Config, log.Logger) (*Node, error)
 // It implements NodeProvider.
 func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) {
 	return NewNode(config,
-		pvm.LoadOrGenFilePV(config.PrivValidatorFile()),
+		privval.LoadOrGenFilePV(config.PrivValidatorFile()),
 		proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()),
 		DefaultGenesisDocProviderFunc(config),
 		DefaultDBProvider,
@@ -159,7 +159,7 @@ func NewNode(config *cfg.Config,
 	// and sync tendermint and the app by performing a handshake
 	// and replaying any necessary blocks
 	consensusLogger := logger.With("module", "consensus")
-	handshaker := cs.NewHandshaker(stateDB, state, blockStore, genDoc.AppState())
+	handshaker := cs.NewHandshaker(stateDB, state, blockStore, genDoc)
 	handshaker.SetLogger(consensusLogger)
 	proxyApp := proxy.NewAppConns(clientCreator, handshaker)
 	proxyApp.SetLogger(logger.With("module", "proxy"))
@@ -177,8 +177,8 @@ func NewNode(config *cfg.Config,
 			// TODO: persist this key so external signer
 			// can actually authenticate us
 			privKey = crypto.GenPrivKeyEd25519()
-			pvsc    = pvm.NewSocketPV(
-				logger.With("module", "pvm"),
+			pvsc    = privval.NewSocketPV(
+				logger.With("module", "privval"),
 				config.PrivValidatorListenAddr,
 				privKey,
 			)
@@ -269,9 +269,6 @@ func NewNode(config *cfg.Config,
 	// but it would still be nice to have a clear list of the current "PersistentPeers"
 	// somewhere that we can return with net_info.
 	//
-	// Let's assume we always have IDs ... and we just dont authenticate them
-	// if auth_enc=false.
-	//
 	// If PEX is on, it should handle dialing the seeds. Otherwise the switch does it.
 	// Note we currently use the addrBook regardless at least for AddOurAddress
 	addrBook := pex.NewAddrBook(config.P2P.AddrBookFile(), config.P2P.AddrBookStrict)
@@ -305,7 +302,7 @@ func NewNode(config *cfg.Config,
 			return nil
 		})
 		sw.SetIDFilter(func(id p2p.ID) error {
-			resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/pubkey/%s", id)})
+			resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/id/%s", id)})
 			if err != nil {
 				return err
 			}
@@ -450,7 +447,7 @@ func (n *Node) OnStop() {
 	n.eventBus.Stop()
 	n.indexerService.Stop()
 
-	if pvsc, ok := n.privValidator.(*pvm.SocketPV); ok {
+	if pvsc, ok := n.privValidator.(*privval.SocketPV); ok {
 		if err := pvsc.Stop(); err != nil {
 			n.Logger.Error("Error stopping priv validator socket client", "err", err)
 		}

p2p/README.md

@@ -4,8 +4,8 @@ The p2p package provides an abstraction around peer-to-peer communication.
 
 Docs:
 
-- [Connection](../docs/specification/new-spec/p2p/connection.md) for details on how connections and multiplexing work
-- [Peer](../docs/specification/new-spec/p2p/peer.md) for details on peer ID, handshakes, and peer exchange
-- [Node](../docs/specification/new-spec/p2p/node.md) for details about different types of nodes and how they should work
-- [Pex](../docs/specification/new-spec/p2p/pex.md) for details on peer discovery and exchange
-- [Config](../docs/specification/new-spec/p2p/config.md) for details on some config option
+- [Connection](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/connection.md) for details on how connections and multiplexing work
+- [Peer](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/peer.md) for details on peer ID, handshakes, and peer exchange
+- [Node](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/node.md) for details about different types of nodes and how they should work
+- [Pex](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/reactors/pex/pex.md) for details on peer discovery and exchange
+- [Config](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/config.md) for details on some config option

p2p/conn/connection.go

@@ -83,7 +83,7 @@ type MConnection struct {
 	onReceive     receiveCbFunc
 	onError       errorCbFunc
 	errored       uint32
-	config        *MConnConfig
+	config        MConnConfig
 
 	quit       chan struct{}
 	flushTimer *cmn.ThrottleTimer // flush writes as necessary but throttled.
@@ -121,8 +121,8 @@ func (cfg *MConnConfig) maxPacketMsgTotalSize() int {
 }
 
 // DefaultMConnConfig returns the default config.
-func DefaultMConnConfig() *MConnConfig {
-	return &MConnConfig{
+func DefaultMConnConfig() MConnConfig {
+	return MConnConfig{
 		SendRate:                defaultSendRate,
 		RecvRate:                defaultRecvRate,
 		MaxPacketMsgPayloadSize: maxPacketMsgPayloadSizeDefault,
@@ -143,7 +143,7 @@ func NewMConnection(conn net.Conn, chDescs []*ChannelDescriptor, onReceive recei
 }
 
 // NewMConnectionWithConfig wraps net.Conn and creates multiplex connection with a config
-func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onReceive receiveCbFunc, onError errorCbFunc, config *MConnConfig) *MConnection {
+func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onReceive receiveCbFunc, onError errorCbFunc, config MConnConfig) *MConnection {
 	if config.PongTimeout >= config.PingInterval {
 		panic("pongTimeout must be less than pingInterval (otherwise, next ping will reset pong timer)")
 	}
@@ -545,9 +545,7 @@ FOR_LOOP:
 // not goroutine-safe
 func (c *MConnection) stopPongTimer() {
 	if c.pongTimer != nil {
-		if !c.pongTimer.Stop() {
-			<-c.pongTimer.C
-		}
+		_ = c.pongTimer.Stop()
 		c.pongTimer = nil
 	}
 }

p2p/conn/connection_test.go

@@ -6,9 +6,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/fortytw2/leaktest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/tendermint/go-amino"
+
+	amino "github.com/tendermint/go-amino"
 	"github.com/tendermint/tmlibs/log"
 )
 
@@ -242,7 +244,11 @@ func TestMConnectionMultiplePings(t *testing.T) {
 }
 
 func TestMConnectionPingPongs(t *testing.T) {
+	// check that we are not leaking any go-routines
+	defer leaktest.CheckTimeout(t, 10*time.Second)()
+
 	server, client := net.Pipe()
+
 	defer server.Close()
 	defer client.Close()
 

p2p/dummy/peer.go

@@ -1,6 +1,8 @@
 package dummy
 
 import (
+	"net"
+
 	p2p "github.com/tendermint/tendermint/p2p"
 	tmconn "github.com/tendermint/tendermint/p2p/conn"
 	cmn "github.com/tendermint/tmlibs/common"
@@ -19,6 +21,7 @@ func NewPeer() *peer {
 		kv: make(map[string]interface{}),
 	}
 	p.BaseService = *cmn.NewBaseService(nil, "peer", p)
+
 	return p
 }
 
@@ -42,6 +45,11 @@ func (p *peer) NodeInfo() p2p.NodeInfo {
 	return p2p.NodeInfo{}
 }
 
+// RemoteIP always returns localhost.
+func (p *peer) RemoteIP() net.IP {
+	return net.ParseIP("127.0.0.1")
+}
+
 // Status always returns empry connection status.
 func (p *peer) Status() tmconn.ConnectionStatus {
 	return tmconn.ConnectionStatus{}

p2p/errors.go

@@ -1,14 +1,38 @@
 package p2p
 
 import (
-	"errors"
 	"fmt"
+	"net"
 )
 
-var (
-	ErrSwitchDuplicatePeer = errors.New("Duplicate peer")
-	ErrSwitchConnectToSelf = errors.New("Connect to self")
-)
+// ErrSwitchDuplicatePeerID to be raised when a peer is connecting with a known
+// ID.
+type ErrSwitchDuplicatePeerID struct {
+	ID ID
+}
+
+func (e ErrSwitchDuplicatePeerID) Error() string {
+	return fmt.Sprintf("Duplicate peer ID %v", e.ID)
+}
+
+// ErrSwitchDuplicatePeerIP to be raised whena a peer is connecting with a known
+// IP.
+type ErrSwitchDuplicatePeerIP struct {
+	IP net.IP
+}
+
+func (e ErrSwitchDuplicatePeerIP) Error() string {
+	return fmt.Sprintf("Duplicate peer IP %v", e.IP.String())
+}
+
+// ErrSwitchConnectToSelf to be raised when trying to connect to itself.
+type ErrSwitchConnectToSelf struct {
+	Addr *NetAddress
+}
+
+func (e ErrSwitchConnectToSelf) Error() string {
+	return fmt.Sprintf("Connect to self: %v", e.Addr)
+}
 
 type ErrSwitchAuthenticationFailure struct {
 	Dialed *NetAddress
@@ -16,7 +40,11 @@ type ErrSwitchAuthenticationFailure struct {
 }
 
 func (e ErrSwitchAuthenticationFailure) Error() string {
-	return fmt.Sprintf("Failed to authenticate peer. Dialed %v, but got peer with ID %s", e.Dialed, e.Got)
+	return fmt.Sprintf(
+		"Failed to authenticate peer. Dialed %v, but got peer with ID %s",
+		e.Dialed,
+		e.Got,
+	)
 }
 
 //-------------------------------------------------------------------

p2p/fuzz.go

@@ -5,16 +5,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/tendermint/tendermint/config"
 	cmn "github.com/tendermint/tmlibs/common"
 )
 
-const (
-	// FuzzModeDrop is a mode in which we randomly drop reads/writes, connections or sleep
-	FuzzModeDrop = iota
-	// FuzzModeDelay is a mode in which we randomly sleep
-	FuzzModeDelay
-)
-
 // FuzzedConnection wraps any net.Conn and depending on the mode either delays
 // reads/writes or randomly drops reads/writes/connections.
 type FuzzedConnection struct {
@@ -24,37 +18,17 @@ type FuzzedConnection struct {
 	start  <-chan time.Time
 	active bool
 
-	config *FuzzConnConfig
-}
-
-// FuzzConnConfig is a FuzzedConnection configuration.
-type FuzzConnConfig struct {
-	Mode         int
-	MaxDelay     time.Duration
-	ProbDropRW   float64
-	ProbDropConn float64
-	ProbSleep    float64
-}
-
-// DefaultFuzzConnConfig returns the default config.
-func DefaultFuzzConnConfig() *FuzzConnConfig {
-	return &FuzzConnConfig{
-		Mode:         FuzzModeDrop,
-		MaxDelay:     3 * time.Second,
-		ProbDropRW:   0.2,
-		ProbDropConn: 0.00,
-		ProbSleep:    0.00,
-	}
+	config *config.FuzzConnConfig
 }
 
 // FuzzConn creates a new FuzzedConnection. Fuzzing starts immediately.
 func FuzzConn(conn net.Conn) net.Conn {
-	return FuzzConnFromConfig(conn, DefaultFuzzConnConfig())
+	return FuzzConnFromConfig(conn, config.DefaultFuzzConnConfig())
 }
 
 // FuzzConnFromConfig creates a new FuzzedConnection from a config. Fuzzing
 // starts immediately.
-func FuzzConnFromConfig(conn net.Conn, config *FuzzConnConfig) net.Conn {
+func FuzzConnFromConfig(conn net.Conn, config *config.FuzzConnConfig) net.Conn {
 	return &FuzzedConnection{
 		conn:   conn,
 		start:  make(<-chan time.Time),
@@ -66,12 +40,16 @@ func FuzzConnFromConfig(conn net.Conn, config *FuzzConnConfig) net.Conn {
 // FuzzConnAfter creates a new FuzzedConnection. Fuzzing starts when the
 // duration elapses.
 func FuzzConnAfter(conn net.Conn, d time.Duration) net.Conn {
-	return FuzzConnAfterFromConfig(conn, d, DefaultFuzzConnConfig())
+	return FuzzConnAfterFromConfig(conn, d, config.DefaultFuzzConnConfig())
 }
 
 // FuzzConnAfterFromConfig creates a new FuzzedConnection from a config.
 // Fuzzing starts when the duration elapses.
-func FuzzConnAfterFromConfig(conn net.Conn, d time.Duration, config *FuzzConnConfig) net.Conn {
+func FuzzConnAfterFromConfig(
+	conn net.Conn,
+	d time.Duration,
+	config *config.FuzzConnConfig,
+) net.Conn {
 	return &FuzzedConnection{
 		conn:   conn,
 		start:  time.After(d),
@@ -81,7 +59,7 @@ func FuzzConnAfterFromConfig(conn net.Conn, d time.Duration, config *FuzzConnCon
 }
 
 // Config returns the connection's config.
-func (fc *FuzzedConnection) Config() *FuzzConnConfig {
+func (fc *FuzzedConnection) Config() *config.FuzzConnConfig {
 	return fc.config
 }
 
@@ -136,7 +114,7 @@ func (fc *FuzzedConnection) fuzz() bool {
 	}
 
 	switch fc.config.Mode {
-	case FuzzModeDrop:
+	case config.FuzzModeDrop:
 		// randomly drop the r/w, drop the conn, or sleep
 		r := cmn.RandFloat64()
 		if r <= fc.config.ProbDropRW {
@@ -149,7 +127,7 @@ func (fc *FuzzedConnection) fuzz() bool {
 		} else if r < fc.config.ProbDropRW+fc.config.ProbDropConn+fc.config.ProbSleep {
 			time.Sleep(fc.randomDuration())
 		}
-	case FuzzModeDelay:
+	case config.FuzzModeDelay:
 		// sleep a bit
 		time.Sleep(fc.randomDuration())
 	}

p2p/peer.go

@@ -3,20 +3,25 @@ package p2p
 import (
 	"fmt"
 	"net"
+	"sync/atomic"
 	"time"
 
-	"github.com/tendermint/go-crypto"
+	crypto "github.com/tendermint/go-crypto"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
 
+	"github.com/tendermint/tendermint/config"
 	tmconn "github.com/tendermint/tendermint/p2p/conn"
 )
 
+var testIPSuffix uint32
+
 // Peer is an interface representing a peer connected on a reactor.
 type Peer interface {
 	cmn.Service
 
 	ID() ID             // peer's cryptographic ID
+	RemoteIP() net.IP   // remote IP of the connection
 	IsOutbound() bool   // did we dial the peer
 	IsPersistent() bool // do we redial this peer when we disconnect
 	NodeInfo() NodeInfo // peer's info
@@ -35,8 +40,9 @@ type Peer interface {
 type peerConn struct {
 	outbound   bool
 	persistent bool
-	config     *PeerConfig
+	config     *config.P2PConfig
 	conn       net.Conn // source connection
+	ip         net.IP
 }
 
 // ID only exists for SecretConnection.
@@ -45,6 +51,35 @@ func (pc peerConn) ID() ID {
 	return PubKeyToID(pc.conn.(*tmconn.SecretConnection).RemotePubKey())
 }
 
+// Return the IP from the connection RemoteAddr
+func (pc peerConn) RemoteIP() net.IP {
+	if pc.ip != nil {
+		return pc.ip
+	}
+
+	// In test cases a conn could not be present at all or be an in-memory
+	// implementation where we want to return a fake ip.
+	if pc.conn == nil || pc.conn.RemoteAddr().String() == "pipe" {
+		pc.ip = net.IP{172, 16, 0, byte(atomic.AddUint32(&testIPSuffix, 1))}
+
+		return pc.ip
+	}
+
+	host, _, err := net.SplitHostPort(pc.conn.RemoteAddr().String())
+	if err != nil {
+		panic(err)
+	}
+
+	ips, err := net.LookupIP(host)
+	if err != nil {
+		panic(err)
+	}
+
+	pc.ip = ips[0]
+
+	return pc.ip
+}
+
 // peer implements Peer.
 //
 // Before using a peer, you will need to perform a handshake on connection.
@@ -65,110 +100,107 @@ type peer struct {
 	Data *cmn.CMap
 }
 
-func newPeer(pc peerConn, nodeInfo NodeInfo,
-	reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor,
-	onPeerError func(Peer, interface{})) *peer {
-
+func newPeer(
+	pc peerConn,
+	mConfig tmconn.MConnConfig,
+	nodeInfo NodeInfo,
+	reactorsByCh map[byte]Reactor,
+	chDescs []*tmconn.ChannelDescriptor,
+	onPeerError func(Peer, interface{}),
+) *peer {
 	p := &peer{
 		peerConn: pc,
 		nodeInfo: nodeInfo,
 		channels: nodeInfo.Channels,
 		Data:     cmn.NewCMap(),
 	}
-	p.mconn = createMConnection(pc.conn, p, reactorsByCh, chDescs, onPeerError, pc.config.MConfig)
+
+	p.mconn = createMConnection(
+		pc.conn,
+		p,
+		reactorsByCh,
+		chDescs,
+		onPeerError,
+		mConfig,
+	)
 	p.BaseService = *cmn.NewBaseService(nil, "Peer", p)
+
 	return p
 }
 
-// PeerConfig is a Peer configuration.
-type PeerConfig struct {
-	AuthEnc bool `mapstructure:"auth_enc"` // authenticated encryption
-
-	// times are in seconds
-	HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
-	DialTimeout      time.Duration `mapstructure:"dial_timeout"`
-
-	MConfig *tmconn.MConnConfig `mapstructure:"connection"`
-
-	DialFail   bool            `mapstructure:"dial_fail"` // for testing
-	Fuzz       bool            `mapstructure:"fuzz"`      // fuzz connection (for testing)
-	FuzzConfig *FuzzConnConfig `mapstructure:"fuzz_config"`
-}
-
-// DefaultPeerConfig returns the default config.
-func DefaultPeerConfig() *PeerConfig {
-	return &PeerConfig{
-		AuthEnc:          true,
-		HandshakeTimeout: 20, // * time.Second,
-		DialTimeout:      3,  // * time.Second,
-		MConfig:          tmconn.DefaultMConnConfig(),
-		DialFail:         false,
-		Fuzz:             false,
-		FuzzConfig:       DefaultFuzzConnConfig(),
-	}
-}
-
-func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool, ourNodePrivKey crypto.PrivKey) (peerConn, error) {
-	var pc peerConn
-
+func newOutboundPeerConn(
+	addr *NetAddress,
+	config *config.P2PConfig,
+	persistent bool,
+	ourNodePrivKey crypto.PrivKey,
+) (peerConn, error) {
 	conn, err := dial(addr, config)
 	if err != nil {
-		return pc, cmn.ErrorWrap(err, "Error creating peer")
+		return peerConn{}, cmn.ErrorWrap(err, "Error creating peer")
 	}
 
-	pc, err = newPeerConn(conn, config, true, persistent, ourNodePrivKey)
+	pc, err := newPeerConn(conn, config, true, persistent, ourNodePrivKey)
 	if err != nil {
-		if err2 := conn.Close(); err2 != nil {
-			return pc, cmn.ErrorWrap(err, err2.Error())
+		if cerr := conn.Close(); cerr != nil {
+			return peerConn{}, cmn.ErrorWrap(err, cerr.Error())
 		}
-		return pc, err
+		return peerConn{}, err
 	}
 
 	// ensure dialed ID matches connection ID
-	if config.AuthEnc && addr.ID != pc.ID() {
-		if err2 := conn.Close(); err2 != nil {
-			return pc, cmn.ErrorWrap(err, err2.Error())
+	if addr.ID != pc.ID() {
+		if cerr := conn.Close(); cerr != nil {
+			return peerConn{}, cmn.ErrorWrap(err, cerr.Error())
 		}
-		return pc, ErrSwitchAuthenticationFailure{addr, pc.ID()}
+		return peerConn{}, ErrSwitchAuthenticationFailure{addr, pc.ID()}
 	}
+
 	return pc, nil
 }
 
-func newInboundPeerConn(conn net.Conn, config *PeerConfig, ourNodePrivKey crypto.PrivKey) (peerConn, error) {
+func newInboundPeerConn(
+	conn net.Conn,
+	config *config.P2PConfig,
+	ourNodePrivKey crypto.PrivKey,
+) (peerConn, error) {
 
 	// TODO: issue PoW challenge
 
 	return newPeerConn(conn, config, false, false, ourNodePrivKey)
 }
 
-func newPeerConn(rawConn net.Conn,
-	config *PeerConfig, outbound, persistent bool,
-	ourNodePrivKey crypto.PrivKey) (pc peerConn, err error) {
-
+func newPeerConn(
+	rawConn net.Conn,
+	cfg *config.P2PConfig,
+	outbound, persistent bool,
+	ourNodePrivKey crypto.PrivKey,
+) (pc peerConn, err error) {
 	conn := rawConn
 
 	// Fuzz connection
-	if config.Fuzz {
+	if cfg.TestFuzz {
 		// so we have time to do peer handshakes and get set up
-		conn = FuzzConnAfterFromConfig(conn, 10*time.Second, config.FuzzConfig)
+		conn = FuzzConnAfterFromConfig(conn, 10*time.Second, cfg.TestFuzzConfig)
 	}
 
-	if config.AuthEnc {
-		// Set deadline for secret handshake
-		if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil {
-			return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection")
-		}
+	// Set deadline for secret handshake
+	dl := time.Now().Add(cfg.HandshakeTimeout)
+	if err := conn.SetDeadline(dl); err != nil {
+		return pc, cmn.ErrorWrap(
+			err,
+			"Error setting deadline while encrypting connection",
+		)
+	}
 
-		// Encrypt connection
-		conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey)
-		if err != nil {
-			return pc, cmn.ErrorWrap(err, "Error creating peer")
-		}
+	// Encrypt connection
+	conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey)
+	if err != nil {
+		return pc, cmn.ErrorWrap(err, "Error creating peer")
 	}
 
 	// Only the information we already have
 	return peerConn{
-		config:     config,
+		config:     cfg,
 		outbound:   outbound,
 		persistent: persistent,
 		conn:       conn,
@@ -271,22 +303,33 @@ func (p *peer) hasChannel(chID byte) bool {
 	}
 	// NOTE: probably will want to remove this
 	// but could be helpful while the feature is new
-	p.Logger.Debug("Unknown channel for peer", "channel", chID, "channels", p.channels)
+	p.Logger.Debug(
+		"Unknown channel for peer",
+		"channel",
+		chID,
+		"channels",
+		p.channels,
+	)
 	return false
 }
 
 //---------------------------------------------------
 // methods used by the Switch
 
-// CloseConn should be called by the Switch if the peer was created but never started.
+// CloseConn should be called by the Switch if the peer was created but never
+// started.
 func (pc *peerConn) CloseConn() {
 	pc.conn.Close() // nolint: errcheck
 }
 
-// HandshakeTimeout performs the Tendermint P2P handshake between a given node and the peer
-// by exchanging their NodeInfo. It sets the received nodeInfo on the peer.
+// HandshakeTimeout performs the Tendermint P2P handshake between a given node
+// and the peer by exchanging their NodeInfo. It sets the received nodeInfo on
+// the peer.
 // NOTE: blocking
-func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration) (peerNodeInfo NodeInfo, err error) {
+func (pc *peerConn) HandshakeTimeout(
+	ourNodeInfo NodeInfo,
+	timeout time.Duration,
+) (peerNodeInfo NodeInfo, err error) {
 	// Set deadline for handshake so we don't block forever on conn.ReadFull
 	if err := pc.conn.SetDeadline(time.Now().Add(timeout)); err != nil {
 		return peerNodeInfo, cmn.ErrorWrap(err, "Error setting deadline")
@@ -298,7 +341,11 @@ func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration
 			return
 		},
 		func(_ int) (val interface{}, err error, abort bool) {
-			_, err = cdc.UnmarshalBinaryReader(pc.conn, &peerNodeInfo, int64(MaxNodeInfoSize()))
+			_, err = cdc.UnmarshalBinaryReader(
+				pc.conn,
+				&peerNodeInfo,
+				int64(MaxNodeInfoSize()),
+			)
 			return
 		},
 	)
@@ -339,20 +386,26 @@ func (p *peer) String() string {
 //------------------------------------------------------------------
 // helper funcs
 
-func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) {
-	if config.DialFail {
+func dial(addr *NetAddress, cfg *config.P2PConfig) (net.Conn, error) {
+	if cfg.TestDialFail {
 		return nil, fmt.Errorf("dial err (peerConfig.DialFail == true)")
 	}
 
-	conn, err := addr.DialTimeout(config.DialTimeout * time.Second)
+	conn, err := addr.DialTimeout(cfg.DialTimeout)
 	if err != nil {
 		return nil, err
 	}
 	return conn, nil
 }
 
-func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor,
-	onPeerError func(Peer, interface{}), config *tmconn.MConnConfig) *tmconn.MConnection {
+func createMConnection(
+	conn net.Conn,
+	p *peer,
+	reactorsByCh map[byte]Reactor,
+	chDescs []*tmconn.ChannelDescriptor,
+	onPeerError func(Peer, interface{}),
+	config tmconn.MConnConfig,
+) *tmconn.MConnection {
 
 	onReceive := func(chID byte, msgBytes []byte) {
 		reactor := reactorsByCh[chID]
@@ -368,5 +421,11 @@ func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, ch
 		onPeerError(p, r)
 	}
 
-	return tmconn.NewMConnectionWithConfig(conn, chDescs, onReceive, onError, config)
+	return tmconn.NewMConnectionWithConfig(
+		conn,
+		chDescs,
+		onReceive,
+		onError,
+		config,
+	)
 }

p2p/peer_set.go

@@ -1,12 +1,14 @@
 package p2p
 
 import (
+	"net"
 	"sync"
 )
 
 // IPeerSet has a (immutable) subset of the methods of PeerSet.
 type IPeerSet interface {
 	Has(key ID) bool
+	HasIP(ip net.IP) bool
 	Get(key ID) Peer
 	List() []Peer
 	Size() int
@@ -36,12 +38,13 @@ func NewPeerSet() *PeerSet {
 }
 
 // Add adds the peer to the PeerSet.
-// It returns ErrSwitchDuplicatePeer if the peer is already present.
+// It returns an error carrying the reason, if the peer is already present.
 func (ps *PeerSet) Add(peer Peer) error {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
+
 	if ps.lookup[peer.ID()] != nil {
-		return ErrSwitchDuplicatePeer
+		return ErrSwitchDuplicatePeerID{peer.ID()}
 	}
 
 	index := len(ps.list)
@@ -61,6 +64,27 @@ func (ps *PeerSet) Has(peerKey ID) bool {
 	return ok
 }
 
+// HasIP returns true if the PeerSet contains the peer referred to by this IP
+// address.
+func (ps *PeerSet) HasIP(peerIP net.IP) bool {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	return ps.hasIP(peerIP)
+}
+
+// hasIP does not acquire a lock so it can be used in public methods which
+// already lock.
+func (ps *PeerSet) hasIP(peerIP net.IP) bool {
+	for _, item := range ps.lookup {
+		if item.peer.RemoteIP().Equal(peerIP) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // Get looks up a peer by the provided peerKey.
 func (ps *PeerSet) Get(peerKey ID) Peer {
 	ps.mtx.Lock()
@@ -76,6 +100,7 @@ func (ps *PeerSet) Get(peerKey ID) Peer {
 func (ps *PeerSet) Remove(peer Peer) {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
+
 	item := ps.lookup[peer.ID()]
 	if item == nil {
 		return

p2p/peer_set_test.go

@@ -2,6 +2,7 @@ package p2p
 
 import (
 	"math/rand"
+	"net"
 	"sync"
 	"testing"
 
@@ -12,23 +13,32 @@ import (
 )
 
 // Returns an empty kvstore peer
-func randPeer() *peer {
+func randPeer(ip net.IP) *peer {
+	if ip == nil {
+		ip = net.IP{127, 0, 0, 1}
+	}
+
 	nodeKey := NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
-	return &peer{
+	p := &peer{
 		nodeInfo: NodeInfo{
 			ID:         nodeKey.ID(),
 			ListenAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256),
 		},
 	}
+
+	p.ip = ip
+
+	return p
 }
 
 func TestPeerSetAddRemoveOne(t *testing.T) {
 	t.Parallel()
+
 	peerSet := NewPeerSet()
 
 	var peerList []Peer
 	for i := 0; i < 5; i++ {
-		p := randPeer()
+		p := randPeer(net.IP{127, 0, 0, byte(i)})
 		if err := peerSet.Add(p); err != nil {
 			t.Error(err)
 		}
@@ -72,7 +82,7 @@ func TestPeerSetAddRemoveMany(t *testing.T) {
 	peers := []Peer{}
 	N := 100
 	for i := 0; i < N; i++ {
-		peer := randPeer()
+		peer := randPeer(net.IP{127, 0, 0, byte(i)})
 		if err := peerSet.Add(peer); err != nil {
 			t.Errorf("Failed to add new peer")
 		}
@@ -96,7 +106,7 @@ func TestPeerSetAddRemoveMany(t *testing.T) {
 func TestPeerSetAddDuplicate(t *testing.T) {
 	t.Parallel()
 	peerSet := NewPeerSet()
-	peer := randPeer()
+	peer := randPeer(nil)
 
 	n := 20
 	errsChan := make(chan error)
@@ -112,25 +122,35 @@ func TestPeerSetAddDuplicate(t *testing.T) {
 	}
 
 	// Now collect and tally the results
-	errsTally := make(map[error]int)
+	errsTally := make(map[string]int)
 	for i := 0; i < n; i++ {
 		err := <-errsChan
-		errsTally[err]++
+
+		switch err.(type) {
+		case ErrSwitchDuplicatePeerID:
+			errsTally["duplicateID"]++
+		default:
+			errsTally["other"]++
+		}
 	}
 
 	// Our next procedure is to ensure that only one addition
 	// succeeded and that the rest are each ErrSwitchDuplicatePeer.
-	wantErrCount, gotErrCount := n-1, errsTally[ErrSwitchDuplicatePeer]
+	wantErrCount, gotErrCount := n-1, errsTally["duplicateID"]
 	assert.Equal(t, wantErrCount, gotErrCount, "invalid ErrSwitchDuplicatePeer count")
 
-	wantNilErrCount, gotNilErrCount := 1, errsTally[nil]
+	wantNilErrCount, gotNilErrCount := 1, errsTally["other"]
 	assert.Equal(t, wantNilErrCount, gotNilErrCount, "invalid nil errCount")
 }
 
 func TestPeerSetGet(t *testing.T) {
 	t.Parallel()
-	peerSet := NewPeerSet()
-	peer := randPeer()
+
+	var (
+		peerSet = NewPeerSet()
+		peer    = randPeer(nil)
+	)
+
 	assert.Nil(t, peerSet.Get(peer.ID()), "expecting a nil lookup, before .Add")
 
 	if err := peerSet.Add(peer); err != nil {
@@ -144,8 +164,8 @@ func TestPeerSetGet(t *testing.T) {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
-			got, want := peerSet.Get(peer.ID()), peer
-			assert.Equal(t, got, want, "#%d: got=%v want=%v", i, got, want)
+			have, want := peerSet.Get(peer.ID()), peer
+			assert.Equal(t, have, want, "%d: have %v, want %v", i, have, want)
 		}(i)
 	}
 	wg.Wait()

p2p/peer_test.go

@@ -10,8 +10,11 @@ import (
 	"github.com/stretchr/testify/require"
 
 	crypto "github.com/tendermint/go-crypto"
-	tmconn "github.com/tendermint/tendermint/p2p/conn"
+	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/config"
+	tmconn "github.com/tendermint/tendermint/p2p/conn"
 )
 
 const testCh = 0x01
@@ -20,11 +23,11 @@ func TestPeerBasic(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 
 	// simulate remote peer
-	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()}
+	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: cfg}
 	rp.Start()
 	defer rp.Stop()
 
-	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), DefaultPeerConfig())
+	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), cfg, tmconn.DefaultMConnConfig())
 	require.Nil(err)
 
 	err = p.Start()
@@ -40,39 +43,17 @@ func TestPeerBasic(t *testing.T) {
 	assert.Equal(rp.ID(), p.ID())
 }
 
-func TestPeerWithoutAuthEnc(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	config := DefaultPeerConfig()
-	config.AuthEnc = false
-
-	// simulate remote peer
-	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config}
-	rp.Start()
-	defer rp.Stop()
-
-	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), config)
-	require.Nil(err)
-
-	err = p.Start()
-	require.Nil(err)
-	defer p.Stop()
-
-	assert.True(p.IsRunning())
-}
-
 func TestPeerSend(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 
-	config := DefaultPeerConfig()
-	config.AuthEnc = false
+	config := cfg
 
 	// simulate remote peer
 	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config}
 	rp.Start()
 	defer rp.Stop()
 
-	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), config)
+	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), config, tmconn.DefaultMConnConfig())
 	require.Nil(err)
 
 	err = p.Start()
@@ -84,7 +65,11 @@ func TestPeerSend(t *testing.T) {
 	assert.True(p.Send(testCh, []byte("Asylum")))
 }
 
-func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) (*peer, error) {
+func createOutboundPeerAndPerformHandshake(
+	addr *NetAddress,
+	config *config.P2PConfig,
+	mConfig tmconn.MConnConfig,
+) (*peer, error) {
 	chDescs := []*tmconn.ChannelDescriptor{
 		{ID: testCh, Priority: 1},
 	}
@@ -105,41 +90,50 @@ func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig)
 		return nil, err
 	}
 
-	p := newPeer(pc, nodeInfo, reactorsByCh, chDescs, func(p Peer, r interface{}) {})
+	p := newPeer(pc, mConfig, nodeInfo, reactorsByCh, chDescs, func(p Peer, r interface{}) {})
 	p.SetLogger(log.TestingLogger().With("peer", addr))
 	return p, nil
 }
 
 type remotePeer struct {
-	PrivKey crypto.PrivKey
-	Config  *PeerConfig
-	addr    *NetAddress
-	quit    chan struct{}
+	PrivKey    crypto.PrivKey
+	Config     *config.P2PConfig
+	addr       *NetAddress
+	quit       chan struct{}
+	channels   cmn.HexBytes
+	listenAddr string
 }
 
-func (p *remotePeer) Addr() *NetAddress {
-	return p.addr
+func (rp *remotePeer) Addr() *NetAddress {
+	return rp.addr
 }
 
-func (p *remotePeer) ID() ID {
-	return PubKeyToID(p.PrivKey.PubKey())
+func (rp *remotePeer) ID() ID {
+	return PubKeyToID(rp.PrivKey.PubKey())
 }
 
-func (p *remotePeer) Start() {
-	l, e := net.Listen("tcp", "127.0.0.1:0") // any available address
+func (rp *remotePeer) Start() {
+	if rp.listenAddr == "" {
+		rp.listenAddr = "127.0.0.1:0"
+	}
+
+	l, e := net.Listen("tcp", rp.listenAddr) // any available address
 	if e != nil {
 		golog.Fatalf("net.Listen tcp :0: %+v", e)
 	}
-	p.addr = NewNetAddress(PubKeyToID(p.PrivKey.PubKey()), l.Addr())
-	p.quit = make(chan struct{})
-	go p.accept(l)
+	rp.addr = NewNetAddress(PubKeyToID(rp.PrivKey.PubKey()), l.Addr())
+	rp.quit = make(chan struct{})
+	if rp.channels == nil {
+		rp.channels = []byte{testCh}
+	}
+	go rp.accept(l)
 }
 
-func (p *remotePeer) Stop() {
-	close(p.quit)
+func (rp *remotePeer) Stop() {
+	close(rp.quit)
 }
 
-func (p *remotePeer) accept(l net.Listener) {
+func (rp *remotePeer) accept(l net.Listener) {
 	conns := []net.Conn{}
 
 	for {
@@ -147,17 +141,19 @@ func (p *remotePeer) accept(l net.Listener) {
 		if err != nil {
 			golog.Fatalf("Failed to accept conn: %+v", err)
 		}
-		pc, err := newInboundPeerConn(conn, p.Config, p.PrivKey)
+
+		pc, err := newInboundPeerConn(conn, rp.Config, rp.PrivKey)
 		if err != nil {
 			golog.Fatalf("Failed to create a peer: %+v", err)
 		}
+
 		_, err = pc.HandshakeTimeout(NodeInfo{
-			ID:         p.Addr().ID,
+			ID:         rp.Addr().ID,
 			Moniker:    "remote_peer",
 			Network:    "testing",
 			Version:    "123.123.123",
 			ListenAddr: l.Addr().String(),
-			Channels:   []byte{testCh},
+			Channels:   rp.channels,
 		}, 1*time.Second)
 		if err != nil {
 			golog.Fatalf("Failed to perform handshake: %+v", err)
@@ -166,7 +162,7 @@ func (p *remotePeer) accept(l net.Listener) {
 		conns = append(conns, conn)
 
 		select {
-		case <-p.quit:
+		case <-rp.quit:
 			for _, conn := range conns {
 				if err := conn.Close(); err != nil {
 					golog.Fatal(err)

p2p/pex/pex_reactor.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 	cmn "github.com/tendermint/tmlibs/common"
 
 	"github.com/tendermint/tendermint/p2p"
@@ -267,7 +267,7 @@ func (r *PEXReactor) receiveRequest(src Peer) error {
 	now := time.Now()
 	minInterval := r.minReceiveRequestInterval()
 	if now.Sub(lastReceived) < minInterval {
-		return fmt.Errorf("Peer (%v) send next PEX request too soon. lastReceived: %v, now: %v, minInterval: %v. Disconnecting",
+		return fmt.Errorf("Peer (%v) sent next PEX request too soon. lastReceived: %v, now: %v, minInterval: %v. Disconnecting",
 			src.ID(),
 			lastReceived,
 			now,
@@ -281,6 +281,7 @@ func (r *PEXReactor) receiveRequest(src Peer) error {
 // RequestAddrs asks peer for more addresses if we do not already
 // have a request out for this peer.
 func (r *PEXReactor) RequestAddrs(p Peer) {
+	r.Logger.Debug("Request addrs", "from", p)
 	id := string(p.ID())
 	if r.requestsSent.Has(id) {
 		return

p2p/pex/pex_reactor_test.go

@@ -3,6 +3,7 @@ package pex
 import (
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 	"testing"
@@ -12,20 +13,22 @@ import (
 	"github.com/stretchr/testify/require"
 
 	crypto "github.com/tendermint/go-crypto"
-	cfg "github.com/tendermint/tendermint/config"
-	"github.com/tendermint/tendermint/p2p"
-	"github.com/tendermint/tendermint/p2p/conn"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/p2p/conn"
 )
 
 var (
-	config *cfg.P2PConfig
+	cfg *config.P2PConfig
 )
 
 func init() {
-	config = cfg.DefaultP2PConfig()
-	config.PexReactor = true
+	cfg = config.DefaultP2PConfig()
+	cfg.PexReactor = true
+	cfg.AllowDuplicateIP = true
 }
 
 func TestPEXReactorBasic(t *testing.T) {
@@ -47,7 +50,6 @@ func TestPEXReactorAddRemovePeer(t *testing.T) {
 	assert.Equal(t, size+1, book.Size())
 
 	r.RemovePeer(peer, "peer not available")
-	assert.Equal(t, size+1, book.Size())
 
 	outboundPeer := p2p.CreateRandomPeer(true)
 
@@ -55,9 +57,18 @@ func TestPEXReactorAddRemovePeer(t *testing.T) {
 	assert.Equal(t, size+1, book.Size(), "outbound peers should not be added to the address book")
 
 	r.RemovePeer(outboundPeer, "peer not available")
-	assert.Equal(t, size+1, book.Size())
 }
 
+// --- FAIL: TestPEXReactorRunning (11.10s)
+// 				pex_reactor_test.go:411: expected all switches to be connected to at
+// 				least one peer (switches: 0 => {outbound: 1, inbound: 0}, 1 =>
+// 				{outbound: 0, inbound: 1}, 2 => {outbound: 0, inbound: 0}, )
+//
+// EXPLANATION: peers are getting rejected because in switch#addPeer we check
+// if any peer (who we already connected to) has the same IP. Even though local
+// peers have different IP addresses, they all have the same underlying remote
+// IP: 127.0.0.1.
+//
 func TestPEXReactorRunning(t *testing.T) {
 	N := 3
 	switches := make([]*p2p.Switch, N)
@@ -72,7 +83,7 @@ func TestPEXReactorRunning(t *testing.T) {
 
 	// create switches
 	for i := 0; i < N; i++ {
-		switches[i] = p2p.MakeSwitch(config, i, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch {
+		switches[i] = p2p.MakeSwitch(cfg, i, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch {
 			books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false)
 			books[i].SetLogger(logger.With("pex", i))
 			sw.SetAddrBook(books[i])
@@ -200,7 +211,7 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) {
 
 	// 1. create seed
 	seed := p2p.MakeSwitch(
-		config,
+		cfg,
 		0,
 		"127.0.0.1",
 		"123.123.123",
@@ -230,7 +241,7 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) {
 
 	// 2. create usual peer with only seed configured.
 	peer := p2p.MakeSwitch(
-		config,
+		cfg,
 		1,
 		"127.0.0.1",
 		"123.123.123",
@@ -365,6 +376,7 @@ func (mp mockPeer) NodeInfo() p2p.NodeInfo {
 		ListenAddr: mp.addr.DialString(),
 	}
 }
+func (mp mockPeer) RemoteIP() net.IP              { return net.ParseIP("127.0.0.1") }
 func (mp mockPeer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} }
 func (mp mockPeer) Send(byte, []byte) bool        { return false }
 func (mp mockPeer) TrySend(byte, []byte) bool     { return false }
@@ -415,7 +427,7 @@ func assertPeersWithTimeout(
 	}
 }
 
-func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) {
+func createReactor(conf *PEXReactorConfig) (r *PEXReactor, book *addrBook) {
 	// directory to store address book
 	dir, err := ioutil.TempDir("", "pex_reactor")
 	if err != nil {
@@ -424,7 +436,7 @@ func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) {
 	book = NewAddrBook(filepath.Join(dir, "addrbook.json"), true)
 	book.SetLogger(log.TestingLogger())
 
-	r = NewPEXReactor(book, config)
+	r = NewPEXReactor(book, conf)
 	r.SetLogger(log.TestingLogger())
 	return
 }
@@ -437,7 +449,7 @@ func teardownReactor(book *addrBook) {
 }
 
 func createSwitchAndAddReactors(reactors ...p2p.Reactor) *p2p.Switch {
-	sw := p2p.MakeSwitch(config, 0, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { return sw })
+	sw := p2p.MakeSwitch(cfg, 0, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { return sw })
 	sw.SetLogger(log.TestingLogger())
 	for _, r := range reactors {
 		sw.AddReactor(r.String(), r)

p2p/switch.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"time"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p/conn"
 	cmn "github.com/tendermint/tmlibs/common"
 )
@@ -55,8 +55,7 @@ type AddrBook interface {
 type Switch struct {
 	cmn.BaseService
 
-	config       *cfg.P2PConfig
-	peerConfig   *PeerConfig
+	config       *config.P2PConfig
 	listeners    []Listener
 	reactors     map[string]Reactor
 	chDescs      []*conn.ChannelDescriptor
@@ -71,14 +70,15 @@ type Switch struct {
 	filterConnByAddr func(net.Addr) error
 	filterConnByID   func(ID) error
 
+	mConfig conn.MConnConfig
+
 	rng *cmn.Rand // seed for randomizing dial times and orders
 }
 
 // NewSwitch creates a new Switch with the given config.
-func NewSwitch(config *cfg.P2PConfig) *Switch {
+func NewSwitch(cfg *config.P2PConfig) *Switch {
 	sw := &Switch{
-		config:       config,
-		peerConfig:   DefaultPeerConfig(),
+		config:       cfg,
 		reactors:     make(map[string]Reactor),
 		chDescs:      make([]*conn.ChannelDescriptor, 0),
 		reactorsByCh: make(map[byte]Reactor),
@@ -90,12 +90,13 @@ func NewSwitch(config *cfg.P2PConfig) *Switch {
 	// Ensure we have a completely undeterministic PRNG.
 	sw.rng = cmn.NewRand()
 
-	// TODO: collapse the peerConfig into the config ?
-	sw.peerConfig.MConfig.FlushThrottle = time.Duration(config.FlushThrottleTimeout) * time.Millisecond
-	sw.peerConfig.MConfig.SendRate = config.SendRate
-	sw.peerConfig.MConfig.RecvRate = config.RecvRate
-	sw.peerConfig.MConfig.MaxPacketMsgPayloadSize = config.MaxPacketMsgPayloadSize
-	sw.peerConfig.AuthEnc = config.AuthEnc
+	mConfig := conn.DefaultMConnConfig()
+	mConfig.FlushThrottle = time.Duration(cfg.FlushThrottleTimeout) * time.Millisecond
+	mConfig.SendRate = cfg.SendRate
+	mConfig.RecvRate = cfg.RecvRate
+	mConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize
+
+	sw.mConfig = mConfig
 
 	sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw)
 	return sw
@@ -403,8 +404,8 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b
 			sw.randomSleep(0)
 			err := sw.DialPeerWithAddress(addr, persistent)
 			if err != nil {
-				switch err {
-				case ErrSwitchConnectToSelf, ErrSwitchDuplicatePeer:
+				switch err.(type) {
+				case ErrSwitchConnectToSelf, ErrSwitchDuplicatePeerID:
 					sw.Logger.Debug("Error dialing peer", "err", err)
 				default:
 					sw.Logger.Error("Error dialing peer", "err", err)
@@ -420,7 +421,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b
 func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error {
 	sw.dialing.Set(string(addr.ID), addr)
 	defer sw.dialing.Delete(string(addr.ID))
-	return sw.addOutboundPeerWithConfig(addr, sw.peerConfig, persistent)
+	return sw.addOutboundPeerWithConfig(addr, sw.config, persistent)
 }
 
 // sleep for interval plus some random amount of ms on [0, dialRandomizerIntervalMilliseconds]
@@ -477,7 +478,7 @@ func (sw *Switch) listenerRoutine(l Listener) {
 		}
 
 		// New inbound connection!
-		err := sw.addInboundPeerWithConfig(inConn, sw.peerConfig)
+		err := sw.addInboundPeerWithConfig(inConn, sw.config)
 		if err != nil {
 			sw.Logger.Info("Ignoring inbound connection: error while adding peer", "address", inConn.RemoteAddr().String(), "err", err)
 			continue
@@ -487,7 +488,10 @@ func (sw *Switch) listenerRoutine(l Listener) {
 	// cleanup
 }
 
-func (sw *Switch) addInboundPeerWithConfig(conn net.Conn, config *PeerConfig) error {
+func (sw *Switch) addInboundPeerWithConfig(
+	conn net.Conn,
+	config *config.P2PConfig,
+) error {
 	peerConn, err := newInboundPeerConn(conn, config, sw.nodeKey.PrivKey)
 	if err != nil {
 		conn.Close() // peer is nil
@@ -504,10 +508,20 @@ func (sw *Switch) addInboundPeerWithConfig(conn net.Conn, config *PeerConfig) er
 // dial the peer; make secret connection; authenticate against the dialed ID;
 // add the peer.
 // if dialing fails, start the reconnect loop. If handhsake fails, its over.
-// If peer is started succesffuly, reconnectLoop will start when StopPeerForError is called
-func (sw *Switch) addOutboundPeerWithConfig(addr *NetAddress, config *PeerConfig, persistent bool) error {
+// If peer is started succesffuly, reconnectLoop will start when
+// StopPeerForError is called
+func (sw *Switch) addOutboundPeerWithConfig(
+	addr *NetAddress,
+	config *config.P2PConfig,
+	persistent bool,
+) error {
 	sw.Logger.Info("Dialing peer", "address", addr)
-	peerConn, err := newOutboundPeerConn(addr, config, persistent, sw.nodeKey.PrivKey)
+	peerConn, err := newOutboundPeerConn(
+		addr,
+		config,
+		persistent,
+		sw.nodeKey.PrivKey,
+	)
 	if err != nil {
 		if persistent {
 			go sw.reconnectToPeer(addr)
@@ -526,7 +540,8 @@ func (sw *Switch) addOutboundPeerWithConfig(addr *NetAddress, config *PeerConfig
 // that already has a SecretConnection. If all goes well,
 // it starts the peer and adds it to the switch.
 // NOTE: This performs a blocking handshake before the peer is added.
-// NOTE: If error is returned, caller is responsible for calling peer.CloseConn()
+// NOTE: If error is returned, caller is responsible for calling
+// peer.CloseConn()
 func (sw *Switch) addPeer(pc peerConn) error {
 
 	addr := pc.conn.RemoteAddr()
@@ -534,12 +549,8 @@ func (sw *Switch) addPeer(pc peerConn) error {
 		return err
 	}
 
-	// NOTE: if AuthEnc==false, we don't have a peerID until after the handshake.
-	// If AuthEnc==true then we already know the ID and could do the checks first before the handshake,
-	// but it's simple to just deal with both cases the same after the handshake.
-
 	// Exchange NodeInfo on the conn
-	peerNodeInfo, err := pc.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.peerConfig.HandshakeTimeout*time.Second))
+	peerNodeInfo, err := pc.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.HandshakeTimeout))
 	if err != nil {
 		return err
 	}
@@ -547,13 +558,14 @@ func (sw *Switch) addPeer(pc peerConn) error {
 	peerID := peerNodeInfo.ID
 
 	// ensure connection key matches self reported key
-	if pc.config.AuthEnc {
-		connID := pc.ID()
+	connID := pc.ID()
 
-		if peerID != connID {
-			return fmt.Errorf("nodeInfo.ID() (%v) doesn't match conn.ID() (%v)",
-				peerID, connID)
-		}
+	if peerID != connID {
+		return fmt.Errorf(
+			"nodeInfo.ID() (%v) doesn't match conn.ID() (%v)",
+			peerID,
+			connID,
+		)
 	}
 
 	// Validate the peers nodeInfo
@@ -564,20 +576,23 @@ func (sw *Switch) addPeer(pc peerConn) error {
 	// Avoid self
 	if sw.nodeKey.ID() == peerID {
 		addr := peerNodeInfo.NetAddress()
-
-		// remove the given address from the address book if we added it earlier
+		// remove the given address from the address book
+		// and add to our addresses to avoid dialing again
 		sw.addrBook.RemoveAddress(addr)
-
-		// add the given address to the address book to avoid dialing ourselves
-		// again this is our public address
 		sw.addrBook.AddOurAddress(addr)
-
-		return ErrSwitchConnectToSelf
+		return ErrSwitchConnectToSelf{addr}
 	}
 
 	// Avoid duplicate
 	if sw.peers.Has(peerID) {
-		return ErrSwitchDuplicatePeer
+		return ErrSwitchDuplicatePeerID{peerID}
+	}
+
+	// Check for duplicate connection or peer info IP.
+	if !sw.config.AllowDuplicateIP &&
+		(sw.peers.HasIP(pc.RemoteIP()) ||
+			sw.peers.HasIP(peerNodeInfo.NetAddress().IP)) {
+		return ErrSwitchDuplicatePeerIP{pc.RemoteIP()}
 	}
 
 	// Filter peer against ID white list
@@ -590,7 +605,7 @@ func (sw *Switch) addPeer(pc peerConn) error {
 		return err
 	}
 
-	peer := newPeer(pc, peerNodeInfo, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError)
+	peer := newPeer(pc, sw.mConfig, peerNodeInfo, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError)
 	peer.SetLogger(sw.Logger.With("peer", addr))
 
 	peer.Logger.Info("Successful handshake with peer", "peerNodeInfo", peerNodeInfo)

p2p/switch_test.go

@@ -14,17 +14,18 @@ import (
 	crypto "github.com/tendermint/go-crypto"
 	"github.com/tendermint/tmlibs/log"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p/conn"
 )
 
 var (
-	config *cfg.P2PConfig
+	cfg *config.P2PConfig
 )
 
 func init() {
-	config = cfg.DefaultP2PConfig()
-	config.PexReactor = true
+	cfg = config.DefaultP2PConfig()
+	cfg.PexReactor = true
+	cfg.AllowDuplicateIP = true
 }
 
 type PeerMessage struct {
@@ -84,7 +85,7 @@ func (tr *TestReactor) getMsgs(chID byte) []PeerMessage {
 // XXX: note this uses net.Pipe and not a proper TCP conn
 func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switch, *Switch) {
 	// Create two switches that will be interconnected.
-	switches := MakeConnectedSwitches(config, 2, initSwitch, Connect2Switches)
+	switches := MakeConnectedSwitches(cfg, 2, initSwitch, Connect2Switches)
 	return switches[0], switches[1]
 }
 
@@ -151,8 +152,8 @@ func assertMsgReceivedWithTimeout(t *testing.T, msgBytes []byte, channel byte, r
 }
 
 func TestConnAddrFilter(t *testing.T) {
-	s1 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc)
-	s2 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc)
+	s1 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc)
+	s2 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc)
 	defer s1.Stop()
 	defer s2.Stop()
 
@@ -180,20 +181,20 @@ func TestConnAddrFilter(t *testing.T) {
 }
 
 func TestSwitchFiltersOutItself(t *testing.T) {
-	s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc)
+	s1 := MakeSwitch(cfg, 1, "127.0.0.1", "123.123.123", initSwitchFunc)
 	// addr := s1.NodeInfo().NetAddress()
 
 	// // add ourselves like we do in node.go#427
 	// s1.addrBook.AddOurAddress(addr)
 
 	// simulate s1 having a public IP by creating a remote peer with the same ID
-	rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()}
+	rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: cfg}
 	rp.Start()
 
 	// addr should be rejected in addPeer based on the same ID
 	err := s1.DialPeerWithAddress(rp.Addr(), false)
 	if assert.Error(t, err) {
-		assert.Equal(t, ErrSwitchConnectToSelf, err)
+		assert.Equal(t, ErrSwitchConnectToSelf{rp.Addr()}.Error(), err.Error())
 	}
 
 	assert.True(t, s1.addrBook.OurAddress(rp.Addr()))
@@ -213,8 +214,8 @@ func assertNoPeersAfterTimeout(t *testing.T, sw *Switch, timeout time.Duration)
 }
 
 func TestConnIDFilter(t *testing.T) {
-	s1 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc)
-	s2 := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc)
+	s1 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc)
+	s2 := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc)
 	defer s1.Stop()
 	defer s2.Stop()
 
@@ -250,7 +251,7 @@ func TestConnIDFilter(t *testing.T) {
 func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 
-	sw := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc)
+	sw := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc)
 	err := sw.Start()
 	if err != nil {
 		t.Error(err)
@@ -258,11 +259,11 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) {
 	defer sw.Stop()
 
 	// simulate remote peer
-	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()}
+	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: cfg}
 	rp.Start()
 	defer rp.Stop()
 
-	pc, err := newOutboundPeerConn(rp.Addr(), DefaultPeerConfig(), false, sw.nodeKey.PrivKey)
+	pc, err := newOutboundPeerConn(rp.Addr(), cfg, false, sw.nodeKey.PrivKey)
 	require.Nil(err)
 	err = sw.addPeer(pc)
 	require.Nil(err)
@@ -280,7 +281,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) {
 func TestSwitchReconnectsToPersistentPeer(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 
-	sw := MakeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc)
+	sw := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc)
 	err := sw.Start()
 	if err != nil {
 		t.Error(err)
@@ -288,11 +289,11 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) {
 	defer sw.Stop()
 
 	// simulate remote peer
-	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()}
+	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: cfg}
 	rp.Start()
 	defer rp.Stop()
 
-	pc, err := newOutboundPeerConn(rp.Addr(), DefaultPeerConfig(), true, sw.nodeKey.PrivKey)
+	pc, err := newOutboundPeerConn(rp.Addr(), cfg, true, sw.nodeKey.PrivKey)
 	//	sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodeKey.PrivKey,
 	require.Nil(err)
 
@@ -317,14 +318,20 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) {
 	assert.False(peer.IsRunning())
 
 	// simulate another remote peer
-	rp = &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: DefaultPeerConfig()}
+	rp = &remotePeer{
+		PrivKey: crypto.GenPrivKeyEd25519(),
+		Config:  cfg,
+		// Use different interface to prevent duplicate IP filter, this will break
+		// beyond two peers.
+		listenAddr: "127.0.0.1:0",
+	}
 	rp.Start()
 	defer rp.Stop()
 
 	// simulate first time dial failure
-	peerConfig := DefaultPeerConfig()
-	peerConfig.DialFail = true
-	err = sw.addOutboundPeerWithConfig(rp.Addr(), peerConfig, true)
+	conf := config.DefaultP2PConfig()
+	conf.TestDialFail = true
+	err = sw.addOutboundPeerWithConfig(rp.Addr(), conf, true)
 	require.NotNil(err)
 
 	// DialPeerWithAddres - sw.peerConfig resets the dialer
@@ -341,7 +348,7 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) {
 }
 
 func TestSwitchFullConnectivity(t *testing.T) {
-	switches := MakeConnectedSwitches(config, 3, initSwitchFunc, Connect2Switches)
+	switches := MakeConnectedSwitches(cfg, 3, initSwitchFunc, Connect2Switches)
 	defer func() {
 		for _, sw := range switches {
 			sw.Stop()