Merge pull request #1779 from tendermint/release/v0.21.0

Release/v0.21.0

.circleci/config.yml

@@ -0,0 +1,220 @@
+version: 2
+
+defaults: &defaults
+  working_directory: /go/src/github.com/tendermint/tendermint
+  docker:
+    - image: circleci/golang:1.10.3
+  environment:
+    GOBIN: /tmp/workspace/bin
+
+jobs:
+  setup_dependencies:
+    <<: *defaults
+    steps:
+      - run: mkdir -p /tmp/workspace/bin
+      - run: mkdir -p /tmp/workspace/profiles
+      - checkout
+      - restore_cache:
+          keys:
+            - v2-pkg-cache
+      - run:
+          name: tools
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make get_tools
+      - run:
+          name: dependencies
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make get_vendor_deps
+      - run:
+          name: binaries
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make install
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - bin
+            - profiles
+      - save_cache:
+          key: v2-pkg-cache
+          paths:
+            - /go/pkg
+      - save_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          paths:
+            - /go/src/github.com/tendermint/tendermint
+
+  setup_abci:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-pkg-cache
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Checkout abci
+          command: |
+            commit=$(bash scripts/dep_utils/parse.sh abci)
+            go get -v -u -d github.com/tendermint/abci/...
+            cd /go/src/github.com/tendermint/abci
+            git checkout "$commit"
+      - run:
+          working_directory: /go/src/github.com/tendermint/abci
+          name: Install abci
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make get_tools
+            make get_vendor_deps
+            make install
+      - run: ls -lah /tmp/workspace/bin
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - "bin/abci*"
+
+  build_slate:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-pkg-cache
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: slate docs
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make build-slate
+
+  lint:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-pkg-cache
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: metalinter
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make metalinter
+
+  test_apps:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-pkg-cache
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils
+      - run:
+          name: Run tests
+          command: bash test/app/test.sh
+
+  test_cover:
+    <<: *defaults
+    parallelism: 4
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-pkg-cache
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run: mkdir -p /tmp/logs
+      - run:
+          name: Run tests
+          command: |
+            for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
+              id=$(basename "$pkg")
+
+              GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
+            done
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - "profiles/*"
+      - store_artifacts:
+          path: /tmp/logs
+
+  test_persistence:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-pkg-cache
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Run tests
+          command: bash test/persist/test_failure_indices.sh
+
+  test_p2p:
+    environment:
+      GOBIN: /home/circleci/.go_workspace/bin
+      GOPATH: /home/circleci/.go_workspace
+    machine:
+      image: circleci/classic:latest
+    steps:
+      - checkout
+      - run: mkdir -p $GOPATH/src/github.com/tendermint
+      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
+      - run: bash test/p2p/circleci.sh
+
+  upload_coverage:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: gather
+          command: |
+            set -ex
+
+            echo "mode: atomic" > coverage.txt
+            for prof in $(ls /tmp/workspace/profiles/); do
+              tail -n +2 /tmp/workspace/profiles/"$prof" >> coverage.txt
+            done
+      - run:
+          name: upload
+          command: bash <(curl -s https://codecov.io/bash) -f coverage.txt
+
+workflows:
+  version: 2
+  test-suite:
+    jobs:
+      - setup_dependencies
+      - setup_abci:
+          requires:
+            - setup_dependencies
+      - lint:
+          requires:
+            - setup_dependencies
+      - test_apps:
+          requires:
+            - setup_abci
+      - test_cover:
+          requires:
+            - setup_dependencies
+      - test_persistence:
+          requires:
+            - setup_abci
+      - test_p2p
+      - upload_coverage:
+          requires:
+            - test_cover

.codecov.yml

@@ -1,21 +0,0 @@
-
-#
-# This codecov.yml is the default configuration for
-# all repositories on Codecov. You may adjust the settings
-# below in your own codecov.yml in your repository.
-#
-coverage:
-  precision: 2
-  round: down
-  range: 70...100
-
-  status:
-    # Learn more at https://codecov.io/docs#yaml_default_commit_status
-    project: 
-      default:
-        threshold: 1% # allow this much decrease on project
-    changes: false
-
-comment:
-  layout: "header, diff"
-  behavior: default  # update if exists else create new

.editorconfig

@@ -8,10 +8,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[Makefile]
-indent_style = tab
-
-[*.sh]
+[*.{sh,Makefile}]
 indent_style = tab
 
 [*.proto]

.github/CODEOWNERS

@@ -1,5 +1,4 @@
 # CODEOWNERS: https://help.github.com/articles/about-codeowners/
 
-# Everything goes through Bucky. For now.
-*       @ebuchman
-
+# Everything goes through Bucky, Anton, Alex. For now.
+*       @ebuchman @melekes @xla

.github/ISSUE_TEMPLATE

@@ -19,10 +19,6 @@ in a case of bug.
 
 **ABCI app** (name for built-in, URL for self-written if it's publicly available):
 
-
-**Merkleeyes version** (use `git rev-parse --verify HEAD`, skip if you don't use it):
-
-
 **Environment**:
 - **OS** (e.g. from /etc/os-release):
 - **Install tools**:
@@ -37,5 +33,10 @@ in a case of bug.
 
 **How to reproduce it** (as minimally and precisely as possible):
 
+**Logs (you can paste a part showing an error or attach the whole file)**:
+
+**Config (you can paste only the changes you've made)**:
+
+**`/dump_consensus_state` output for consensus bugs**
 
 **Anything else do we need to know**:

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+<!-- Thanks for filing a PR! Before hitting the button, please check the following items.-->
+
+* [ ] Updated all relevant documentation in docs
+* [ ] Updated all code comments where relevant
+* [ ] Wrote tests
+* [ ] Updated CHANGELOG.md

.gitignore

@@ -5,7 +5,6 @@
 .DS_Store
 build/*
 rpc/test/.tendermint
-.debora
 .tendermint
 remote_dump
 .revision
@@ -13,7 +12,16 @@ vendor
 .vagrant
 test/p2p/data/
 test/logs
-.glide
 coverage.txt
 docs/_build
 docs/tools
+docs/abci-spec.md
+*.log
+
+scripts/wal2json/wal2json
+scripts/cutWALUntil/cutWALUntil
+
+.idea/
+*.iml
+
+libs/pubsub/query/fuzz_test/output

CHANGELOG.md

@@ -1,32 +1,445 @@
 # Changelog
 
-## Roadmap
+## 0.21.0
 
-BREAKING CHANGES:
-- Upgrade the header to support better proofs on validtors, results, evidence, and possibly more
-- Better support for injecting randomness
-- Pass evidence/voteInfo through ABCI
-- Upgrade consensus for more real-time use of evidence
+*June 21th, 2018*
 
-FEATURES:
-- Peer reputation management
-- Use the chain as its own CA for nodes and validators
-- Tooling to run multiple blockchains/apps, possibly in a single process
-- State syncing (without transaction replay)
-- Improved support for querying history and state
-- Add authentication and rate-limitting to the RPC
+BREAKING CHANGES
+
+- [config] Change default ports from 4665X to 2665X. Ports over 32768 are
+  ephemeral and reserved for use by the kernel.
+- [cmd] `unsafe_reset_all` removes the addrbook.json
+
+IMPROVEMENT
+
+- [pubsub] Set default capacity to 0
+- [docs] Various improvements
+
+BUG FIXES
+
+- [consensus] Fix an issue where we don't make blocks after `fast_sync` when `create_empty_blocks=false`
+- [mempool] Fix #1761 where we don't process txs if `cache_size=0`
+- [rpc] Fix memory leak in Websocket (when using `/subscribe` method)
+- [config] Escape paths in config - fixes config paths on Windows
+
+## 0.20.0
+
+*June 6th, 2018*
+
+This is the first in a series of breaking releases coming to Tendermint after
+soliciting developer feedback and conducting security audits.
+
+This release does not break any blockchain data structures or
+protocols other than the ABCI messages between Tendermint and the application.
+
+Applications that upgrade for ABCI v0.11.0 should be able to continue running Tendermint
+v0.20.0 on blockchains created with v0.19.X
+
+BREAKING CHANGES
+
+- [abci] Upgrade to
+  [v0.11.0](https://github.com/tendermint/abci/blob/master/CHANGELOG.md#0110)
+- [abci] Change Query path for filtering peers by node ID from
+  `p2p/filter/pubkey/<id>` to `p2p/filter/id/<id>`
+
+## 0.19.9
+
+*June 5th, 2018*
+
+BREAKING CHANGES
+
+- [types/priv_validator] Moved to top level `privval` package
+
+FEATURES
+
+- [config] Collapse PeerConfig into P2PConfig
+- [docs] Add quick-install script
+- [docs/spec] Add table of Amino prefixes
+
+BUG FIXES
+
+- [rpc] Return 404 for unknown endpoints
+- [consensus] Flush WAL on stop
+- [evidence] Don't send evidence to peers that are behind
+- [p2p] Fix memory leak on peer disconnects
+- [rpc] Fix panic when `per_page=0`
+
+## 0.19.8
+
+*June 4th, 2018*
+
+BREAKING:
+
+- [p2p] Remove `auth_enc` config option, peer connections are always auth
+  encrypted. Technically a breaking change but seems no one was using it and
+  arguably a bug fix :)
+
+BUG FIXES
+
+- [mempool] Fix deadlock under high load when `skip_timeout_commit=true` and
+  `create_empty_blocks=false`
+
+## 0.19.7
+
+*May 31st, 2018*
+
+BREAKING:
+
+- [libs/pubsub] TagMap#Get returns a string value
+- [libs/pubsub] NewTagMap accepts a map of strings
+
+FEATURES
+
+- [rpc] the RPC documentation is now published to https://tendermint.github.io/slate
+- [p2p] AllowDuplicateIP config option to refuse connections from same IP.
+    - true by default for now, false by default in next breaking release
+- [docs] Add docs for query, tx indexing, events, pubsub
+- [docs] Add some notes about running Tendermint in production
 
 IMPROVEMENTS:
-- Improve subtleties around mempool caching and logic
-- Consensus optimizations:
-	- cache block parts for faster agreement after round changes
-	- propagate block parts rarest first
-- Better testing of the consensus state machine (ie. use a DSL)
-- Auto compiled serialization/deserialization code instead of go-wire reflection
+
+- [consensus] Consensus reactor now receives events from a separate synchronous event bus,
+  which is not dependant on external RPC load
+- [consensus/wal] do not look for height in older files if we've seen height - 1
+- [docs] Various cleanup and link fixes
+
+## 0.19.6
+
+*May 29th, 2018*
+
+BUG FIXES
+
+- [blockchain] Fix fast-sync deadlock during high peer turnover
+
+BUG FIX:
+
+- [evidence] Dont send peers evidence from heights they haven't synced to yet
+- [p2p] Refuse connections to more than one peer with the same IP
+- [docs] Various fixes
+
+## 0.19.5
+
+*May 20th, 2018*
+
+BREAKING CHANGES
+
+- [rpc/client] TxSearch and UnconfirmedTxs have new arguments (see below)
+- [rpc/client] TxSearch returns ResultTxSearch
+- [version] Breaking changes to Go APIs will not be reflected in breaking
+  version change, but will be included in changelog.
+
+FEATURES
+
+- [rpc] `/tx_search` takes `page` (starts at 1) and `per_page` (max 100, default 30) args to paginate results
+- [rpc] `/unconfirmed_txs` takes `limit` (max 100, default 30) arg to limit the output
+- [config] `mempool.size` and `mempool.cache_size` options
+
+IMPROVEMENTS
+
+- [docs] Lots of updates
+- [consensus] Only Fsync() the WAL before executing msgs from ourselves
+
+BUG FIXES
+
+- [mempool] Enforce upper bound on number of transactions
+
+## 0.19.4 (May 17th, 2018)
+
+IMPROVEMENTS
+
+- [state] Improve tx indexing by using batches
+- [consensus, state] Improve logging (more consensus logs, fewer tx logs)
+- [spec] Moved to `docs/spec` (TODO cleanup the rest of the docs ...)
+
+BUG FIXES
+
+- [consensus] Fix issue #1575 where a late proposer can get stuck
+
+## 0.19.3 (May 14th, 2018)
+
+FEATURES
+
+- [rpc] New `/consensus_state` returns just the votes seen at the current height
+
+IMPROVEMENTS
+
+- [rpc] Add stringified votes and fraction of power voted to `/dump_consensus_state`
+- [rpc] Add PeerStateStats to `/dump_consensus_state`
+
+BUG FIXES
+
+- [cmd] Set GenesisTime during `tendermint init`
+- [consensus] fix ValidBlock rules
+
+## 0.19.2 (April 30th, 2018)
+
+FEATURES:
+
+- [p2p] Allow peers with different Minor versions to connect
+- [rpc] `/net_info` includes `n_peers`
+
+IMPROVEMENTS:
+
+- [p2p] Various code comments, cleanup, error types
+- [p2p] Change some Error logs to Debug
 
 BUG FIXES:
-- Graceful handling/recovery for apps that have non-determinism or fail to halt
-- Graceful handling/recovery for violations of safety, or liveness
+
+- [p2p] Fix reconnect to persistent peer when first dial fails
+- [p2p] Validate NodeInfo.ListenAddr
+- [p2p] Only allow (MaxNumPeers - MaxNumOutboundPeers) inbound peers
+- [p2p/pex] Limit max msg size to 64kB
+- [p2p] Fix panic when pex=false
+- [p2p] Allow multiple IPs per ID in AddrBook
+- [p2p] Fix before/after bugs in addrbook isBad()
+
+## 0.19.1 (April 27th, 2018)
+
+Note this release includes some small breaking changes in the RPC and one in the
+config that are really bug fixes. v0.19.1 will work with existing chains, and make Tendermint
+easier to use and debug. With <3
+
+BREAKING (MINOR)
+
+- [config] Removed `wal_light` setting. If you really needed this, let us know
+
+FEATURES:
+
+- [networks] moved in tooling from devops repo: terraform and ansible scripts for deploying testnets !
+- [cmd] Added `gen_node_key` command
+
+BUG FIXES
+
+Some of these are breaking in the RPC response, but they're really bugs!
+
+- [spec] Document address format and pubkey encoding pre and post Amino
+- [rpc] Lower case JSON field names
+- [rpc] Fix missing entries, improve, and lower case the fields in `/dump_consensus_state`
+- [rpc] Fix NodeInfo.Channels format to hex
+- [rpc] Add Validator address to `/status`
+- [rpc] Fix `prove` in ABCIQuery
+- [cmd] MarshalJSONIndent on init
+
+## 0.19.0 (April 13th, 2018)
+
+BREAKING:
+- [cmd] improved `testnet` command; now it can fill in `persistent_peers` for you in the config file and much more (see `tendermint testnet --help` for details)
+- [cmd] `show_node_id` now returns an error if there is no node key
+- [rpc]: changed the output format for the `/status` endpoint (see https://godoc.org/github.com/tendermint/tendermint/rpc/core#Status)
+
+Upgrade from go-wire to go-amino. This is a sweeping change that breaks everything that is
+serialized to disk or over the network.
+
+See github.com/tendermint/go-amino for details on the new format.
+
+See `scripts/wire2amino.go` for a tool to upgrade
+genesis/priv_validator/node_key JSON files.
+
+FEATURES
+
+- [test] docker-compose for local testnet setup (thanks Greg!)
+
+## 0.18.0 (April 6th, 2018)
+
+BREAKING:
+
+- [types] Merkle tree uses different encoding for varints (see tmlibs v0.8.0)
+- [types] ValidtorSet.GetByAddress returns -1 if no validator found
+- [p2p] require all addresses come with an ID no matter what
+- [rpc] Listening address must contain tcp:// or unix:// prefix
+
+FEATURES:
+
+- [rpc] StartHTTPAndTLSServer (not used yet)
+- [rpc] Include validator's voting power in `/status`
+- [rpc] `/tx` and `/tx_search` responses now include the transaction hash
+- [rpc] Include peer NodeIDs in `/net_info`
+
+IMPROVEMENTS:
+- [config] trim whitespace from elements of lists (like `persistent_peers`)
+- [rpc] `/tx_search` results are sorted by height
+- [p2p] do not try to connect to ourselves (ok, maybe only once)
+- [p2p] seeds respond with a bias towards good peers
+
+BUG FIXES:
+- [rpc] fix subscribing using an abci.ResponseDeliverTx tag
+- [rpc] fix tx_indexers matchRange
+- [rpc] fix unsubscribing (see tmlibs v0.8.0)
+
+## 0.17.1 (March 27th, 2018)
+
+BUG FIXES:
+- [types] Actually support `app_state` in genesis as `AppStateJSON`
+
+## 0.17.0 (March 27th, 2018)
+
+BREAKING:
+- [types] WriteSignBytes -> SignBytes
+
+IMPROVEMENTS:
+- [all] renamed `dummy` (`persistent_dummy`) to `kvstore` (`persistent_kvstore`) (name "dummy" is deprecated and will not work in the next breaking release)
+- [docs] note on determinism (docs/determinism.rst)
+- [genesis] `app_options` field is deprecated. please rename it to `app_state` in your genesis file(s). `app_options` will not work in the next breaking release
+- [p2p] dial seeds directly without potential peers
+- [p2p] exponential backoff for addrs in the address book
+- [p2p] mark peer as good if it contributed enough votes or block parts
+- [p2p] stop peer if it sends incorrect data, msg to unknown channel, msg we did not expect
+- [p2p] when `auth_enc` is true, all dialed peers must have a node ID in their address
+- [spec] various improvements
+- switched from glide to dep internally for package management
+- [wire] prep work for upgrading to new go-wire (which is now called go-amino)
+
+FEATURES:
+- [config] exposed `auth_enc` flag to enable/disable encryption
+- [config] added the `--p2p.private_peer_ids` flag and `PrivatePeerIDs` config variable (see config for description)
+- [rpc] added `/health` endpoint, which returns empty result for now
+- [types/priv_validator] new format and socket client, allowing for remote signing
+
+BUG FIXES:
+- [consensus] fix liveness bug by introducing ValidBlock mechanism
+
+## 0.16.0 (February 20th, 2018)
+
+BREAKING CHANGES:
+- [config] use $TMHOME/config for all config and json files
+- [p2p] old `--p2p.seeds` is now `--p2p.persistent_peers` (persistent peers to which TM will always connect to)
+- [p2p] now `--p2p.seeds` only used for getting addresses (if addrbook is empty; not persistent)
+- [p2p] NodeInfo: remove RemoteAddr and add Channels
+    - we must have at least one overlapping channel with peer
+    - we only send msgs for channels the peer advertised
+- [p2p/conn] pong timeout
+- [lite] comment out IAVL related code
+
+FEATURES:
+- [p2p] added new `/dial_peers&persistent=_` **unsafe** endpoint
+- [p2p] persistent node key in `$THMHOME/config/node_key.json`
+- [p2p] introduce peer ID and authenticate peers by ID using addresses like `ID@IP:PORT`
+- [p2p/pex] new seed mode crawls the network and serves as a seed.
+- [config] MempoolConfig.CacheSize
+- [config] P2P.SeedMode (`--p2p.seed_mode`)
+
+IMPROVEMENT:
+- [p2p/pex] stricter rules in the PEX reactor for better handling of abuse
+- [p2p] various improvements to code structure including subpackages for `pex` and `conn`
+- [docs] new spec!
+- [all] speed up the tests!
+
+BUG FIX:
+- [blockchain] StopPeerForError on timeout
+- [consensus] StopPeerForError on a bad Maj23 message
+- [state] flush mempool conn before calling commit
+- [types] fix priv val signing things that only differ by timestamp
+- [mempool] fix memory leak causing zombie peers
+- [p2p/conn] fix potential deadlock
+
+## 0.15.0 (December 29, 2017)
+
+BREAKING CHANGES:
+- [p2p] enable the Peer Exchange reactor by default
+- [types] add Timestamp field to Proposal/Vote
+- [types] add new fields to Header: TotalTxs, ConsensusParamsHash, LastResultsHash, EvidenceHash
+- [types] add Evidence to Block
+- [types] simplify ValidateBasic
+- [state] updates to support changes to the header
+- [state] Enforce <1/3 of validator set can change at a time
+
+FEATURES:
+- [state] Send indices of absent validators and addresses of byzantine validators in BeginBlock
+- [state] Historical ConsensusParams and ABCIResponses
+- [docs] Specification for the base Tendermint data structures.
+- [evidence] New evidence reactor for gossiping and managing evidence
+- [rpc] `/block_results?height=X` returns the DeliverTx results for a given height.
+
+IMPROVEMENTS:
+- [consensus] Better handling of corrupt WAL file
+
+BUG FIXES:
+- [lite] fix race
+- [state] validate block.Header.ValidatorsHash
+- [p2p] allow seed addresses to be prefixed with eg. `tcp://`
+- [p2p] use consistent key to refer to peers so we dont try to connect to existing peers
+- [cmd] fix `tendermint init` to ignore files that are there and generate files that aren't.
+
+## 0.14.0 (December 11, 2017)
+
+BREAKING CHANGES:
+- consensus/wal: removed separator
+- rpc/client: changed Subscribe/Unsubscribe/UnsubscribeAll funcs signatures to be identical to event bus.
+
+FEATURES:
+- new `tendermint lite` command (and `lite/proxy` pkg) for running a light-client RPC proxy.
+    NOTE it is currently insecure and its APIs are not yet covered by semver
+
+IMPROVEMENTS:
+- rpc/client: can act as event bus subscriber (See https://github.com/tendermint/tendermint/issues/945).
+- p2p: use exponential backoff from seconds to hours when attempting to reconnect to persistent peer
+- config: moniker defaults to the machine's hostname instead of "anonymous"
+
+BUG FIXES:
+- p2p: no longer exit if one of the seed addresses is incorrect
+
+## 0.13.0 (December 6, 2017)
+
+BREAKING CHANGES:
+- abci: update to v0.8 using gogo/protobuf; includes tx tags, vote info in RequestBeginBlock, data.Bytes everywhere, use int64, etc.
+- types: block heights are now `int64` everywhere
+- types & node: EventSwitch and EventCache have been replaced by EventBus and EventBuffer; event types have been overhauled
+- node: EventSwitch methods now refer to EventBus
+- rpc/lib/types: RPCResponse is no longer a pointer; WSRPCConnection interface has been modified
+- rpc/client: WaitForOneEvent takes an EventsClient instead of types.EventSwitch
+- rpc/client: Add/RemoveListenerForEvent are now Subscribe/Unsubscribe
+- rpc/core/types: ResultABCIQuery wraps an abci.ResponseQuery
+- rpc: `/subscribe` and `/unsubscribe` take `query` arg instead of `event`
+- rpc: `/status` returns the LatestBlockTime in human readable form instead of in nanoseconds
+- mempool: cached transactions return an error instead of an ABCI response with BadNonce
+
+FEATURES:
+- rpc: new `/unsubscribe_all` WebSocket RPC endpoint
+- rpc: new `/tx_search` endpoint for filtering transactions by more complex queries
+- p2p/trust: new trust metric for tracking peers. See ADR-006
+- config: TxIndexConfig allows to set what DeliverTx tags to index
+
+IMPROVEMENTS:
+- New asynchronous events system using `tmlibs/pubsub`
+- logging: Various small improvements
+- consensus: Graceful shutdown when app crashes
+- tests: Fix various non-deterministic errors
+- p2p: more defensive programming
+
+BUG FIXES:
+- consensus: fix panic where prs.ProposalBlockParts is not initialized
+- p2p: fix panic on bad channel
+
+## 0.12.1 (November 27, 2017)
+
+BUG FIXES:
+- upgrade tmlibs dependency to enable Windows builds for Tendermint
+
+## 0.12.0 (October 27, 2017)
+
+BREAKING CHANGES:
+ - rpc/client: websocket ResultsCh and ErrorsCh unified in ResponsesCh.
+ - rpc/client: ABCIQuery no longer takes `prove`
+ - state: remove GenesisDoc from state.
+ - consensus: new binary WAL format provides efficiency and uses checksums to detect corruption
+    - use scripts/wal2json to convert to json for debugging
+
+FEATURES:
+ - new `certifiers` pkg contains the tendermint light-client library (name subject to change)!
+ - rpc: `/genesis` includes the `app_options` .
+ - rpc: `/abci_query` takes an additional `height` parameter to support historical queries.
+ - rpc/client: new ABCIQueryWithOptions supports options like `trusted` (set false to get a proof) and `height` to query a historical height.
+
+IMPROVEMENTS:
+ - rpc: `/genesis` result includes `app_options`
+ - rpc/lib/client: add jitter to reconnects.
+ - rpc/lib/types: `RPCError` satisfies the `error` interface.
+
+BUG FIXES:
+ - rpc/client: fix ws deadlock after stopping
+ - blockchain: fix panic on AddBlock when peer is nil
+ - mempool: fix sending on TxsAvailable when a tx has been invalidated
+ - consensus: dont run WAL catchup if we fast synced
 
 ## 0.11.1 (October 10, 2017)
 

CONTRIBUTING.md

@@ -8,9 +8,9 @@ Please make sure to use `gofmt` before every commit - the easiest way to do this
 
 ## Forking
 
-Please note that Go requires code to live under absolute paths, which complicates forking. 
-While my fork lives at `https://github.com/ebuchman/tendermint`, 
-the code should never exist at  `$GOPATH/src/github.com/ebuchman/tendermint`. 
+Please note that Go requires code to live under absolute paths, which complicates forking.
+While my fork lives at `https://github.com/ebuchman/tendermint`,
+the code should never exist at  `$GOPATH/src/github.com/ebuchman/tendermint`.
 Instead, we use `git remote` to add the fork as a new remote for the original repo,
 `$GOPATH/src/github.com/tendermint/tendermint `, and do all the work there.
 
@@ -34,16 +34,44 @@ Please don't make Pull Requests to `master`.
 
 ## Dependencies
 
-We use [glide](https://github.com/masterminds/glide) to manage dependencies.
-That said, the master branch of every Tendermint repository should just build with `go get`, which means they should be kept up-to-date with their dependencies so we can get away with telling people they can just `go get` our software.
-Since some dependencies are not under our control, a third party may break our build, in which case we can fall back on `glide install`. Even for dependencies under our control, glide helps us keeps multiple repos in sync as they evolve. Anything with an executable, such as apps, tools, and the core, should use glide.
+We use [dep](https://github.com/golang/dep) to manage dependencies.
 
-Run `bash scripts/glide/status.sh` to get a list of vendored dependencies that may not be up-to-date. 
+That said, the master branch of every Tendermint repository should just build
+with `go get`, which means they should be kept up-to-date with their
+dependencies so we can get away with telling people they can just `go get` our
+software.
+
+Since some dependencies are not under our control, a third party may break our
+build, in which case we can fall back on `dep ensure` (or `make
+get_vendor_deps`). Even for dependencies under our control, dep helps us to
+keep multiple repos in sync as they evolve. Anything with an executable, such
+as apps, tools, and the core, should use dep.
+
+Run `dep status` to get a list of vendored dependencies that may not be
+up-to-date.
+
+## Vagrant
+
+If you are a [Vagrant](https://www.vagrantup.com/) user, you can get started
+hacking Tendermint with the commands below.
+
+NOTE: In case you installed Vagrant in 2017, you might need to run
+`vagrant box update` to upgrade to the latest `ubuntu/xenial64`.
+
+```
+vagrant up
+vagrant ssh
+make test
+```
 
 ## Testing
 
-All repos should be hooked up to circle. 
-If they have `.go` files in the root directory, they will be automatically tested by circle using `go test -v -race ./...`. If not, they will need a `circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and includes its continuous integration status using a badge in the `README.md`.
+All repos should be hooked up to [CircleCI](https://circleci.com/).
+
+If they have `.go` files in the root directory, they will be automatically
+tested by circle using `go test -v -race ./...`. If not, they will need a
+`circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and
+includes its continuous integration status using a badge in the `README.md`.
 
 ## Branching Model and Release
 
@@ -75,3 +103,15 @@ especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint
 - push to release-vX.X.X to run the extended integration tests on the CI
 - merge to master
 - merge master back to develop
+
+### Hotfix Procedure:
+- start on `master`
+- checkout a new branch named hotfix-vX.X.X
+- make the required changes
+  - these changes should be small and an absolute necessity
+  - add a note to CHANGELOG.md
+- bumb versions
+- push to hotfix-vX.X.X to run the extended integration tests on the CI
+- merge hotfix-vX.X.X to master
+- merge hotfix-vX.X.X to develop
+- delete the hotfix-vX.X.X branch

DOCKER/.gitignore

@@ -0,0 +1 @@
+tendermint

DOCKER/Dockerfile

@@ -1,45 +1,39 @@
-FROM alpine:3.6
+FROM alpine:3.7
+MAINTAINER Greg Szabo <greg@tendermint.com>
 
-# This is the release of tendermint to pull in.
-ENV TM_VERSION 0.11.0
-ENV TM_SHA256SUM 7e443bac4d42f12e7beaf9cee63b4a565dad8c58895291fdedde8057088b70c5
-
-# Tendermint will be looking for genesis file in /tendermint (unless you change
-# `genesis_file` in config.toml). You can put your config.toml and private
-# validator file into /tendermint.
+# Tendermint will be looking for the genesis file in /tendermint/config/genesis.json
+# (unless you change `genesis_file` in config.toml). You can put your config.toml and
+# private validator file into /tendermint/config.
 #
 # The /tendermint/data dir is used by tendermint to store state.
-ENV DATA_ROOT /tendermint
-ENV TMHOME $DATA_ROOT
-
-# Set user right away for determinism
-RUN addgroup tmuser && \
-    adduser -S -G tmuser tmuser
-
-# Create directory for persistence and give our user ownership
-RUN mkdir -p $DATA_ROOT && \
-    chown -R tmuser:tmuser $DATA_ROOT
+ENV TMHOME /tendermint
 
+# OS environment setup
+# Set user right away for determinism, create directory for persistence and give our user ownership
 # jq and curl used for extracting `pub_key` from private validator while
 # deploying tendermint with Kubernetes. It is nice to have bash so the users
 # could execute bash commands.
-RUN apk add --no-cache bash curl jq
+RUN apk update && \
+    apk upgrade && \
+    apk --no-cache add curl jq bash && \
+    addgroup tmuser && \
+    adduser -S -G tmuser tmuser -h "$TMHOME"
 
-RUN apk add --no-cache openssl && \
-    wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \
-    echo "${TM_SHA256SUM}  tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \
-    unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \
-    apk del openssl && \
-    rm -f tendermint_${TM_VERSION}_linux_amd64.zip
+# Run the container with tmuser by default. (UID=100, GID=1000)
+USER tmuser
 
 # Expose the data directory as a volume since there's mutable state in there
-VOLUME $DATA_ROOT
+VOLUME [ $TMHOME ]
 
-# p2p port
-EXPOSE 46656
-# rpc port
-EXPOSE 46657
+WORKDIR $TMHOME
 
-ENTRYPOINT ["tendermint"]
+# p2p and rpc port
+EXPOSE 26656 26657
 
+ENTRYPOINT ["/usr/bin/tendermint"]
 CMD ["node", "--moniker=`hostname`"]
+STOPSIGNAL SIGTERM
+
+ARG BINARY=tendermint
+COPY $BINARY /usr/bin/tendermint
+

DOCKER/Dockerfile.develop

@@ -1,4 +1,4 @@
-FROM alpine:3.6
+FROM alpine:3.7
 
 ENV DATA_ROOT /tendermint
 ENV TMHOME $DATA_ROOT
@@ -18,18 +18,18 @@ RUN mkdir -p /go/src/github.com/tendermint/tendermint && \
     cd /go/src/github.com/tendermint/tendermint && \
     git clone https://github.com/tendermint/tendermint . && \
     git checkout develop && \
+    make get_tools && \
     make get_vendor_deps && \
     make install && \
-    glide cc && \
     cd - && \
     rm -rf /go/src/github.com/tendermint/tendermint && \
     apk del go build-base git
 
 VOLUME $DATA_ROOT
 
-EXPOSE 46656
-EXPOSE 46657
+EXPOSE 26656
+EXPOSE 26657
 
 ENTRYPOINT ["tendermint"]
 
-CMD ["node", "--moniker=`hostname`", "--proxy_app=dummy"]
+CMD ["node", "--moniker=`hostname`", "--proxy_app=kvstore"]

DOCKER/Dockerfile.testing

@@ -0,0 +1,18 @@
+FROM golang:1.10.1
+
+
+# Grab deps (jq, hexdump, xxd, killall)
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends \
+  jq bsdmainutils vim-common psmisc netcat
+
+# Add testing deps for curl
+RUN echo 'deb http://httpredir.debian.org/debian testing main non-free contrib' >> /etc/apt/sources.list && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends curl
+
+VOLUME /go
+
+EXPOSE 26656
+EXPOSE 26657
+

DOCKER/Makefile

@@ -7,6 +7,9 @@ push:
 build_develop:
 	docker build -t "tendermint/tendermint:develop" -f Dockerfile.develop .
 
+build_testing:
+	docker build --tag tendermint/testing -f ./Dockerfile.testing .
+
 push_develop:
 	docker push "tendermint/tendermint:develop"
 

DOCKER/README.md

@@ -1,6 +1,13 @@
-# Supported tags and respective `Dockerfile` links
+# Docker
 
-- `0.11.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/9177cc1f64ca88a4a0243c5d1773d10fba67e201/DOCKER/Dockerfile)
+## Supported tags and respective `Dockerfile` links
+
+- `0.17.1`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/208ac32fa266657bd6c304e84ec828aa252bb0b8/DOCKER/Dockerfile)
+- `0.15.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile)
+- `0.13.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/a28b3fff49dce2fb31f90abb2fc693834e0029c2/DOCKER/Dockerfile)
+- `0.12.1` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/457c688346b565e90735431619ca3ca597ef9007/DOCKER/Dockerfile)
+- `0.12.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/70d8afa6e952e24c573ece345560a5971bf2cc0e/DOCKER/Dockerfile)
+- `0.11.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/9177cc1f64ca88a4a0243c5d1773d10fba67e201/DOCKER/Dockerfile)
 - `0.10.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/e5342f4054ab784b2cd6150e14f01053d7c8deb2/DOCKER/Dockerfile)
 - `0.9.1`, `0.9`, [(Dockerfile)](https://github.com/tendermint/tendermint/blob/809e0e8c5933604ba8b2d096803ada7c5ec4dfd3/DOCKER/Dockerfile)
 - `0.9.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/d474baeeea6c22b289e7402449572f7c89ee21da/DOCKER/Dockerfile)
@@ -9,10 +16,10 @@
 
 `develop` tag points to the [develop](https://github.com/tendermint/tendermint/tree/develop) branch.
 
-# Quick reference
+## Quick reference
 
 * **Where to get help:**
-  [Chat on Rocket](https://cosmos.rocket.chat/)
+  https://cosmos.network/community
 
 * **Where to file issues:**
   https://github.com/tendermint/tendermint/issues
@@ -20,7 +27,7 @@
 * **Supported Docker versions:**
   [the latest release](https://github.com/moby/moby/releases) (down to 1.6 on a best-effort basis)
 
-# Tendermint
+## Tendermint
 
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines.
 
@@ -28,29 +35,33 @@ For more background, see the [introduction](https://tendermint.readthedocs.io/en
 
 To get started developing applications, see the [application developers guide](https://tendermint.readthedocs.io/en/master/getting-started.html).
 
-# How to use this image
+## How to use this image
 
-## Start one instance of the Tendermint core with the `dummy` app
+### Start one instance of the Tendermint core with the `kvstore` app
 
-A very simple example of a built-in app and Tendermint core in one container.
+A quick example of a built-in app and Tendermint core in one container.
 
 ```
 docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init
-docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint node --proxy_app=dummy
+docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint node --proxy_app=kvstore
 ```
 
-## mintnet-kubernetes
+## Local cluster
 
-If you want to see many containers talking to each other, consider using [mintnet-kubernetes](https://github.com/tendermint/tools/tree/master/mintnet-kubernetes), which is a tool for running Tendermint-based applications on a Kubernetes cluster.
+To run a 4-node network, see the `Makefile` in the root of [the repo](https://github.com/tendermint/tendermint/master/Makefile) and run:
 
-# License
+```
+make build-linux
+make build-docker-localnode
+make localnet-start
+```
 
-View [license information](https://raw.githubusercontent.com/tendermint/tendermint/master/LICENSE) for the software contained in this image.
+Note that this will build and use a different image than the ones provided here.
 
-# User Feedback
+## License
+
+- Tendermint's license is [Apache 2.0](https://github.com/tendermint/tendermint/master/LICENSE).
 
 ## Contributing
 
-You are invited to contribute new features, fixes, or updates, large or small; we are always thrilled to receive pull requests, and do our best to process them as fast as we can.
-
-Before you start to code, we recommend discussing your plans through a [GitHub](https://github.com/tendermint/tendermint/issues) issue, especially for more ambitious contributions. This gives other contributors a chance to point you in the right direction, give you feedback on your design, and help you find out if someone else is working on the same thing.
+Contributions are most welcome! See the [contributing file](https://github.com/tendermint/tendermint/blob/master/CONTRIBUTING.md) for more information.

Gopkg.lock

@@ -0,0 +1,379 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  branch = "master"
+  name = "github.com/btcsuite/btcd"
+  packages = ["btcec"]
+  revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64"
+
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+  version = "v1.1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/ebuchman/fail-test"
+  packages = ["."]
+  revision = "95f809107225be108efcf10a3509e4ea6ceef3c4"
+
+[[projects]]
+  name = "github.com/fortytw2/leaktest"
+  packages = ["."]
+  revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e"
+  version = "v1.2.0"
+
+[[projects]]
+  name = "github.com/fsnotify/fsnotify"
+  packages = ["."]
+  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
+  version = "v1.4.7"
+
+[[projects]]
+  name = "github.com/go-kit/kit"
+  packages = [
+    "log",
+    "log/level",
+    "log/term"
+  ]
+  revision = "4dc7be5d2d12881735283bcab7352178e190fc71"
+  version = "v0.6.0"
+
+[[projects]]
+  name = "github.com/go-logfmt/logfmt"
+  packages = ["."]
+  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
+  version = "v0.3.0"
+
+[[projects]]
+  name = "github.com/go-stack/stack"
+  packages = ["."]
+  revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc"
+  version = "v1.7.0"
+
+[[projects]]
+  name = "github.com/gogo/protobuf"
+  packages = [
+    "gogoproto",
+    "jsonpb",
+    "proto",
+    "protoc-gen-gogo/descriptor",
+    "sortkeys",
+    "types"
+  ]
+  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/golang/protobuf"
+  packages = [
+    "proto",
+    "ptypes",
+    "ptypes/any",
+    "ptypes/duration",
+    "ptypes/timestamp"
+  ]
+  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/snappy"
+  packages = ["."]
+  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
+
+[[projects]]
+  name = "github.com/gorilla/websocket"
+  packages = ["."]
+  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
+  version = "v1.2.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/hashicorp/hcl"
+  packages = [
+    ".",
+    "hcl/ast",
+    "hcl/parser",
+    "hcl/printer",
+    "hcl/scanner",
+    "hcl/strconv",
+    "hcl/token",
+    "json/parser",
+    "json/scanner",
+    "json/token"
+  ]
+  revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168"
+
+[[projects]]
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/jmhodges/levigo"
+  packages = ["."]
+  revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/kr/logfmt"
+  packages = ["."]
+  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
+
+[[projects]]
+  name = "github.com/magiconair/properties"
+  packages = ["."]
+  revision = "c2353362d570a7bfa228149c62842019201cfb71"
+  version = "v1.8.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mitchellh/mapstructure"
+  packages = ["."]
+  revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b"
+
+[[projects]]
+  name = "github.com/pelletier/go-toml"
+  packages = ["."]
+  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
+  version = "v1.2.0"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/rcrowley/go-metrics"
+  packages = ["."]
+  revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
+
+[[projects]]
+  name = "github.com/spf13/afero"
+  packages = [
+    ".",
+    "mem"
+  ]
+  revision = "787d034dfe70e44075ccc060d346146ef53270ad"
+  version = "v1.1.1"
+
+[[projects]]
+  name = "github.com/spf13/cast"
+  packages = ["."]
+  revision = "8965335b8c7107321228e3e3702cab9832751bac"
+  version = "v1.2.0"
+
+[[projects]]
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+  version = "v0.0.3"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/spf13/jwalterweatherman"
+  packages = ["."]
+  revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
+
+[[projects]]
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
+  version = "v1.0.1"
+
+[[projects]]
+  name = "github.com/spf13/viper"
+  packages = ["."]
+  revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736"
+  version = "v1.0.2"
+
+[[projects]]
+  name = "github.com/stretchr/testify"
+  packages = [
+    "assert",
+    "require"
+  ]
+  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+  version = "v1.2.2"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/syndtr/goleveldb"
+  packages = [
+    "leveldb",
+    "leveldb/cache",
+    "leveldb/comparer",
+    "leveldb/errors",
+    "leveldb/filter",
+    "leveldb/iterator",
+    "leveldb/journal",
+    "leveldb/memdb",
+    "leveldb/opt",
+    "leveldb/storage",
+    "leveldb/table",
+    "leveldb/util"
+  ]
+  revision = "e2150783cd35f5b607daca48afd8c57ec54cc995"
+
+[[projects]]
+  name = "github.com/tendermint/abci"
+  packages = [
+    "client",
+    "example/code",
+    "example/counter",
+    "example/kvstore",
+    "server",
+    "types"
+  ]
+  revision = "198dccf0ddfd1bb176f87657e3286a05a6ed9540"
+  version = "v0.12.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/tendermint/ed25519"
+  packages = [
+    ".",
+    "edwards25519",
+    "extra25519"
+  ]
+  revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057"
+
+[[projects]]
+  name = "github.com/tendermint/go-amino"
+  packages = ["."]
+  revision = "ed62928576cfcaf887209dc96142cd79cdfff389"
+  version = "0.9.9"
+
+[[projects]]
+  name = "github.com/tendermint/go-crypto"
+  packages = ["."]
+  revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19"
+  version = "v0.6.2"
+
+[[projects]]
+  name = "github.com/tendermint/tmlibs"
+  packages = [
+    "autofile",
+    "cli",
+    "cli/flags",
+    "clist",
+    "common",
+    "db",
+    "flowrate",
+    "log",
+    "merkle",
+    "test"
+  ]
+  revision = "692f1d86a6e2c0efa698fd1e4541b68c74ffaf38"
+  version = "v0.8.4"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = [
+    "curve25519",
+    "nacl/box",
+    "nacl/secretbox",
+    "openpgp/armor",
+    "openpgp/errors",
+    "poly1305",
+    "ripemd160",
+    "salsa20/salsa"
+  ]
+  revision = "8ac0e0d97ce45cd83d1d7243c060cb8461dda5e9"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/net"
+  packages = [
+    "context",
+    "http/httpguts",
+    "http2",
+    "http2/hpack",
+    "idna",
+    "internal/timeseries",
+    "trace"
+  ]
+  revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  revision = "a9e25c09b96b8870693763211309e213c6ef299d"
+
+[[projects]]
+  name = "golang.org/x/text"
+  packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable"
+  ]
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[[projects]]
+  name = "google.golang.org/genproto"
+  packages = ["googleapis/rpc/status"]
+  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
+
+[[projects]]
+  name = "google.golang.org/grpc"
+  packages = [
+    ".",
+    "balancer",
+    "codes",
+    "connectivity",
+    "credentials",
+    "grpclb/grpc_lb_v1/messages",
+    "grpclog",
+    "internal",
+    "keepalive",
+    "metadata",
+    "naming",
+    "peer",
+    "resolver",
+    "stats",
+    "status",
+    "tap",
+    "transport"
+  ]
+  revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e"
+  version = "v1.7.5"
+
+[[projects]]
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+  version = "v2.2.1"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "62d0626fa963b25411234da915fb3b8bfc7aefffd76824d27ac1647ca2b5d489"
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,99 @@
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/ebuchman/fail-test"
+  branch = "master"
+
+[[constraint]]
+  name = "github.com/fortytw2/leaktest"
+  branch = "master"
+
+[[constraint]]
+  name = "github.com/go-kit/kit"
+  version = "~0.6.0"
+
+[[constraint]]
+  name = "github.com/gogo/protobuf"
+  version = "~1.0.0"
+
+[[constraint]]
+  name = "github.com/golang/protobuf"
+  version = "~1.0.0"
+
+[[constraint]]
+  name = "github.com/gorilla/websocket"
+  version = "~1.2.0"
+
+[[constraint]]
+  name = "github.com/pkg/errors"
+  version = "~0.8.0"
+
+[[constraint]]
+  name = "github.com/rcrowley/go-metrics"
+  branch = "master"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "~0.0.1"
+
+[[constraint]]
+  name = "github.com/spf13/viper"
+  version = "~1.0.0"
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "~1.2.1"
+
+[[constraint]]
+  name = "github.com/tendermint/abci"
+  version = "~0.12.0"
+
+[[constraint]]
+  name = "github.com/tendermint/go-crypto"
+  version = "~0.6.2"
+
+[[constraint]]
+  name = "github.com/tendermint/go-amino"
+  version = "=0.9.9"
+
+[[override]]
+  name = "github.com/tendermint/tmlibs"
+  version = "~0.8.4"
+
+[[constraint]]
+  name = "google.golang.org/grpc"
+  version = "~1.7.3"
+
+# this got updated and broke, so locked to an old working commit ...
+[[override]]
+  name = "google.golang.org/genproto"
+  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
+
+[prune]
+  go-tests = true
+  unused-packages = true

INSTALL.md

@@ -1 +0,0 @@
-The installation guide has moved to the [docs directory](docs/guides/install-from-source.md) in order to easily be rendered by the website. Please update your links accordingly.

Makefile

@@ -1,29 +1,135 @@
 GOTOOLS = \
-					github.com/mitchellh/gox \
-					github.com/tcnksm/ghr \
-					github.com/Masterminds/glide \
-
+	github.com/golang/dep/cmd/dep \
+	gopkg.in/alecthomas/gometalinter.v2
 PACKAGES=$(shell go list ./... | grep -v '/vendor/')
 BUILD_TAGS?=tendermint
-TMHOME = $${TMHOME:-$$HOME/.tendermint}
+BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
 
-all: install test
+all: check build test install
 
-install: get_vendor_deps
-	@go install --ldflags '-extldflags "-static"' \
-		--ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" ./cmd/tendermint
+check: check_tools ensure_deps
+
+
+########################################
+### Build
 
 build:
-	go build \
-		--ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`"  -o build/tendermint ./cmd/tendermint/
+	CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/
 
 build_race:
-	go build -race -o build/tendermint ./cmd/tendermint
+	CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint
+
+install:
+	CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint
+
+########################################
+### Distribution
 
 # dist builds binaries for all platforms and packages them for distribution
 dist:
 	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
 
+########################################
+### Tools & dependencies
+
+check_tools:
+	@# https://stackoverflow.com/a/25668869
+	@echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\
+        $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))"
+
+get_tools:
+	@echo "--> Installing tools"
+	go get -u -v $(GOTOOLS)
+	@gometalinter.v2 --install
+
+update_tools:
+	@echo "--> Updating tools"
+	@go get -u $(GOTOOLS)
+
+#Run this from CI
+get_vendor_deps:
+	@rm -rf vendor/
+	@echo "--> Running dep"
+	@dep ensure -vendor-only
+
+
+#Run this locally.
+ensure_deps:
+	@rm -rf vendor/
+	@echo "--> Running dep"
+	@dep ensure
+
+draw_deps:
+	@# requires brew install graphviz or apt-get install graphviz
+	go get github.com/RobotsAndPencils/goviz
+	@goviz -i github.com/tendermint/tendermint/cmd/tendermint -d 3 | dot -Tpng -o dependency-graph.png
+
+get_deps_bin_size:
+	@# Copy of build recipe with additional flags to perform binary size analysis
+	$(eval $(shell go build -work -a $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ 2>&1))
+	@find $(WORK) -type f -name "*.a" | xargs -I{} du -hxs "{}" | sort -rh | sed -e s:${WORK}/::g > deps_bin_size.log
+	@echo "Results can be found here: $(CURDIR)/deps_bin_size.log"
+
+########################################
+### Testing
+
+## required to be run first by most tests
+build_docker_test_image:
+	docker build -t tester -f ./test/docker/Dockerfile .
+
+### coverage, app, persistence, and libs tests
+test_cover:
+	# run the go unit tests with coverage
+	bash test/test_cover.sh
+
+test_apps:
+	# run the app tests using bash
+	# requires `abci-cli` and `tendermint` binaries installed
+	bash test/app/test.sh
+
+test_persistence:
+	# run the persistence tests using bash
+	# requires `abci-cli` installed
+	docker run --name run_persistence -t tester bash test/persist/test_failure_indices.sh
+
+	# TODO undockerize
+	# bash test/persist/test_failure_indices.sh
+
+test_p2p:
+	docker rm -f rsyslog || true
+	rm -rf test/logs || true
+	mkdir test/logs
+	cd test/
+	docker run -d -v "logs:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+	cd ..
+	# requires 'tester' the image from above
+	bash test/p2p/test.sh tester
+
+need_abci:
+	bash scripts/install_abci_apps.sh
+
+test_integrations:
+	make build_docker_test_image
+	make get_tools
+	make get_vendor_deps
+	make install
+	make need_abci
+	make test_cover
+	make test_apps
+	make test_persistence
+	make test_p2p
+
+test_release:
+	@go test -tags release $(PACKAGES)
+
+test100:
+	@for i in {1..100}; do make test; done
+
+vagrant_test:
+	vagrant up
+	vagrant ssh -c 'make test_integrations'
+
+### go tests
 test:
 	@echo "--> Running go test"
 	@go test $(PACKAGES)
@@ -32,52 +138,99 @@ test_race:
 	@echo "--> Running go test --race"
 	@go test -v -race $(PACKAGES)
 
-test_integrations:
-	@bash ./test/test.sh
-
-test100:
-	@for i in {1..100}; do make test; done
-
-draw_deps:
-	# requires brew install graphviz or apt-get install graphviz
-	go get github.com/RobotsAndPencils/goviz
-	@goviz -i github.com/tendermint/tendermint/cmd/tendermint -d 3 | dot -Tpng -o dependency-graph.png
-
-list_deps:
-	@go list -f '{{join .Deps "\n"}}' ./... | \
-		grep -v /vendor/ | sort | uniq | \
-		xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
-
-get_deps:
-	@echo "--> Running go get"
-	@go get -v -d $(PACKAGES)
-	@go list -f '{{join .TestImports "\n"}}' ./... | \
-		grep -v /vendor/ | sort | uniq | \
-		xargs go get -v -d
-
-get_vendor_deps: ensure_tools
-	@rm -rf vendor/
-	@echo "--> Running glide install"
-	@glide install
-
-update_deps: tools
-	@echo "--> Updating dependencies"
-	@go get -d -u ./...
-
-revision:
-	-echo `git rev-parse --verify HEAD` > $(TMHOME)/revision
-	-echo `git rev-parse --verify HEAD` >> $(TMHOME)/revision_history
-
-tools:
-	go get -u -v $(GOTOOLS)
-
-ensure_tools:
-	go get $(GOTOOLS)
 
+########################################
 ### Formatting, linting, and vetting
 
-megacheck:
-	@for pkg in ${PACKAGES}; do megacheck "$$pkg"; done
+fmt:
+	@go fmt ./...
 
+metalinter:
+	@echo "--> Running linter"
+	@gometalinter.v2 --vendor --deadline=600s --disable-all  \
+		--enable=deadcode \
+		--enable=gosimple \
+	 	--enable=misspell \
+		--enable=safesql \
+		./...
+		#--enable=gas \
+		#--enable=maligned \
+		#--enable=dupl \
+		#--enable=errcheck \
+		#--enable=goconst \
+		#--enable=gocyclo \
+		#--enable=goimports \
+		#--enable=golint \ <== comments on anything exported
+		#--enable=gotype \
+	 	#--enable=ineffassign \
+	   	#--enable=interfacer \
+	   	#--enable=megacheck \
+	   	#--enable=staticcheck \
+	   	#--enable=structcheck \
+	   	#--enable=unconvert \
+	   	#--enable=unparam \
+		#--enable=unused \
+	   	#--enable=varcheck \
+		#--enable=vet \
+		#--enable=vetshadow \
 
-.PHONY: install build build_race dist test test_race test_integrations test100 draw_deps list_deps get_deps get_vendor_deps update_deps revision tools
+metalinter_all:
+	@echo "--> Running linter (all)"
+	gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./...
+
+###########################################################
+### Docker image
+
+build-docker:
+	cp build/tendermint DOCKER/tendermint
+	docker build --label=tendermint --tag="tendermint/tendermint" DOCKER
+	rm -rf DOCKER/tendermint
+
+###########################################################
+### Local testnet using docker
+
+# Build linux binary on other platforms
+build-linux:
+	GOOS=linux GOARCH=amd64 $(MAKE) build
+
+build-docker-localnode:
+	cd networks/local
+	make
+
+# Run a 4-node testnet locally
+localnet-start: localnet-stop
+	@if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi
+	docker-compose up
+
+# Stop testnet
+localnet-stop:
+	docker-compose down
+
+###########################################################
+### Remote full-nodes (sentry) using terraform and ansible
+
+# Server management
+sentry-start:
+	@if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi
+	@if ! [ -f $(HOME)/.ssh/id_rsa.pub ]; then ssh-keygen ; fi
+	cd networks/remote/terraform && terraform init && terraform apply -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub"
+	@if ! [ -f $(CURDIR)/build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 0 --n 4 --o . ; fi
+	cd networks/remote/ansible && ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml
+	@echo "Next step: Add your validator setup in the genesis.json and config.tml files and run \"make sentry-config\". (Public key of validator, chain ID, peer IP and node ID.)"
+
+# Configuration management
+sentry-config:
+	cd networks/remote/ansible && ansible-playbook -i inventory/digital_ocean.py -l sentrynet config.yml -e BINARY=$(CURDIR)/build/tendermint -e CONFIGDIR=$(CURDIR)/build
+
+sentry-stop:
+	@if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi
+	cd networks/remote/terraform && terraform destroy -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub"
+
+# meant for the CI, inspect script & adapt accordingly
+build-slate:
+	bash scripts/slate.sh
+
+# To avoid unintended conflicts with file names, always add to .PHONY
+# unless there is a reason not to.
+# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
+.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate

README.md

@@ -8,7 +8,8 @@ Or [Blockchain](https://en.wikipedia.org/wiki/Blockchain_(database)) for short.
 [![API Reference](
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/tendermint/tendermint)
-[![Rocket.Chat](https://demo.rocket.chat/images/join-chat.svg)](https://cosmos.rocket.chat/)
+[![Go version](https://img.shields.io/badge/go-1.9.2-blue.svg)](https://github.com/moovweb/gvm)
+[![riot.im](https://img.shields.io/badge/riot.im-JOIN%20CHAT-green.svg)](https://riot.im/app/#/room/#tendermint:matrix.org)
 [![license](https://img.shields.io/github/license/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/blob/master/LICENSE)
 [![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint)
 
@@ -18,28 +19,56 @@ Branch    | Tests | Coverage
 master    | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint)
 develop   | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/develop/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint)
 
-_NOTE: This is alpha software. Please contact us if you intend to run it in production._
-
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language -
 and securely replicates it on many machines.
 
-For more information, from introduction to install to application development, [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master).
+For protocol details, see [the specification](/docs/spec).
+
+## A Note on Production Readiness
+
+While Tendermint is being used in production in private, permissioned
+environments, we are still working actively to harden and audit it in preparation
+for use in public blockchains, such as the [Cosmos Network](https://cosmos.network/).
+We are also still making breaking changes to the protocol and the APIs.
+Thus we tag the releases as *alpha software*.
+
+In any case, if you intend to run Tendermint in production,
+please [contact us](https://riot.im/app/#/room/#tendermint:matrix.org) :)
+
+## Security
+
+To report a security vulnerability, see our [bug bounty
+program](https://tendermint.com/security).
+
+For examples of the kinds of bugs we're looking for, see [SECURITY.md](SECURITY.md)
+
+## Minimum requirements
+
+Requirement|Notes
+---|---
+Go version | Go1.9 or higher
 
 ## Install
 
-To download pre-built binaries, see our [downloads page](https://tendermint.com/downloads).
+See the [install instructions](/docs/install.rst)
 
-To install from source, you should be able to:
+## Quick Start
 
-`go get -u github.com/tendermint/tendermint/cmd/tendermint`
-
-For more details (or if it fails), [read the docs](http://tendermint.readthedocs.io/projects/tools/en/master/install.html).
+- [Single node](/docs/using-tendermint.rst)
+- [Local cluster using docker-compose](/networks/local)
+- [Remote cluster using terraform and ansible](/docs/terraform-and-ansible.rst)
+- [Join the public testnet](https://cosmos.network/testnet)
 
 ## Resources
 
 ### Tendermint Core
 
-All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs.
+For details about the blockchain data structures and the p2p protocols, see the
+the [Tendermint specification](/docs/spec).
+
+For details on using the software, [Read The Docs](https://tendermint.readthedocs.io/en/master/).
+Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs.
+
 
 ### Sub-projects
 
@@ -54,8 +83,8 @@ All resources involving the use of, building application on, or developing for,
 
 ### Applications
 
-* [Ethermint](http://github.com/tendermint/ethermint); Ethereum on Tendermint
 * [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework
+* [Ethermint](http://github.com/tendermint/ethermint); Ethereum on Tendermint
 * [Many more](https://tendermint.readthedocs.io/en/master/ecosystem.html#abci-applications)
 
 ### More
@@ -78,7 +107,11 @@ According to SemVer, anything in the public API can change at any time before ve
 
 To provide some stability to Tendermint users in these 0.X.X days, the MINOR version is used
 to signal breaking changes across a subset of the total public API. This subset includes all
-interfaces exposed to other processes (cli, rpc, p2p, etc.), as well as parts of the following packages:
+interfaces exposed to other processes (cli, rpc, p2p, etc.), but does not
+include the in-process Go APIs.
+
+That said, breaking changes in the following packages will be documented in the
+CHANGELOG even if they don't lead to MINOR version bumps:
 
 - types
 - rpc/client

ROADMAP.md

@@ -0,0 +1,23 @@
+# Roadmap
+
+BREAKING CHANGES:
+- Better support for injecting randomness
+- Upgrade consensus for more real-time use of evidence
+
+FEATURES:
+- Use the chain as its own CA for nodes and validators
+- Tooling to run multiple blockchains/apps, possibly in a single process
+- State syncing (without transaction replay)
+- Add authentication and rate-limitting to the RPC
+
+IMPROVEMENTS:
+- Improve subtleties around mempool caching and logic
+- Consensus optimizations:
+	- cache block parts for faster agreement after round changes
+	- propagate block parts rarest first
+- Better testing of the consensus state machine (ie. use a DSL)
+- Auto compiled serialization/deserialization code instead of go-wire reflection
+
+BUG FIXES:
+- Graceful handling/recovery for apps that have non-determinism or fail to halt
+- Graceful handling/recovery for violations of safety, or liveness

SECURITY.md

@@ -0,0 +1,71 @@
+# Security
+
+As part of our [Coordinated Vulnerability Disclosure
+Policy](https://tendermint.com/security), we operate a bug bounty.
+See the policy for more details on submissions and rewards.
+
+Here is a list of examples of the kinds of bugs we're most interested in:
+
+## Specification
+
+- Conceptual flaws
+- Ambiguities, inconsistencies, or incorrect statements
+- Mis-match between specification and implementation of any component
+
+## Consensus
+
+Assuming less than 1/3 of the voting power is Byzantine (malicious):
+
+- Validation of blockchain data structures, including blocks, block parts,
+  votes, and so on
+- Execution of blocks
+- Validator set changes
+- Proposer round robin
+- Two nodes committing conflicting blocks for the same height (safety failure)
+- A correct node signing conflicting votes
+- A node halting (liveness failure)
+- Syncing new and old nodes
+
+## Networking
+
+- Authenticated encryption (MITM, information leakage)
+- Eclipse attacks
+- Sybil attacks
+- Long-range attacks
+- Denial-of-Service
+
+## RPC
+
+- Write-access to anything besides sending transactions
+- Denial-of-Service
+- Leakage of secrets
+
+## Denial-of-Service
+
+Attacks may come through the P2P network or the RPC:
+
+- Amplification attacks
+- Resource abuse
+- Deadlocks and race conditions
+- Panics and unhandled errors
+
+## Libraries
+
+- Serialization (Amino)
+- Reading/Writing files and databases
+- Logging and monitoring
+
+## Cryptography
+
+- Elliptic curves for validator signatures
+- Hash algorithms and Merkle trees for block validation
+- Authenticated encryption for P2P connections
+
+## Light Client
+
+- Validation of blockchain data structures
+- Correctly validating an incorrect proof
+- Incorrectly validating a correct proof
+- Syncing validator set changes
+
+

Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
 
   config.vm.provider "virtualbox" do |v|
     v.memory = 4096
@@ -11,28 +11,48 @@ Vagrant.configure("2") do |config|
 
   config.vm.provision "shell", inline: <<-SHELL
     apt-get update
-    apt-get install -y --no-install-recommends wget curl jq shellcheck bsdmainutils psmisc
 
-    wget -qO- https://get.docker.com/ | sh
+    # install base requirements
+    apt-get install -y --no-install-recommends wget curl jq zip \
+        make shellcheck bsdmainutils psmisc
+    apt-get install -y language-pack-en
+
+    # install docker
+    apt-get install -y --no-install-recommends apt-transport-https \
+      ca-certificates curl software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    add-apt-repository \
+      "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+      $(lsb_release -cs) \
+      stable"
+    apt-get install -y docker-ce
     usermod -a -G docker vagrant
+
+    # install go
+    wget -q https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz
+    tar -xvf go1.10.1.linux-amd64.tar.gz
+    mv go /usr/local
+    rm -f go1.10.1.linux-amd64.tar.gz
+
+    # cleanup
     apt-get autoremove -y
 
-    curl -O https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz
-    tar -xvf go1.9.linux-amd64.tar.gz
-    rm -rf /usr/local/go
-    mv go /usr/local
-    rm -f go1.9.linux-amd64.tar.gz
-    mkdir -p /home/vagrant/go/bin
-    echo 'export PATH=$PATH:/usr/local/go/bin:/home/vagrant/go/bin' >> /home/vagrant/.bash_profile
+    # set env variables
+    echo 'export GOROOT=/usr/local/go' >> /home/vagrant/.bash_profile
     echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.bash_profile
-
+    echo 'export PATH=$PATH:$GOROOT/bin:$GOPATH/bin' >> /home/vagrant/.bash_profile
     echo 'export LC_ALL=en_US.UTF-8' >> /home/vagrant/.bash_profile
+    echo 'cd go/src/github.com/tendermint/tendermint' >> /home/vagrant/.bash_profile
 
+    mkdir -p /home/vagrant/go/bin
     mkdir -p /home/vagrant/go/src/github.com/tendermint
     ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint
 
     chown -R vagrant:vagrant /home/vagrant/go
+    chown vagrant:vagrant /home/vagrant/.bash_profile
 
-    su - vagrant -c 'cd /home/vagrant/go/src/github.com/tendermint/tendermint && make get_vendor_deps'
+    # get all deps and tools, ready to install/test
+    su - vagrant  -c 'source /home/vagrant/.bash_profile'
+    su - vagrant -c 'cd /home/vagrant/go/src/github.com/tendermint/tendermint && make get_tools && make get_vendor_deps'
   SHELL
 end

appveyor.yml

@@ -0,0 +1,13 @@
+version: 1.0.{build}
+configuration: Release
+platform:
+- x64
+- x86
+clone_folder: c:\go\path\src\github.com\tendermint\tendermint
+before_build:
+- cmd: set GOPATH=%GOROOT%\path
+- cmd: set PATH=%GOPATH%\bin;%PATH%
+- cmd: make get_vendor_deps
+build_script:
+- cmd: make test
+test: off

benchmarks/blockchain/.gitignore

@@ -0,0 +1,2 @@
+data
+

benchmarks/blockchain/localsync.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+DATA=$GOPATH/src/github.com/tendermint/tendermint/benchmarks/blockchain/data
+if [ ! -d $DATA ]; then
+  echo "no data found, generating a chain... (this only has to happen once)"
+
+  tendermint init --home $DATA
+  cp $DATA/config.toml $DATA/config2.toml
+  echo "
+  [consensus]
+  timeout_commit = 0
+  " >> $DATA/config.toml
+
+  echo "starting node"
+  tendermint node \
+    --home $DATA \
+    --proxy_app kvstore \
+    --p2p.laddr tcp://127.0.0.1:56656 \
+    --rpc.laddr tcp://127.0.0.1:56657 \
+    --log_level error &
+
+  echo "making blocks for 60s"
+  sleep 60
+
+  mv $DATA/config2.toml $DATA/config.toml
+
+  kill %1
+
+  echo "done generating chain."
+fi
+
+# validator node
+HOME1=$TMPDIR$RANDOM$RANDOM
+cp -R $DATA $HOME1
+echo "starting validator node"
+tendermint node \
+  --home $HOME1 \
+  --proxy_app kvstore \
+  --p2p.laddr tcp://127.0.0.1:56656 \
+  --rpc.laddr tcp://127.0.0.1:56657 \
+  --log_level error &
+sleep 1
+
+# downloader node
+HOME2=$TMPDIR$RANDOM$RANDOM
+tendermint init --home $HOME2
+cp $HOME1/genesis.json $HOME2
+printf "starting downloader node"
+tendermint node \
+  --home $HOME2 \
+  --proxy_app kvstore \
+  --p2p.laddr tcp://127.0.0.1:56666 \
+  --rpc.laddr tcp://127.0.0.1:56667 \
+  --p2p.persistent_peers 127.0.0.1:56656 \
+  --log_level error &
+
+# wait for node to start up so we only count time where we are actually syncing
+sleep 0.5
+while curl localhost:56667/status 2> /dev/null | grep "\"latest_block_height\": 0," > /dev/null
+do
+  printf '.'
+  sleep 0.2
+done
+echo
+
+echo "syncing blockchain for 10s"
+for i in {1..10}
+do
+  sleep 1
+  HEIGHT="$(curl localhost:56667/status 2> /dev/null \
+    | grep 'latest_block_height' \
+    | grep -o ' [0-9]*' \
+    | xargs)"
+  let 'RATE = HEIGHT / i'
+  echo "height: $HEIGHT, blocks/sec: $RATE"
+done
+
+kill %1
+kill %2
+rm -rf $HOME1 $HOME2

benchmarks/codec_test.go

@@ -2,37 +2,47 @@ package benchmarks
 
 import (
 	"testing"
+	"time"
 
+	"github.com/tendermint/go-amino"
 	"github.com/tendermint/go-crypto"
-	"github.com/tendermint/tendermint/p2p"
-	"github.com/tendermint/go-wire"
+
 	proto "github.com/tendermint/tendermint/benchmarks/proto"
+	"github.com/tendermint/tendermint/p2p"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 )
 
 func BenchmarkEncodeStatusWire(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey()
+	cdc := amino.NewCodec()
+	ctypes.RegisterAmino(cdc)
+	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
 	status := &ctypes.ResultStatus{
-		NodeInfo: &p2p.NodeInfo{
-			PubKey:     pubKey.Unwrap().(crypto.PubKeyEd25519),
+		NodeInfo: p2p.NodeInfo{
+			ID:         nodeKey.ID(),
 			Moniker:    "SOMENAME",
 			Network:    "SOMENAME",
-			RemoteAddr: "SOMEADDR",
 			ListenAddr: "SOMEADDR",
 			Version:    "SOMEVER",
 			Other:      []string{"SOMESTRING", "OTHERSTRING"},
 		},
-		PubKey:            pubKey,
-		LatestBlockHash:   []byte("SOMEBYTES"),
-		LatestBlockHeight: 123,
-		LatestBlockTime:   1234,
+		SyncInfo: ctypes.SyncInfo{
+			LatestBlockHash:   []byte("SOMEBYTES"),
+			LatestBlockHeight: 123,
+			LatestBlockTime:   time.Unix(0, 1234),
+		},
+		ValidatorInfo: ctypes.ValidatorInfo{
+			PubKey: nodeKey.PubKey(),
+		},
 	}
 	b.StartTimer()
 
 	counter := 0
 	for i := 0; i < b.N; i++ {
-		jsonBytes := wire.JSONBytes(status)
+		jsonBytes, err := cdc.MarshalJSON(status)
+		if err != nil {
+			panic(err)
+		}
 		counter += len(jsonBytes)
 	}
 
@@ -40,12 +50,13 @@ func BenchmarkEncodeStatusWire(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoWire(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519)
-	nodeInfo := &p2p.NodeInfo{
-		PubKey:     pubKey,
+	cdc := amino.NewCodec()
+	ctypes.RegisterAmino(cdc)
+	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeInfo := p2p.NodeInfo{
+		ID:         nodeKey.ID(),
 		Moniker:    "SOMENAME",
 		Network:    "SOMENAME",
-		RemoteAddr: "SOMEADDR",
 		ListenAddr: "SOMEADDR",
 		Version:    "SOMEVER",
 		Other:      []string{"SOMESTRING", "OTHERSTRING"},
@@ -54,19 +65,23 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) {
 
 	counter := 0
 	for i := 0; i < b.N; i++ {
-		jsonBytes := wire.JSONBytes(nodeInfo)
+		jsonBytes, err := cdc.MarshalJSON(nodeInfo)
+		if err != nil {
+			panic(err)
+		}
 		counter += len(jsonBytes)
 	}
 }
 
 func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519)
-	nodeInfo := &p2p.NodeInfo{
-		PubKey:     pubKey,
+	cdc := amino.NewCodec()
+	ctypes.RegisterAmino(cdc)
+	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeInfo := p2p.NodeInfo{
+		ID:         nodeKey.ID(),
 		Moniker:    "SOMENAME",
 		Network:    "SOMENAME",
-		RemoteAddr: "SOMEADDR",
 		ListenAddr: "SOMEADDR",
 		Version:    "SOMEVER",
 		Other:      []string{"SOMESTRING", "OTHERSTRING"},
@@ -75,7 +90,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 
 	counter := 0
 	for i := 0; i < b.N; i++ {
-		jsonBytes := wire.BinaryBytes(nodeInfo)
+		jsonBytes := cdc.MustMarshalBinaryBare(nodeInfo)
 		counter += len(jsonBytes)
 	}
 
@@ -83,16 +98,20 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoProto(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519)
-	pubKey2 := &proto.PubKey{Ed25519: &proto.PubKeyEd25519{Bytes: pubKey[:]}}
-	nodeInfo := &proto.NodeInfo{
-		PubKey:     pubKey2,
-		Moniker:    "SOMENAME",
-		Network:    "SOMENAME",
-		RemoteAddr: "SOMEADDR",
-		ListenAddr: "SOMEADDR",
-		Version:    "SOMEVER",
-		Other:      []string{"SOMESTRING", "OTHERSTRING"},
+	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeID := string(nodeKey.ID())
+	someName := "SOMENAME"
+	someAddr := "SOMEADDR"
+	someVer := "SOMEVER"
+	someString := "SOMESTRING"
+	otherString := "OTHERSTRING"
+	nodeInfo := proto.NodeInfo{
+		Id:         &proto.ID{Id: &nodeID},
+		Moniker:    &someName,
+		Network:    &someName,
+		ListenAddr: &someAddr,
+		Version:    &someVer,
+		Other:      []string{someString, otherString},
 	}
 	b.StartTimer()
 

benchmarks/os_test.go

@@ -18,12 +18,16 @@ func BenchmarkFileWrite(b *testing.B) {
 	b.StartTimer()
 
 	for i := 0; i < b.N; i++ {
-		file.Write([]byte(testString))
+		_, err := file.Write([]byte(testString))
+		if err != nil {
+			b.Error(err)
+		}
 	}
 
-	file.Close()
-	err = os.Remove("benchmark_file_write.out")
-	if err != nil {
+	if err := file.Close(); err != nil {
+		b.Error(err)
+	}
+	if err := os.Remove("benchmark_file_write.out"); err != nil {
 		b.Error(err)
 	}
 }

benchmarks/proto/test.proto

@@ -7,7 +7,7 @@ message ResultStatus {
 }
 
 message NodeInfo {
-  required PubKey pubKey = 1;
+  required ID id = 1;
   required string moniker = 2;
   required string network = 3;
   required string remoteAddr = 4;
@@ -16,6 +16,10 @@ message NodeInfo {
   repeated string other = 7;
 }
 
+message ID {
+  required string id = 1;
+}
+
 message PubKey {
   optional PubKeyEd25519 ed25519 = 1;
 }

benchmarks/simu/counter.go

@@ -11,8 +11,8 @@ import (
 )
 
 func main() {
-	wsc := rpcclient.NewWSClient("127.0.0.1:46657", "/websocket")
-	_, err := wsc.Start()
+	wsc := rpcclient.NewWSClient("127.0.0.1:26657", "/websocket")
+	err := wsc.Start()
 	if err != nil {
 		cmn.Exit(err.Error())
 	}
@@ -21,7 +21,7 @@ func main() {
 	// Read a bunch of responses
 	go func() {
 		for {
-			_, ok := <-wsc.ResultsCh
+			_, ok := <-wsc.ResponsesCh
 			if !ok {
 				break
 			}

blockchain/pool.go

@@ -1,25 +1,52 @@
 package blockchain
 
 import (
+	"errors"
+	"fmt"
 	"math"
 	"sync"
+	"sync/atomic"
 	"time"
 
-	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 	flow "github.com/tendermint/tmlibs/flowrate"
 	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
 )
 
+/*
+eg, L = latency = 0.1s
+	P = num peers = 10
+	FN = num full nodes
+	BS = 1kB block size
+	CB = 1 Mbit/s = 128 kB/s
+	CB/P = 12.8 kB
+	B/S = CB/P/BS = 12.8 blocks/s
+
+	12.8 * 0.1 = 1.28 blocks on conn
+*/
+
 const (
-	requestIntervalMS         = 250
-	maxTotalRequesters        = 300
+	requestIntervalMS         = 100
+	maxTotalRequesters        = 1000
 	maxPendingRequests        = maxTotalRequesters
-	maxPendingRequestsPerPeer = 75
-	minRecvRate               = 10240 // 10Kb/s
+	maxPendingRequestsPerPeer = 50
+
+	// Minimum recv rate to ensure we're receiving blocks from a peer fast
+	// enough. If a peer is not sending us data at at least that rate, we
+	// consider them to have timedout and we disconnect.
+	//
+	// Assuming a DSL connection (not a good choice) 128 Kbps (upload) ~ 15 KB/s,
+	// sending data across atlantic ~ 7.5 KB/s.
+	minRecvRate = 7680
+
+	// Maximum difference between current and new block's height.
+	maxDiffBetweenCurrentAndReceivedBlockHeight = 100
 )
 
-var peerTimeoutSeconds = time.Duration(15) // not const so we can override with tests
+var peerTimeout = 15 * time.Second // not const so we can override with tests
 
 /*
 	Peers self report their heights when we join the block pool.
@@ -38,26 +65,29 @@ type BlockPool struct {
 
 	mtx sync.Mutex
 	// block requests
-	requesters map[int]*bpRequester
-	height     int   // the lowest key in requesters.
-	numPending int32 // number of requests pending assignment or block response
+	requesters map[int64]*bpRequester
+	height     int64 // the lowest key in requesters.
 	// peers
-	peers map[string]*bpPeer
+	peers         map[p2p.ID]*bpPeer
+	maxPeerHeight int64
+
+	// atomic
+	numPending int32 // number of requests pending assignment or block response
 
 	requestsCh chan<- BlockRequest
-	timeoutsCh chan<- string
+	errorsCh   chan<- peerError
 }
 
-func NewBlockPool(start int, requestsCh chan<- BlockRequest, timeoutsCh chan<- string) *BlockPool {
+func NewBlockPool(start int64, requestsCh chan<- BlockRequest, errorsCh chan<- peerError) *BlockPool {
 	bp := &BlockPool{
-		peers: make(map[string]*bpPeer),
+		peers: make(map[p2p.ID]*bpPeer),
 
-		requesters: make(map[int]*bpRequester),
+		requesters: make(map[int64]*bpRequester),
 		height:     start,
 		numPending: 0,
 
 		requestsCh: requestsCh,
-		timeoutsCh: timeoutsCh,
+		errorsCh:   errorsCh,
 	}
 	bp.BaseService = *cmn.NewBaseService(nil, "BlockPool", bp)
 	return bp
@@ -69,9 +99,7 @@ func (pool *BlockPool) OnStart() error {
 	return nil
 }
 
-func (pool *BlockPool) OnStop() {
-	pool.BaseService.OnStop()
-}
+func (pool *BlockPool) OnStop() {}
 
 // Run spawns requesters as needed.
 func (pool *BlockPool) makeRequestersRoutine() {
@@ -79,6 +107,7 @@ func (pool *BlockPool) makeRequestersRoutine() {
 		if !pool.IsRunning() {
 			break
 		}
+
 		_, numPending, lenRequesters := pool.GetStatus()
 		if numPending >= maxPendingRequests {
 			// sleep for a bit.
@@ -104,10 +133,14 @@ func (pool *BlockPool) removeTimedoutPeers() {
 	for _, peer := range pool.peers {
 		if !peer.didTimeout && peer.numPending > 0 {
 			curRate := peer.recvMonitor.Status().CurRate
-			// XXX remove curRate != 0
+			// curRate can be 0 on start
 			if curRate != 0 && curRate < minRecvRate {
-				pool.sendTimeout(peer.id)
-				pool.Logger.Error("SendTimeout", "peer", peer.id, "reason", "curRate too low")
+				err := errors.New("peer is not sending us data fast enough")
+				pool.sendError(err, peer.id)
+				pool.Logger.Error("SendTimeout", "peer", peer.id,
+					"reason", err,
+					"curRate", fmt.Sprintf("%d KB/s", curRate/1024),
+					"minRate", fmt.Sprintf("%d KB/s", minRecvRate/1024))
 				peer.didTimeout = true
 			}
 		}
@@ -117,11 +150,11 @@ func (pool *BlockPool) removeTimedoutPeers() {
 	}
 }
 
-func (pool *BlockPool) GetStatus() (height int, numPending int32, lenRequesters int) {
+func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequesters int) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
-	return pool.height, pool.numPending, len(pool.requesters)
+	return pool.height, atomic.LoadInt32(&pool.numPending), len(pool.requesters)
 }
 
 // TODO: relax conditions, prevent abuse.
@@ -135,16 +168,10 @@ func (pool *BlockPool) IsCaughtUp() bool {
 		return false
 	}
 
-	maxPeerHeight := 0
-	for _, peer := range pool.peers {
-		maxPeerHeight = cmn.MaxInt(maxPeerHeight, peer.height)
-	}
-
 	// some conditions to determine if we're caught up
 	receivedBlockOrTimedOut := (pool.height > 0 || time.Since(pool.startTime) > 5*time.Second)
-	ourChainIsLongestAmongPeers := maxPeerHeight == 0 || pool.height >= maxPeerHeight
+	ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= pool.maxPeerHeight
 	isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers
-	pool.Logger.Info(cmn.Fmt("IsCaughtUp: %v", isCaughtUp), "height", pool.height, "maxPeerHeight", maxPeerHeight)
 	return isCaughtUp
 }
 
@@ -180,46 +207,66 @@ func (pool *BlockPool) PopRequest() {
 		delete(pool.requesters, pool.height)
 		pool.height++
 	} else {
-		cmn.PanicSanity(cmn.Fmt("Expected requester to pop, got nothing at height %v", pool.height))
+		panic(fmt.Sprintf("Expected requester to pop, got nothing at height %v", pool.height))
 	}
 }
 
 // Invalidates the block at pool.height,
 // Remove the peer and redo request from others.
-func (pool *BlockPool) RedoRequest(height int) {
+// Returns the ID of the removed peer.
+func (pool *BlockPool) RedoRequest(height int64) p2p.ID {
 	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
 	request := pool.requesters[height]
-	pool.mtx.Unlock()
 
 	if request.block == nil {
-		cmn.PanicSanity("Expected block to be non-nil")
+		panic("Expected block to be non-nil")
 	}
+
 	// RemovePeer will redo all requesters associated with this peer.
-	// TODO: record this malfeasance
-	pool.RemovePeer(request.peerID)
+	pool.removePeer(request.peerID)
+	return request.peerID
 }
 
 // TODO: ensure that blocks come in order for each peer.
-func (pool *BlockPool) AddBlock(peerID string, block *types.Block, blockSize int) {
+func (pool *BlockPool) AddBlock(peerID p2p.ID, block *types.Block, blockSize int) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
 	requester := pool.requesters[block.Height]
 	if requester == nil {
+		pool.Logger.Info("peer sent us a block we didn't expect", "peer", peerID, "curHeight", pool.height, "blockHeight", block.Height)
+		diff := pool.height - block.Height
+		if diff < 0 {
+			diff *= -1
+		}
+		if diff > maxDiffBetweenCurrentAndReceivedBlockHeight {
+			pool.sendError(errors.New("peer sent us a block we didn't expect with a height too far ahead/behind"), peerID)
+		}
 		return
 	}
 
 	if requester.setBlock(block, peerID) {
-		pool.numPending--
+		atomic.AddInt32(&pool.numPending, -1)
 		peer := pool.peers[peerID]
-		peer.decrPending(blockSize)
+		if peer != nil {
+			peer.decrPending(blockSize)
+		}
 	} else {
 		// Bad peer?
 	}
 }
 
+// MaxPeerHeight returns the highest height reported by a peer.
+func (pool *BlockPool) MaxPeerHeight() int64 {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+	return pool.maxPeerHeight
+}
+
 // Sets the peer's alleged blockchain height.
-func (pool *BlockPool) SetPeerHeight(peerID string, height int) {
+func (pool *BlockPool) SetPeerHeight(peerID p2p.ID, height int64) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
@@ -231,22 +278,23 @@ func (pool *BlockPool) SetPeerHeight(peerID string, height int) {
 		peer.setLogger(pool.Logger.With("peer", peerID))
 		pool.peers[peerID] = peer
 	}
+
+	if height > pool.maxPeerHeight {
+		pool.maxPeerHeight = height
+	}
 }
 
-func (pool *BlockPool) RemovePeer(peerID string) {
+func (pool *BlockPool) RemovePeer(peerID p2p.ID) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
 	pool.removePeer(peerID)
 }
 
-func (pool *BlockPool) removePeer(peerID string) {
+func (pool *BlockPool) removePeer(peerID p2p.ID) {
 	for _, requester := range pool.requesters {
 		if requester.getPeerID() == peerID {
-			if requester.getBlock() != nil {
-				pool.numPending++
-			}
-			go requester.redo() // pick another peer and ...
+			requester.redo()
 		}
 	}
 	delete(pool.peers, peerID)
@@ -254,7 +302,7 @@ func (pool *BlockPool) removePeer(peerID string) {
 
 // Pick an available peer with at least the given minHeight.
 // If no peers are available, returns nil.
-func (pool *BlockPool) pickIncrAvailablePeer(minHeight int) *bpPeer {
+func (pool *BlockPool) pickIncrAvailablePeer(minHeight int64) *bpPeer {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
@@ -279,37 +327,45 @@ func (pool *BlockPool) makeNextRequester() {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
-	nextHeight := pool.height + len(pool.requesters)
+	nextHeight := pool.height + pool.requestersLen()
 	request := newBPRequester(pool, nextHeight)
-	request.SetLogger(pool.Logger.With("height", nextHeight))
+	// request.SetLogger(pool.Logger.With("height", nextHeight))
 
 	pool.requesters[nextHeight] = request
-	pool.numPending++
+	atomic.AddInt32(&pool.numPending, 1)
 
-	request.Start()
+	err := request.Start()
+	if err != nil {
+		request.Logger.Error("Error starting request", "err", err)
+	}
 }
 
-func (pool *BlockPool) sendRequest(height int, peerID string) {
+func (pool *BlockPool) requestersLen() int64 {
+	return int64(len(pool.requesters))
+}
+
+func (pool *BlockPool) sendRequest(height int64, peerID p2p.ID) {
 	if !pool.IsRunning() {
 		return
 	}
 	pool.requestsCh <- BlockRequest{height, peerID}
 }
 
-func (pool *BlockPool) sendTimeout(peerID string) {
+func (pool *BlockPool) sendError(err error, peerID p2p.ID) {
 	if !pool.IsRunning() {
 		return
 	}
-	pool.timeoutsCh <- peerID
+	pool.errorsCh <- peerError{err, peerID}
 }
 
 // unused by tendermint; left for debugging purposes
 func (pool *BlockPool) debug() string {
-	pool.mtx.Lock() // Lock
+	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
 	str := ""
-	for h := pool.height; h < pool.height+len(pool.requesters); h++ {
+	nextHeight := pool.height + pool.requestersLen()
+	for h := pool.height; h < nextHeight; h++ {
 		if pool.requesters[h] == nil {
 			str += cmn.Fmt("H(%v):X ", h)
 		} else {
@@ -324,10 +380,10 @@ func (pool *BlockPool) debug() string {
 
 type bpPeer struct {
 	pool        *BlockPool
-	id          string
+	id          p2p.ID
 	recvMonitor *flow.Monitor
 
-	height     int
+	height     int64
 	numPending int32
 	timeout    *time.Timer
 	didTimeout bool
@@ -335,7 +391,7 @@ type bpPeer struct {
 	logger log.Logger
 }
 
-func newBPPeer(pool *BlockPool, peerID string, height int) *bpPeer {
+func newBPPeer(pool *BlockPool, peerID p2p.ID, height int64) *bpPeer {
 	peer := &bpPeer{
 		pool:       pool,
 		id:         peerID,
@@ -358,9 +414,9 @@ func (peer *bpPeer) resetMonitor() {
 
 func (peer *bpPeer) resetTimeout() {
 	if peer.timeout == nil {
-		peer.timeout = time.AfterFunc(time.Second*peerTimeoutSeconds, peer.onTimeout)
+		peer.timeout = time.AfterFunc(peerTimeout, peer.onTimeout)
 	} else {
-		peer.timeout.Reset(time.Second * peerTimeoutSeconds)
+		peer.timeout.Reset(peerTimeout)
 	}
 }
 
@@ -386,8 +442,9 @@ func (peer *bpPeer) onTimeout() {
 	peer.pool.mtx.Lock()
 	defer peer.pool.mtx.Unlock()
 
-	peer.pool.sendTimeout(peer.id)
-	peer.logger.Error("SendTimeout", "reason", "onTimeout")
+	err := errors.New("peer did not send us anything")
+	peer.pool.sendError(err, peer.id)
+	peer.logger.Error("SendTimeout", "reason", err, "timeout", peerTimeout)
 	peer.didTimeout = true
 }
 
@@ -396,21 +453,21 @@ func (peer *bpPeer) onTimeout() {
 type bpRequester struct {
 	cmn.BaseService
 	pool       *BlockPool
-	height     int
+	height     int64
 	gotBlockCh chan struct{}
 	redoCh     chan struct{}
 
 	mtx    sync.Mutex
-	peerID string
+	peerID p2p.ID
 	block  *types.Block
 }
 
-func newBPRequester(pool *BlockPool, height int) *bpRequester {
+func newBPRequester(pool *BlockPool, height int64) *bpRequester {
 	bpr := &bpRequester{
 		pool:       pool,
 		height:     height,
-		gotBlockCh: make(chan struct{}),
-		redoCh:     make(chan struct{}),
+		gotBlockCh: make(chan struct{}, 1),
+		redoCh:     make(chan struct{}, 1),
 
 		peerID: "",
 		block:  nil,
@@ -424,8 +481,8 @@ func (bpr *bpRequester) OnStart() error {
 	return nil
 }
 
-// Returns true if the peer matches
-func (bpr *bpRequester) setBlock(block *types.Block, peerID string) bool {
+// Returns true if the peer matches and block doesn't already exist.
+func (bpr *bpRequester) setBlock(block *types.Block, peerID p2p.ID) bool {
 	bpr.mtx.Lock()
 	if bpr.block != nil || bpr.peerID != peerID {
 		bpr.mtx.Unlock()
@@ -434,7 +491,10 @@ func (bpr *bpRequester) setBlock(block *types.Block, peerID string) bool {
 	bpr.block = block
 	bpr.mtx.Unlock()
 
-	bpr.gotBlockCh <- struct{}{}
+	select {
+	case bpr.gotBlockCh <- struct{}{}:
+	default:
+	}
 	return true
 }
 
@@ -444,23 +504,33 @@ func (bpr *bpRequester) getBlock() *types.Block {
 	return bpr.block
 }
 
-func (bpr *bpRequester) getPeerID() string {
+func (bpr *bpRequester) getPeerID() p2p.ID {
 	bpr.mtx.Lock()
 	defer bpr.mtx.Unlock()
 	return bpr.peerID
 }
 
+// This is called from the requestRoutine, upon redo().
 func (bpr *bpRequester) reset() {
 	bpr.mtx.Lock()
+	defer bpr.mtx.Unlock()
+
+	if bpr.block != nil {
+		atomic.AddInt32(&bpr.pool.numPending, 1)
+	}
+
 	bpr.peerID = ""
 	bpr.block = nil
-	bpr.mtx.Unlock()
 }
 
 // Tells bpRequester to pick another peer and try again.
-// NOTE: blocking
+// NOTE: Nonblocking, and does nothing if another redo
+// was already requested.
 func (bpr *bpRequester) redo() {
-	bpr.redoCh <- struct{}{}
+	select {
+	case bpr.redoCh <- struct{}{}:
+	default:
+	}
 }
 
 // Responsible for making more requests as necessary
@@ -469,7 +539,7 @@ func (bpr *bpRequester) requestRoutine() {
 OUTER_LOOP:
 	for {
 		// Pick a peer to send request to.
-		var peer *bpPeer = nil
+		var peer *bpPeer
 	PICK_PEER_LOOP:
 		for {
 			if !bpr.IsRunning() || !bpr.pool.IsRunning() {
@@ -489,26 +559,21 @@ OUTER_LOOP:
 
 		// Send request and wait.
 		bpr.pool.sendRequest(bpr.height, peer.id)
-		select {
-		case <-bpr.pool.Quit:
-			bpr.Stop()
-			return
-		case <-bpr.Quit:
-			return
-		case <-bpr.redoCh:
-			bpr.reset()
-			continue OUTER_LOOP // When peer is removed
-		case <-bpr.gotBlockCh:
-			// We got the block, now see if it's good.
+	WAIT_LOOP:
+		for {
 			select {
-			case <-bpr.pool.Quit:
+			case <-bpr.pool.Quit():
 				bpr.Stop()
 				return
-			case <-bpr.Quit:
+			case <-bpr.Quit():
 				return
 			case <-bpr.redoCh:
 				bpr.reset()
 				continue OUTER_LOOP
+			case <-bpr.gotBlockCh:
+				// We got a block!
+				// Continue the for-loop and wait til Quit.
+				continue WAIT_LOOP
 			}
 		}
 	}
@@ -517,6 +582,6 @@ OUTER_LOOP:
 //-------------------------------------
 
 type BlockRequest struct {
-	Height int
-	PeerID string
+	Height int64
+	PeerID p2p.ID
 }

blockchain/pool_test.go

@@ -5,38 +5,45 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
 )
 
 func init() {
-	peerTimeoutSeconds = time.Duration(2)
+	peerTimeout = 2 * time.Second
 }
 
 type testPeer struct {
-	id     string
-	height int
+	id     p2p.ID
+	height int64
 }
 
-func makePeers(numPeers int, minHeight, maxHeight int) map[string]testPeer {
-	peers := make(map[string]testPeer, numPeers)
+func makePeers(numPeers int, minHeight, maxHeight int64) map[p2p.ID]testPeer {
+	peers := make(map[p2p.ID]testPeer, numPeers)
 	for i := 0; i < numPeers; i++ {
-		peerID := cmn.RandStr(12)
-		height := minHeight + rand.Intn(maxHeight-minHeight)
+		peerID := p2p.ID(cmn.RandStr(12))
+		height := minHeight + rand.Int63n(maxHeight-minHeight)
 		peers[peerID] = testPeer{peerID, height}
 	}
 	return peers
 }
 
 func TestBasic(t *testing.T) {
-	start := 42
+	start := int64(42)
 	peers := makePeers(10, start+1, 1000)
-	timeoutsCh := make(chan string, 100)
-	requestsCh := make(chan BlockRequest, 100)
-	pool := NewBlockPool(start, requestsCh, timeoutsCh)
+	errorsCh := make(chan peerError, 1000)
+	requestsCh := make(chan BlockRequest, 1000)
+	pool := NewBlockPool(start, requestsCh, errorsCh)
 	pool.SetLogger(log.TestingLogger())
-	pool.Start()
+
+	err := pool.Start()
+	if err != nil {
+		t.Error(err)
+	}
+
 	defer pool.Stop()
 
 	// Introduce each peer.
@@ -64,8 +71,8 @@ func TestBasic(t *testing.T) {
 	// Pull from channels
 	for {
 		select {
-		case peerID := <-timeoutsCh:
-			t.Errorf("timeout: %v", peerID)
+		case err := <-errorsCh:
+			t.Error(err)
 		case request := <-requestsCh:
 			t.Logf("Pulled new BlockRequest %v", request)
 			if request.Height == 300 {
@@ -82,13 +89,16 @@ func TestBasic(t *testing.T) {
 }
 
 func TestTimeout(t *testing.T) {
-	start := 42
+	start := int64(42)
 	peers := makePeers(10, start+1, 1000)
-	timeoutsCh := make(chan string, 100)
-	requestsCh := make(chan BlockRequest, 100)
-	pool := NewBlockPool(start, requestsCh, timeoutsCh)
+	errorsCh := make(chan peerError, 1000)
+	requestsCh := make(chan BlockRequest, 1000)
+	pool := NewBlockPool(start, requestsCh, errorsCh)
 	pool.SetLogger(log.TestingLogger())
-	pool.Start()
+	err := pool.Start()
+	if err != nil {
+		t.Error(err)
+	}
 	defer pool.Stop()
 
 	for _, peer := range peers {
@@ -119,12 +129,13 @@ func TestTimeout(t *testing.T) {
 
 	// Pull from channels
 	counter := 0
-	timedOut := map[string]struct{}{}
+	timedOut := map[p2p.ID]struct{}{}
 	for {
 		select {
-		case peerID := <-timeoutsCh:
-			t.Logf("Peer %v timeouted", peerID)
-			if _, ok := timedOut[peerID]; !ok {
+		case err := <-errorsCh:
+			t.Log(err)
+			// consider error to be always timeout here
+			if _, ok := timedOut[err.peerID]; !ok {
 				counter++
 				if counter == len(peers) {
 					return // Done!

blockchain/reactor.go

@@ -1,25 +1,23 @@
 package blockchain
 
 import (
-	"bytes"
-	"errors"
+	"fmt"
 	"reflect"
 	"time"
 
-	wire "github.com/tendermint/go-wire"
+	"github.com/tendermint/go-amino"
 	"github.com/tendermint/tendermint/p2p"
-	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 )
 
 const (
 	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
 	BlockchainChannel = byte(0x40)
 
-	defaultChannelCapacity = 100
-	trySyncIntervalMS      = 100
+	trySyncIntervalMS = 50
 	// stop syncing when last block's time is
 	// within this much of the system time.
 	// stopSyncingDurationMinutes = 10
@@ -28,62 +26,91 @@ const (
 	statusUpdateIntervalSeconds = 10
 	// check if we should switch to consensus reactor
 	switchToConsensusIntervalSeconds = 1
+
+	// NOTE: keep up to date with bcBlockResponseMessage
+	bcBlockResponseMessagePrefixSize   = 4
+	bcBlockResponseMessageFieldKeySize = 1
+	maxMsgSize                         = types.MaxBlockSizeBytes +
+		bcBlockResponseMessagePrefixSize +
+		bcBlockResponseMessageFieldKeySize
 )
 
 type consensusReactor interface {
 	// for when we switch from blockchain reactor and fast sync to
 	// the consensus machine
-	SwitchToConsensus(*sm.State)
+	SwitchToConsensus(sm.State, int)
+}
+
+type peerError struct {
+	err    error
+	peerID p2p.ID
+}
+
+func (e peerError) Error() string {
+	return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error())
 }
 
 // BlockchainReactor handles long-term catchup syncing.
 type BlockchainReactor struct {
 	p2p.BaseReactor
 
-	state        *sm.State
-	proxyAppConn proxy.AppConnConsensus // same as consensus.proxyAppConn
-	store        *BlockStore
-	pool         *BlockPool
-	fastSync     bool
-	requestsCh   chan BlockRequest
-	timeoutsCh   chan string
+	// immutable
+	initialState sm.State
 
-	evsw types.EventSwitch
+	blockExec *sm.BlockExecutor
+	store     *BlockStore
+	pool      *BlockPool
+	fastSync  bool
+
+	requestsCh <-chan BlockRequest
+	errorsCh   <-chan peerError
 }
 
 // NewBlockchainReactor returns new reactor instance.
-func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor {
-	if state.LastBlockHeight == store.Height()-1 {
-		store.height-- // XXX HACK, make this better
-	}
+func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *BlockStore,
+	fastSync bool) *BlockchainReactor {
+
 	if state.LastBlockHeight != store.Height() {
-		cmn.PanicSanity(cmn.Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
+		panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight,
+			store.Height()))
 	}
-	requestsCh := make(chan BlockRequest, defaultChannelCapacity)
-	timeoutsCh := make(chan string, defaultChannelCapacity)
+
+	const capacity = 1000 // must be bigger than peers count
+	requestsCh := make(chan BlockRequest, capacity)
+	errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock
+
 	pool := NewBlockPool(
 		store.Height()+1,
 		requestsCh,
-		timeoutsCh,
+		errorsCh,
 	)
+
 	bcR := &BlockchainReactor{
-		state:        state,
-		proxyAppConn: proxyAppConn,
+		initialState: state,
+		blockExec:    blockExec,
 		store:        store,
 		pool:         pool,
 		fastSync:     fastSync,
 		requestsCh:   requestsCh,
-		timeoutsCh:   timeoutsCh,
+		errorsCh:     errorsCh,
 	}
 	bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
 	return bcR
 }
 
-// OnStart implements BaseService
+// SetLogger implements cmn.Service by setting the logger on reactor and pool.
+func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
+	bcR.BaseService.Logger = l
+	bcR.pool.Logger = l
+}
+
+// OnStart implements cmn.Service.
 func (bcR *BlockchainReactor) OnStart() error {
-	bcR.BaseReactor.OnStart()
+	if err := bcR.BaseReactor.OnStart(); err != nil {
+		return err
+	}
 	if bcR.fastSync {
-		_, err := bcR.pool.Start()
+		err := bcR.pool.Start()
 		if err != nil {
 			return err
 		}
@@ -92,7 +119,7 @@ func (bcR *BlockchainReactor) OnStart() error {
 	return nil
 }
 
-// OnStop implements BaseService
+// OnStop implements cmn.Service.
 func (bcR *BlockchainReactor) OnStop() {
 	bcR.BaseReactor.OnStop()
 	bcR.pool.Stop()
@@ -101,55 +128,61 @@ func (bcR *BlockchainReactor) OnStop() {
 // GetChannels implements Reactor
 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	return []*p2p.ChannelDescriptor{
-		&p2p.ChannelDescriptor{
-			ID:                BlockchainChannel,
-			Priority:          5,
-			SendQueueCapacity: 100,
+		{
+			ID:                  BlockchainChannel,
+			Priority:            10,
+			SendQueueCapacity:   1000,
+			RecvBufferCapacity:  50 * 4096,
+			RecvMessageCapacity: maxMsgSize,
 		},
 	}
 }
 
 // AddPeer implements Reactor by sending our state to peer.
 func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) {
-	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
+	msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()})
+	if !peer.Send(BlockchainChannel, msgBytes) {
 		// doing nothing, will try later in `poolRoutine`
 	}
+	// peer is added to the pool once we receive the first
+	// bcStatusResponseMessage from the peer and call pool.SetPeerHeight
 }
 
 // RemovePeer implements Reactor by removing peer from the pool.
 func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
-	bcR.pool.RemovePeer(peer.Key())
+	bcR.pool.RemovePeer(peer.ID())
 }
 
 // respondToPeer loads a block and sends it to the requesting peer,
 // if we have it. Otherwise, we'll respond saying we don't have it.
 // According to the Tendermint spec, if all nodes are honest,
 // no node should be requesting for a block that's non-existent.
-func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, src p2p.Peer) (queued bool) {
+func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage,
+	src p2p.Peer) (queued bool) {
+
 	block := bcR.store.LoadBlock(msg.Height)
 	if block != nil {
-		msg := &bcBlockResponseMessage{Block: block}
-		return src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
+		msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block})
+		return src.TrySend(BlockchainChannel, msgBytes)
 	}
 
 	bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height)
 
-	return src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{
-		&bcNoBlockResponseMessage{Height: msg.Height},
-	})
+	msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height})
+	return src.TrySend(BlockchainChannel, msgBytes)
 }
 
 // Receive implements Reactor by handling 4 types of messages (look below).
 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
-	_, msg, err := DecodeMessage(msgBytes, bcR.maxMsgSize())
+	msg, err := DecodeMessage(msgBytes)
 	if err != nil {
-		bcR.Logger.Error("Error decoding message", "err", err)
+		bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
+		bcR.Switch.StopPeerForError(src, err)
 		return
 	}
 
 	bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg)
 
-	// TODO: improve logic to satisfy megacheck
 	switch msg := msg.(type) {
 	case *bcBlockRequestMessage:
 		if queued := bcR.respondToPeer(msg, src); !queued {
@@ -157,27 +190,22 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 		}
 	case *bcBlockResponseMessage:
 		// Got a block.
-		bcR.pool.AddBlock(src.Key(), msg.Block, len(msgBytes))
+		bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes))
 	case *bcStatusRequestMessage:
 		// Send peer our state.
-		queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
+		msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()})
+		queued := src.TrySend(BlockchainChannel, msgBytes)
 		if !queued {
 			// sorry
 		}
 	case *bcStatusResponseMessage:
 		// Got a peer status. Unverified.
-		bcR.pool.SetPeerHeight(src.Key(), msg.Height)
+		bcR.pool.SetPeerHeight(src.ID(), msg.Height)
 	default:
 		bcR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 	}
 }
 
-// maxMsgSize returns the maximum allowable size of a
-// message on the blockchain reactor.
-func (bcR *BlockchainReactor) maxMsgSize() int {
-	return bcR.state.Params().BlockSizeParams.MaxBytes + 2
-}
-
 // Handle messages from the poolReactor telling the reactor what to do.
 // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
 // (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.)
@@ -187,41 +215,48 @@ func (bcR *BlockchainReactor) poolRoutine() {
 	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
 	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
 
+	blocksSynced := 0
+
+	chainID := bcR.initialState.ChainID
+	state := bcR.initialState
+
+	lastHundred := time.Now()
+	lastRate := 0.0
+
 FOR_LOOP:
 	for {
 		select {
-		case request := <-bcR.requestsCh: // chan BlockRequest
+		case request := <-bcR.requestsCh:
 			peer := bcR.Switch.Peers().Get(request.PeerID)
 			if peer == nil {
 				continue FOR_LOOP // Peer has since been disconnected.
 			}
-			msg := &bcBlockRequestMessage{request.Height}
-			queued := peer.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
+			msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height})
+			queued := peer.TrySend(BlockchainChannel, msgBytes)
 			if !queued {
 				// We couldn't make the request, send-queue full.
 				// The pool handles timeouts, just let it go.
 				continue FOR_LOOP
 			}
-		case peerID := <-bcR.timeoutsCh: // chan string
-			// Peer timed out.
-			peer := bcR.Switch.Peers().Get(peerID)
+		case err := <-bcR.errorsCh:
+			peer := bcR.Switch.Peers().Get(err.peerID)
 			if peer != nil {
-				bcR.Switch.StopPeerForError(peer, errors.New("BlockchainReactor Timeout"))
+				bcR.Switch.StopPeerForError(peer, err)
 			}
 		case <-statusUpdateTicker.C:
 			// ask for status updates
-			go bcR.BroadcastStatusRequest()
+			go bcR.BroadcastStatusRequest() // nolint: errcheck
 		case <-switchToConsensusTicker.C:
-			height, numPending, _ := bcR.pool.GetStatus()
+			height, numPending, lenRequesters := bcR.pool.GetStatus()
 			outbound, inbound, _ := bcR.Switch.NumPeers()
-			bcR.Logger.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
+			bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters,
 				"outbound", outbound, "inbound", inbound)
 			if bcR.pool.IsCaughtUp() {
 				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
 				bcR.pool.Stop()
 
 				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
-				conR.SwitchToConsensus(bcR.state)
+				conR.SwitchToConsensus(state, blocksSynced)
 
 				break FOR_LOOP
 			}
@@ -236,36 +271,50 @@ FOR_LOOP:
 					// We need both to sync the first block.
 					break SYNC_LOOP
 				}
-				firstParts := first.MakePartSet(bcR.state.Params().BlockPartSizeBytes)
+				firstParts := first.MakePartSet(state.ConsensusParams.BlockPartSizeBytes)
 				firstPartsHeader := firstParts.Header()
+				firstID := types.BlockID{first.Hash(), firstPartsHeader}
 				// Finally, verify the first block using the second's commit
 				// NOTE: we can probably make this more efficient, but note that calling
 				// first.Hash() doesn't verify the tx contents, so MakePartSet() is
 				// currently necessary.
-				err := bcR.state.Validators.VerifyCommit(
-					bcR.state.ChainID, types.BlockID{first.Hash(), firstPartsHeader}, first.Height, second.LastCommit)
+				err := state.Validators.VerifyCommit(
+					chainID, firstID, first.Height, second.LastCommit)
 				if err != nil {
 					bcR.Logger.Error("Error in validation", "err", err)
-					bcR.pool.RedoRequest(first.Height)
+					peerID := bcR.pool.RedoRequest(first.Height)
+					peer := bcR.Switch.Peers().Get(peerID)
+					if peer != nil {
+						bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err))
+					}
 					break SYNC_LOOP
 				} else {
 					bcR.pool.PopRequest()
 
+					// TODO: batch saves so we dont persist to disk every block
 					bcR.store.SaveBlock(first, firstParts, second.LastCommit)
 
-					// TODO: should we be firing events? need to fire NewBlock events manually ...
-					// NOTE: we could improve performance if we
-					// didn't make the app commit to disk every block
-					// ... but we would need a way to get the hash without it persisting
-					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{})
+					// TODO: same thing for app - but we would need a way to
+					// get the hash without persisting the state
+					var err error
+					state, err = bcR.blockExec.ApplyBlock(state, firstID, first)
 					if err != nil {
 						// TODO This is bad, are we zombie?
-						cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+						cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v",
+							first.Height, first.Hash(), err))
+					}
+					blocksSynced++
+
+					if blocksSynced%100 == 0 {
+						lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+						bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
+							"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
+						lastHundred = time.Now()
 					}
 				}
 			}
 			continue FOR_LOOP
-		case <-bcR.Quit:
+		case <-bcR.Quit():
 			break FOR_LOOP
 		}
 	}
@@ -273,47 +322,36 @@ FOR_LOOP:
 
 // BroadcastStatusRequest broadcasts `BlockStore` height.
 func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
-	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusRequestMessage{bcR.store.Height()}})
+	msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()})
+	bcR.Switch.Broadcast(BlockchainChannel, msgBytes)
 	return nil
 }
 
-// SetEventSwitch implements events.Eventable
-func (bcR *BlockchainReactor) SetEventSwitch(evsw types.EventSwitch) {
-	bcR.evsw = evsw
-}
-
 //-----------------------------------------------------------------------------
 // Messages
 
-const (
-	msgTypeBlockRequest    = byte(0x10)
-	msgTypeBlockResponse   = byte(0x11)
-	msgTypeNoBlockResponse = byte(0x12)
-	msgTypeStatusResponse  = byte(0x20)
-	msgTypeStatusRequest   = byte(0x21)
-)
-
 // BlockchainMessage is a generic message for this reactor.
 type BlockchainMessage interface{}
 
-var _ = wire.RegisterInterface(
-	struct{ BlockchainMessage }{},
-	wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest},
-	wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse},
-	wire.ConcreteType{&bcNoBlockResponseMessage{}, msgTypeNoBlockResponse},
-	wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse},
-	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
-)
+func RegisterBlockchainMessages(cdc *amino.Codec) {
+	cdc.RegisterInterface((*BlockchainMessage)(nil), nil)
+	cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/mempool/BlockRequest", nil)
+	cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/mempool/BlockResponse", nil)
+	cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/mempool/NoBlockResponse", nil)
+	cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/mempool/StatusResponse", nil)
+	cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/mempool/StatusRequest", nil)
+}
 
 // DecodeMessage decodes BlockchainMessage.
 // TODO: ensure that bz is completely read.
-func DecodeMessage(bz []byte, maxSize int) (msgType byte, msg BlockchainMessage, err error) {
-	msgType = bz[0]
-	n := int(0)
-	r := bytes.NewReader(bz)
-	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
-	if err != nil && n != len(bz) {
-		err = errors.New("DecodeMessage() had bytes left over")
+func DecodeMessage(bz []byte) (msg BlockchainMessage, err error) {
+	if len(bz) > maxMsgSize {
+		return msg, fmt.Errorf("Msg exceeds max size (%d > %d)",
+			len(bz), maxMsgSize)
+	}
+	err = cdc.UnmarshalBinaryBare(bz, &msg)
+	if err != nil {
+		err = cmn.ErrorWrap(err, "DecodeMessage() had bytes left over")
 	}
 	return
 }
@@ -321,7 +359,7 @@ func DecodeMessage(bz []byte, maxSize int) (msgType byte, msg BlockchainMessage,
 //-------------------------------------
 
 type bcBlockRequestMessage struct {
-	Height int
+	Height int64
 }
 
 func (m *bcBlockRequestMessage) String() string {
@@ -329,7 +367,7 @@ func (m *bcBlockRequestMessage) String() string {
 }
 
 type bcNoBlockResponseMessage struct {
-	Height int
+	Height int64
 }
 
 func (brm *bcNoBlockResponseMessage) String() string {
@@ -338,7 +376,6 @@ func (brm *bcNoBlockResponseMessage) String() string {
 
 //-------------------------------------
 
-// NOTE: keep up-to-date with maxBlockchainResponseSize
 type bcBlockResponseMessage struct {
 	Block *types.Block
 }
@@ -350,7 +387,7 @@ func (m *bcBlockResponseMessage) String() string {
 //-------------------------------------
 
 type bcStatusRequestMessage struct {
-	Height int
+	Height int64
 }
 
 func (m *bcStatusRequestMessage) String() string {
@@ -360,7 +397,7 @@ func (m *bcStatusRequestMessage) String() string {
 //-------------------------------------
 
 type bcStatusResponseMessage struct {
-	Height int
+	Height int64
 }
 
 func (m *bcStatusResponseMessage) String() string {

blockchain/reactor_test.go

@@ -1,71 +1,75 @@
 package blockchain
 
 import (
-	"bytes"
+	"net"
 	"testing"
 
-	wire "github.com/tendermint/go-wire"
 	cmn "github.com/tendermint/tmlibs/common"
-	"github.com/tendermint/tmlibs/db"
+	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
-func newBlockchainReactor(logger log.Logger, maxBlockHeight int) *BlockchainReactor {
-	config := cfg.ResetTestRoot("node_node_test")
+func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) {
+	config := cfg.ResetTestRoot("blockchain_reactor_test")
+	// blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB())
+	// stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB())
+	blockDB := dbm.NewMemDB()
+	stateDB := dbm.NewMemDB()
+	blockStore := NewBlockStore(blockDB)
+	state, err := sm.LoadStateFromDBOrGenesisFile(stateDB, config.GenesisFile())
+	if err != nil {
+		panic(cmn.ErrorWrap(err, "error constructing state from genesis file"))
+	}
+	return state, blockStore
+}
 
-	blockStoreDB := db.NewDB("blockstore", config.DBBackend, config.DBDir())
-	blockStore := NewBlockStore(blockStoreDB)
-
-	stateLogger := logger.With("module", "state")
-
-	// Get State
-	stateDB := db.NewDB("state", config.DBBackend, config.DBDir())
-	state, _ := sm.GetState(stateDB, config.GenesisFile())
-
-	state.SetLogger(stateLogger)
-	state.Save()
+func newBlockchainReactor(logger log.Logger, maxBlockHeight int64) *BlockchainReactor {
+	state, blockStore := makeStateAndBlockStore(logger)
 
 	// Make the blockchainReactor itself
 	fastSync := true
-	bcReactor := NewBlockchainReactor(state.Copy(), nil, blockStore, fastSync)
+	var nilApp proxy.AppConnConsensus
+	blockExec := sm.NewBlockExecutor(dbm.NewMemDB(), log.TestingLogger(), nilApp,
+		sm.MockMempool{}, sm.MockEvidencePool{})
+
+	bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync)
+	bcReactor.SetLogger(logger.With("module", "blockchain"))
 
 	// Next: we need to set a switch in order for peers to be added in
 	bcReactor.Switch = p2p.NewSwitch(cfg.DefaultP2PConfig())
-	bcReactor.SetLogger(logger.With("module", "blockchain"))
 
 	// Lastly: let's add some blocks in
-	for blockHeight := 1; blockHeight <= maxBlockHeight; blockHeight++ {
+	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
 		firstBlock := makeBlock(blockHeight, state)
 		secondBlock := makeBlock(blockHeight+1, state)
-		firstParts := firstBlock.MakePartSet(state.Params().BlockGossipParams.BlockPartSizeBytes)
+		firstParts := firstBlock.MakePartSet(state.ConsensusParams.BlockGossip.BlockPartSizeBytes)
 		blockStore.SaveBlock(firstBlock, firstParts, secondBlock.LastCommit)
 	}
 
 	return bcReactor
 }
 
-func TestNoBlockMessageResponse(t *testing.T) {
-	logBuf := new(bytes.Buffer)
-	logger := log.NewTMLogger(logBuf)
-	maxBlockHeight := 20
+func TestNoBlockResponse(t *testing.T) {
+	maxBlockHeight := int64(20)
 
-	bcr := newBlockchainReactor(logger, maxBlockHeight)
-	go bcr.OnStart()
+	bcr := newBlockchainReactor(log.TestingLogger(), maxBlockHeight)
+	bcr.Start()
 	defer bcr.Stop()
 
 	// Add some peers in
-	peer := newbcrTestPeer(cmn.RandStr(12))
+	peer := newbcrTestPeer(p2p.ID(cmn.RandStr(12)))
 	bcr.AddPeer(peer)
 
 	chID := byte(0x01)
 
 	tests := []struct {
-		height   int
+		height   int64
 		existent bool
 	}{
 		{maxBlockHeight + 2, false},
@@ -74,12 +78,13 @@ func TestNoBlockMessageResponse(t *testing.T) {
 		{100, false},
 	}
 
+	// receive a request message from peer,
+	// wait for our response to be received on the peer
 	for _, tt := range tests {
 		reqBlockMsg := &bcBlockRequestMessage{tt.height}
-		reqBlockBytes := wire.BinaryBytes(struct{ BlockchainMessage }{reqBlockMsg})
+		reqBlockBytes := cdc.MustMarshalBinaryBare(reqBlockMsg)
 		bcr.Receive(chID, peer, reqBlockBytes)
-		value := peer.lastValue()
-		msg := value.(struct{ BlockchainMessage }).BlockchainMessage
+		msg := peer.lastBlockchainMessage()
 
 		if tt.existent {
 			if blockMsg, ok := msg.(*bcBlockResponseMessage); !ok {
@@ -97,62 +102,107 @@ func TestNoBlockMessageResponse(t *testing.T) {
 	}
 }
 
+/*
+// NOTE: This is too hard to test without
+// an easy way to add test peer to switch
+// or without significant refactoring of the module.
+// Alternatively we could actually dial a TCP conn but
+// that seems extreme.
+func TestBadBlockStopsPeer(t *testing.T) {
+	maxBlockHeight := int64(20)
+
+	bcr := newBlockchainReactor(log.TestingLogger(), maxBlockHeight)
+	bcr.Start()
+	defer bcr.Stop()
+
+	// Add some peers in
+	peer := newbcrTestPeer(p2p.ID(cmn.RandStr(12)))
+
+	// XXX: This doesn't add the peer to anything,
+	// so it's hard to check that it's later removed
+	bcr.AddPeer(peer)
+	assert.True(t, bcr.Switch.Peers().Size() > 0)
+
+	// send a bad block from the peer
+	// default blocks already dont have commits, so should fail
+	block := bcr.store.LoadBlock(3)
+	msg := &bcBlockResponseMessage{Block: block}
+	peer.Send(BlockchainChannel, struct{ BlockchainMessage }{msg})
+
+	ticker := time.NewTicker(time.Millisecond * 10)
+	timer := time.NewTimer(time.Second * 2)
+LOOP:
+	for {
+		select {
+		case <-ticker.C:
+			if bcr.Switch.Peers().Size() == 0 {
+				break LOOP
+			}
+		case <-timer.C:
+			t.Fatal("Timed out waiting to disconnect peer")
+		}
+	}
+}
+*/
+
 //----------------------------------------------
 // utility funcs
 
-func makeTxs(blockNumber int) (txs []types.Tx) {
+func makeTxs(height int64) (txs []types.Tx) {
 	for i := 0; i < 10; i++ {
-		txs = append(txs, types.Tx([]byte{byte(blockNumber), byte(i)}))
+		txs = append(txs, types.Tx([]byte{byte(height), byte(i)}))
 	}
 	return txs
 }
 
-func makeBlock(blockNumber int, state *sm.State) *types.Block {
-	prevHash := state.LastBlockID.Hash
-	prevParts := types.PartSetHeader{}
-	valHash := state.Validators.Hash()
-	prevBlockID := types.BlockID{prevHash, prevParts}
-	block, _ := types.MakeBlock(blockNumber, "test_chain", makeTxs(blockNumber),
-		new(types.Commit), prevBlockID, valHash, state.AppHash, state.Params().BlockGossipParams.BlockPartSizeBytes)
+func makeBlock(height int64, state sm.State) *types.Block {
+	block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit))
 	return block
 }
 
 // The Test peer
 type bcrTestPeer struct {
-	cmn.Service
-	key string
-	ch  chan interface{}
+	cmn.BaseService
+	id p2p.ID
+	ch chan interface{}
 }
 
 var _ p2p.Peer = (*bcrTestPeer)(nil)
 
-func newbcrTestPeer(key string) *bcrTestPeer {
-	return &bcrTestPeer{
-		Service: cmn.NewBaseService(nil, "bcrTestPeer", nil),
-		key:     key,
-		ch:      make(chan interface{}, 2),
+func newbcrTestPeer(id p2p.ID) *bcrTestPeer {
+	bcr := &bcrTestPeer{
+		id: id,
+		ch: make(chan interface{}, 2),
 	}
+	bcr.BaseService = *cmn.NewBaseService(nil, "bcrTestPeer", bcr)
+	return bcr
 }
 
-func (tp *bcrTestPeer) lastValue() interface{} { return <-tp.ch }
+func (tp *bcrTestPeer) lastBlockchainMessage() interface{} { return <-tp.ch }
 
-func (tp *bcrTestPeer) TrySend(chID byte, value interface{}) bool {
-	if _, ok := value.(struct{ BlockchainMessage }).BlockchainMessage.(*bcStatusResponseMessage); ok {
+func (tp *bcrTestPeer) TrySend(chID byte, msgBytes []byte) bool {
+	var msg BlockchainMessage
+	err := cdc.UnmarshalBinaryBare(msgBytes, &msg)
+	if err != nil {
+		panic(cmn.ErrorWrap(err, "Error while trying to parse a BlockchainMessage"))
+	}
+	if _, ok := msg.(*bcStatusResponseMessage); ok {
 		// Discard status response messages since they skew our results
 		// We only want to deal with:
 		// + bcBlockResponseMessage
 		// + bcNoBlockResponseMessage
 	} else {
-		tp.ch <- value
+		tp.ch <- msg
 	}
 	return true
 }
 
-func (tp *bcrTestPeer) Send(chID byte, data interface{}) bool { return tp.TrySend(chID, data) }
-func (tp *bcrTestPeer) NodeInfo() *p2p.NodeInfo               { return nil }
-func (tp *bcrTestPeer) Status() p2p.ConnectionStatus          { return p2p.ConnectionStatus{} }
-func (tp *bcrTestPeer) Key() string                           { return tp.key }
-func (tp *bcrTestPeer) IsOutbound() bool                      { return false }
-func (tp *bcrTestPeer) IsPersistent() bool                    { return true }
-func (tp *bcrTestPeer) Get(s string) interface{}              { return s }
-func (tp *bcrTestPeer) Set(string, interface{})               {}
+func (tp *bcrTestPeer) Send(chID byte, msgBytes []byte) bool { return tp.TrySend(chID, msgBytes) }
+func (tp *bcrTestPeer) NodeInfo() p2p.NodeInfo               { return p2p.NodeInfo{} }
+func (tp *bcrTestPeer) Status() p2p.ConnectionStatus         { return p2p.ConnectionStatus{} }
+func (tp *bcrTestPeer) ID() p2p.ID                           { return tp.id }
+func (tp *bcrTestPeer) IsOutbound() bool                     { return false }
+func (tp *bcrTestPeer) IsPersistent() bool                   { return true }
+func (tp *bcrTestPeer) Get(s string) interface{}             { return s }
+func (tp *bcrTestPeer) Set(string, interface{})              {}
+func (tp *bcrTestPeer) RemoteIP() net.IP                     { return []byte{127, 0, 0, 1} }

blockchain/store.go

@@ -1,20 +1,17 @@
 package blockchain
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
-	"io"
 	"sync"
 
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
-	"github.com/tendermint/go-wire"
+
 	"github.com/tendermint/tendermint/types"
 )
 
 /*
-Simple low level store for blocks.
+BlockStore is a simple low level store for blocks.
 
 There are three types of information stored:
  - BlockMeta:   Meta information about each block
@@ -23,17 +20,20 @@ There are three types of information stored:
 
 Currently the precommit signatures are duplicated in the Block parts as
 well as the Commit.  In the future this may change, perhaps by moving
-the Commit data outside the Block.
+the Commit data outside the Block. (TODO)
 
-Panics indicate probable corruption in the data
+// NOTE: BlockStore methods will panic if they encounter errors
+// deserializing loaded data, indicating probable corruption on disk.
 */
 type BlockStore struct {
 	db dbm.DB
 
 	mtx    sync.RWMutex
-	height int
+	height int64
 }
 
+// NewBlockStore returns a new BlockStore with the given DB,
+// initialized to the last height that was committed to the DB.
 func NewBlockStore(db dbm.DB) *BlockStore {
 	bsjson := LoadBlockStoreStateJSON(db)
 	return &BlockStore{
@@ -42,134 +42,136 @@ func NewBlockStore(db dbm.DB) *BlockStore {
 	}
 }
 
-// Height() returns the last known contiguous block height.
-func (bs *BlockStore) Height() int {
+// Height returns the last known contiguous block height.
+func (bs *BlockStore) Height() int64 {
 	bs.mtx.RLock()
 	defer bs.mtx.RUnlock()
 	return bs.height
 }
 
-func (bs *BlockStore) GetReader(key []byte) io.Reader {
-	bytez := bs.db.Get(key)
-	if bytez == nil {
+// LoadBlock returns the block with the given height.
+// If no block is found for that height, it returns nil.
+func (bs *BlockStore) LoadBlock(height int64) *types.Block {
+	var blockMeta = bs.LoadBlockMeta(height)
+	if blockMeta == nil {
 		return nil
 	}
-	return bytes.NewReader(bytez)
-}
 
-func (bs *BlockStore) LoadBlock(height int) *types.Block {
-	var n int
-	var err error
-	r := bs.GetReader(calcBlockMetaKey(height))
-	if r == nil {
-		return nil
-	}
-	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
-	if err != nil {
-		PanicCrisis(Fmt("Error reading block meta: %v", err))
-	}
-	bytez := []byte{}
+	var block = new(types.Block)
+	buf := []byte{}
 	for i := 0; i < blockMeta.BlockID.PartsHeader.Total; i++ {
 		part := bs.LoadBlockPart(height, i)
-		bytez = append(bytez, part.Bytes...)
+		buf = append(buf, part.Bytes...)
 	}
-	block := wire.ReadBinary(&types.Block{}, bytes.NewReader(bytez), 0, &n, &err).(*types.Block)
+	err := cdc.UnmarshalBinary(buf, block)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block: %v", err))
+		// NOTE: The existence of meta should imply the existence of the
+		// block. So, make sure meta is only saved after blocks are saved.
+		panic(cmn.ErrorWrap(err, "Error reading block"))
 	}
 	return block
 }
 
-func (bs *BlockStore) LoadBlockPart(height int, index int) *types.Part {
-	var n int
-	var err error
-	r := bs.GetReader(calcBlockPartKey(height, index))
-	if r == nil {
+// LoadBlockPart returns the Part at the given index
+// from the block at the given height.
+// If no part is found for the given height and index, it returns nil.
+func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
+	var part = new(types.Part)
+	bz := bs.db.Get(calcBlockPartKey(height, index))
+	if len(bz) == 0 {
 		return nil
 	}
-	part := wire.ReadBinary(&types.Part{}, r, 0, &n, &err).(*types.Part)
+	err := cdc.UnmarshalBinaryBare(bz, part)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block part: %v", err))
+		panic(cmn.ErrorWrap(err, "Error reading block part"))
 	}
 	return part
 }
 
-func (bs *BlockStore) LoadBlockMeta(height int) *types.BlockMeta {
-	var n int
-	var err error
-	r := bs.GetReader(calcBlockMetaKey(height))
-	if r == nil {
+// LoadBlockMeta returns the BlockMeta for the given height.
+// If no block is found for the given height, it returns nil.
+func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
+	var blockMeta = new(types.BlockMeta)
+	bz := bs.db.Get(calcBlockMetaKey(height))
+	if len(bz) == 0 {
 		return nil
 	}
-	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
+	err := cdc.UnmarshalBinaryBare(bz, blockMeta)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block meta: %v", err))
+		panic(cmn.ErrorWrap(err, "Error reading block meta"))
 	}
 	return blockMeta
 }
 
-// The +2/3 and other Precommit-votes for block at `height`.
-// This Commit comes from block.LastCommit for `height+1`.
-func (bs *BlockStore) LoadBlockCommit(height int) *types.Commit {
-	var n int
-	var err error
-	r := bs.GetReader(calcBlockCommitKey(height))
-	if r == nil {
+// LoadBlockCommit returns the Commit for the given height.
+// This commit consists of the +2/3 and other Precommit-votes for block at `height`,
+// and it comes from the block.LastCommit for `height+1`.
+// If no commit is found for the given height, it returns nil.
+func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
+	var commit = new(types.Commit)
+	bz := bs.db.Get(calcBlockCommitKey(height))
+	if len(bz) == 0 {
 		return nil
 	}
-	commit := wire.ReadBinary(&types.Commit{}, r, 0, &n, &err).(*types.Commit)
+	err := cdc.UnmarshalBinaryBare(bz, commit)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading commit: %v", err))
+		panic(cmn.ErrorWrap(err, "Error reading block commit"))
 	}
 	return commit
 }
 
-// NOTE: the Precommit-vote heights are for the block at `height`
-func (bs *BlockStore) LoadSeenCommit(height int) *types.Commit {
-	var n int
-	var err error
-	r := bs.GetReader(calcSeenCommitKey(height))
-	if r == nil {
+// LoadSeenCommit returns the locally seen Commit for the given height.
+// This is useful when we've seen a commit, but there has not yet been
+// a new block at `height + 1` that includes this commit in its block.LastCommit.
+func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
+	var commit = new(types.Commit)
+	bz := bs.db.Get(calcSeenCommitKey(height))
+	if len(bz) == 0 {
 		return nil
 	}
-	commit := wire.ReadBinary(&types.Commit{}, r, 0, &n, &err).(*types.Commit)
+	err := cdc.UnmarshalBinaryBare(bz, commit)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading commit: %v", err))
+		panic(cmn.ErrorWrap(err, "Error reading block seen commit"))
 	}
 	return commit
 }
 
+// SaveBlock persists the given block, blockParts, and seenCommit to the underlying db.
 // blockParts: Must be parts of the block
 // seenCommit: The +2/3 precommits that were seen which committed at height.
 //             If all the nodes restart after committing a block,
 //             we need this to reload the precommits to catch-up nodes to the
 //             most recent height.  Otherwise they'd stall at H-1.
 func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+	if block == nil {
+		cmn.PanicSanity("BlockStore can only save a non-nil block")
+	}
 	height := block.Height
-	if height != bs.Height()+1 {
-		PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
+	if g, w := height, bs.Height()+1; g != w {
+		cmn.PanicSanity(cmn.Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", w, g))
 	}
 	if !blockParts.IsComplete() {
-		PanicSanity(Fmt("BlockStore can only save complete block part sets"))
+		cmn.PanicSanity(cmn.Fmt("BlockStore can only save complete block part sets"))
 	}
 
 	// Save block meta
 	blockMeta := types.NewBlockMeta(block, blockParts)
-	metaBytes := wire.BinaryBytes(blockMeta)
+	metaBytes := cdc.MustMarshalBinaryBare(blockMeta)
 	bs.db.Set(calcBlockMetaKey(height), metaBytes)
 
 	// Save block parts
 	for i := 0; i < blockParts.Total(); i++ {
-		bs.saveBlockPart(height, i, blockParts.GetPart(i))
+		part := blockParts.GetPart(i)
+		bs.saveBlockPart(height, i, part)
 	}
 
 	// Save block commit (duplicate and separate from the Block)
-	blockCommitBytes := wire.BinaryBytes(block.LastCommit)
+	blockCommitBytes := cdc.MustMarshalBinaryBare(block.LastCommit)
 	bs.db.Set(calcBlockCommitKey(height-1), blockCommitBytes)
 
 	// Save seen commit (seen +2/3 precommits for block)
 	// NOTE: we can delete this at a later height
-	seenCommitBytes := wire.BinaryBytes(seenCommit)
+	seenCommitBytes := cdc.MustMarshalBinaryBare(seenCommit)
 	bs.db.Set(calcSeenCommitKey(height), seenCommitBytes)
 
 	// Save new BlockStoreStateJSON descriptor
@@ -184,29 +186,29 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
 	bs.db.SetSync(nil, nil)
 }
 
-func (bs *BlockStore) saveBlockPart(height int, index int, part *types.Part) {
+func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) {
 	if height != bs.Height()+1 {
-		PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
+		cmn.PanicSanity(cmn.Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
 	}
-	partBytes := wire.BinaryBytes(part)
+	partBytes := cdc.MustMarshalBinaryBare(part)
 	bs.db.Set(calcBlockPartKey(height, index), partBytes)
 }
 
 //-----------------------------------------------------------------------------
 
-func calcBlockMetaKey(height int) []byte {
+func calcBlockMetaKey(height int64) []byte {
 	return []byte(fmt.Sprintf("H:%v", height))
 }
 
-func calcBlockPartKey(height int, partIndex int) []byte {
+func calcBlockPartKey(height int64, partIndex int) []byte {
 	return []byte(fmt.Sprintf("P:%v:%v", height, partIndex))
 }
 
-func calcBlockCommitKey(height int) []byte {
+func calcBlockCommitKey(height int64) []byte {
 	return []byte(fmt.Sprintf("C:%v", height))
 }
 
-func calcSeenCommitKey(height int) []byte {
+func calcSeenCommitKey(height int64) []byte {
 	return []byte(fmt.Sprintf("SC:%v", height))
 }
 
@@ -215,28 +217,31 @@ func calcSeenCommitKey(height int) []byte {
 var blockStoreKey = []byte("blockStore")
 
 type BlockStoreStateJSON struct {
-	Height int
+	Height int64 `json:"height"`
 }
 
+// Save persists the blockStore state to the database as JSON.
 func (bsj BlockStoreStateJSON) Save(db dbm.DB) {
-	bytes, err := json.Marshal(bsj)
+	bytes, err := cdc.MarshalJSON(bsj)
 	if err != nil {
-		PanicSanity(Fmt("Could not marshal state bytes: %v", err))
+		cmn.PanicSanity(cmn.Fmt("Could not marshal state bytes: %v", err))
 	}
 	db.SetSync(blockStoreKey, bytes)
 }
 
+// LoadBlockStoreStateJSON returns the BlockStoreStateJSON as loaded from disk.
+// If no BlockStoreStateJSON was previously persisted, it returns the zero value.
 func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON {
 	bytes := db.Get(blockStoreKey)
-	if bytes == nil {
+	if len(bytes) == 0 {
 		return BlockStoreStateJSON{
 			Height: 0,
 		}
 	}
 	bsj := BlockStoreStateJSON{}
-	err := json.Unmarshal(bytes, &bsj)
+	err := cdc.UnmarshalJSON(bytes, &bsj)
 	if err != nil {
-		PanicCrisis(Fmt("Could not unmarshal bytes: %X", bytes))
+		panic(fmt.Sprintf("Could not unmarshal bytes: %X", bytes))
 	}
 	return bsj
 }

blockchain/store_test.go

@@ -0,0 +1,383 @@
+package blockchain
+
+import (
+	"bytes"
+	"fmt"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestLoadBlockStoreStateJSON(t *testing.T) {
+	db := db.NewMemDB()
+
+	bsj := &BlockStoreStateJSON{Height: 1000}
+	bsj.Save(db)
+
+	retrBSJ := LoadBlockStoreStateJSON(db)
+
+	assert.Equal(t, *bsj, retrBSJ, "expected the retrieved DBs to match")
+}
+
+func TestNewBlockStore(t *testing.T) {
+	db := db.NewMemDB()
+	db.Set(blockStoreKey, []byte(`{"height": 10000}`))
+	bs := NewBlockStore(db)
+	require.Equal(t, int64(10000), bs.Height(), "failed to properly parse blockstore")
+
+	panicCausers := []struct {
+		data    []byte
+		wantErr string
+	}{
+		{[]byte("artful-doger"), "not unmarshal bytes"},
+		{[]byte(" "), "unmarshal bytes"},
+	}
+
+	for i, tt := range panicCausers {
+		// Expecting a panic here on trying to parse an invalid blockStore
+		_, _, panicErr := doFn(func() (interface{}, error) {
+			db.Set(blockStoreKey, tt.data)
+			_ = NewBlockStore(db)
+			return nil, nil
+		})
+		require.NotNil(t, panicErr, "#%d panicCauser: %q expected a panic", i, tt.data)
+		assert.Contains(t, panicErr.Error(), tt.wantErr, "#%d data: %q", i, tt.data)
+	}
+
+	db.Set(blockStoreKey, nil)
+	bs = NewBlockStore(db)
+	assert.Equal(t, bs.Height(), int64(0), "expecting nil bytes to be unmarshaled alright")
+}
+
+func freshBlockStore() (*BlockStore, db.DB) {
+	db := db.NewMemDB()
+	return NewBlockStore(db), db
+}
+
+var (
+	state, _ = makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+
+	block       = makeBlock(1, state)
+	partSet     = block.MakePartSet(2)
+	part1       = partSet.GetPart(0)
+	part2       = partSet.GetPart(1)
+	seenCommit1 = &types.Commit{Precommits: []*types.Vote{{Height: 10,
+		Timestamp: time.Now().UTC()}}}
+)
+
+// TODO: This test should be simplified ...
+
+func TestBlockStoreSaveLoadBlock(t *testing.T) {
+	state, bs := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	require.Equal(t, bs.Height(), int64(0), "initially the height should be zero")
+
+	// check there are no blocks at various heights
+	noBlockHeights := []int64{0, -1, 100, 1000, 2}
+	for i, height := range noBlockHeights {
+		if g := bs.LoadBlock(height); g != nil {
+			t.Errorf("#%d: height(%d) got a block; want nil", i, height)
+		}
+	}
+
+	// save a block
+	block := makeBlock(bs.Height()+1, state)
+	validPartSet := block.MakePartSet(2)
+	seenCommit := &types.Commit{Precommits: []*types.Vote{{Height: 10,
+		Timestamp: time.Now().UTC()}}}
+	bs.SaveBlock(block, partSet, seenCommit)
+	require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed")
+
+	incompletePartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 2})
+	uncontiguousPartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 0})
+	uncontiguousPartSet.AddPart(part2)
+
+	header1 := types.Header{
+		Height:  1,
+		NumTxs:  100,
+		ChainID: "block_test",
+		Time:    time.Now(),
+	}
+	header2 := header1
+	header2.Height = 4
+
+	// End of setup, test data
+
+	commitAtH10 := &types.Commit{Precommits: []*types.Vote{{Height: 10,
+		Timestamp: time.Now().UTC()}}}
+	tuples := []struct {
+		block      *types.Block
+		parts      *types.PartSet
+		seenCommit *types.Commit
+		wantErr    bool
+		wantPanic  string
+
+		corruptBlockInDB      bool
+		corruptCommitInDB     bool
+		corruptSeenCommitInDB bool
+		eraseCommitInDB       bool
+		eraseSeenCommitInDB   bool
+	}{
+		{
+			block:      newBlock(&header1, commitAtH10),
+			parts:      validPartSet,
+			seenCommit: seenCommit1,
+		},
+
+		{
+			block:     nil,
+			wantPanic: "only save a non-nil block",
+		},
+
+		{
+			block:     newBlock(&header2, commitAtH10),
+			parts:     uncontiguousPartSet,
+			wantPanic: "only save contiguous blocks", // and incomplete and uncontiguous parts
+		},
+
+		{
+			block:     newBlock(&header1, commitAtH10),
+			parts:     incompletePartSet,
+			wantPanic: "only save complete block", // incomplete parts
+		},
+
+		{
+			block:             newBlock(&header1, commitAtH10),
+			parts:             validPartSet,
+			seenCommit:        seenCommit1,
+			corruptCommitInDB: true, // Corrupt the DB's commit entry
+			wantPanic:         "Error reading block commit",
+		},
+
+		{
+			block:            newBlock(&header1, commitAtH10),
+			parts:            validPartSet,
+			seenCommit:       seenCommit1,
+			wantPanic:        "Error reading block",
+			corruptBlockInDB: true, // Corrupt the DB's block entry
+		},
+
+		{
+			block:      newBlock(&header1, commitAtH10),
+			parts:      validPartSet,
+			seenCommit: seenCommit1,
+
+			// Expecting no error and we want a nil back
+			eraseSeenCommitInDB: true,
+		},
+
+		{
+			block:      newBlock(&header1, commitAtH10),
+			parts:      validPartSet,
+			seenCommit: seenCommit1,
+
+			corruptSeenCommitInDB: true,
+			wantPanic:             "Error reading block seen commit",
+		},
+
+		{
+			block:      newBlock(&header1, commitAtH10),
+			parts:      validPartSet,
+			seenCommit: seenCommit1,
+
+			// Expecting no error and we want a nil back
+			eraseCommitInDB: true,
+		},
+	}
+
+	type quad struct {
+		block  *types.Block
+		commit *types.Commit
+		meta   *types.BlockMeta
+
+		seenCommit *types.Commit
+	}
+
+	for i, tuple := range tuples {
+		bs, db := freshBlockStore()
+		// SaveBlock
+		res, err, panicErr := doFn(func() (interface{}, error) {
+			bs.SaveBlock(tuple.block, tuple.parts, tuple.seenCommit)
+			if tuple.block == nil {
+				return nil, nil
+			}
+
+			if tuple.corruptBlockInDB {
+				db.Set(calcBlockMetaKey(tuple.block.Height), []byte("block-bogus"))
+			}
+			bBlock := bs.LoadBlock(tuple.block.Height)
+			bBlockMeta := bs.LoadBlockMeta(tuple.block.Height)
+
+			if tuple.eraseSeenCommitInDB {
+				db.Delete(calcSeenCommitKey(tuple.block.Height))
+			}
+			if tuple.corruptSeenCommitInDB {
+				db.Set(calcSeenCommitKey(tuple.block.Height), []byte("bogus-seen-commit"))
+			}
+			bSeenCommit := bs.LoadSeenCommit(tuple.block.Height)
+
+			commitHeight := tuple.block.Height - 1
+			if tuple.eraseCommitInDB {
+				db.Delete(calcBlockCommitKey(commitHeight))
+			}
+			if tuple.corruptCommitInDB {
+				db.Set(calcBlockCommitKey(commitHeight), []byte("foo-bogus"))
+			}
+			bCommit := bs.LoadBlockCommit(commitHeight)
+			return &quad{block: bBlock, seenCommit: bSeenCommit, commit: bCommit,
+				meta: bBlockMeta}, nil
+		})
+
+		if subStr := tuple.wantPanic; subStr != "" {
+			if panicErr == nil {
+				t.Errorf("#%d: want a non-nil panic", i)
+			} else if got := panicErr.Error(); !strings.Contains(got, subStr) {
+				t.Errorf("#%d:\n\tgotErr: %q\nwant substring: %q", i, got, subStr)
+			}
+			continue
+		}
+
+		if tuple.wantErr {
+			if err == nil {
+				t.Errorf("#%d: got nil error", i)
+			}
+			continue
+		}
+
+		assert.Nil(t, panicErr, "#%d: unexpected panic", i)
+		assert.Nil(t, err, "#%d: expecting a non-nil error", i)
+		qua, ok := res.(*quad)
+		if !ok || qua == nil {
+			t.Errorf("#%d: got nil quad back; gotType=%T", i, res)
+			continue
+		}
+		if tuple.eraseSeenCommitInDB {
+			assert.Nil(t, qua.seenCommit,
+				"erased the seenCommit in the DB hence we should get back a nil seenCommit")
+		}
+		if tuple.eraseCommitInDB {
+			assert.Nil(t, qua.commit,
+				"erased the commit in the DB hence we should get back a nil commit")
+		}
+	}
+}
+
+func TestLoadBlockPart(t *testing.T) {
+	bs, db := freshBlockStore()
+	height, index := int64(10), 1
+	loadPart := func() (interface{}, error) {
+		part := bs.LoadBlockPart(height, index)
+		return part, nil
+	}
+
+	// Initially no contents.
+	// 1. Requesting for a non-existent block shouldn't fail
+	res, _, panicErr := doFn(loadPart)
+	require.Nil(t, panicErr, "a non-existent block part shouldn't cause a panic")
+	require.Nil(t, res, "a non-existent block part should return nil")
+
+	// 2. Next save a corrupted block then try to load it
+	db.Set(calcBlockPartKey(height, index), []byte("Tendermint"))
+	res, _, panicErr = doFn(loadPart)
+	require.NotNil(t, panicErr, "expecting a non-nil panic")
+	require.Contains(t, panicErr.Error(), "Error reading block part")
+
+	// 3. A good block serialized and saved to the DB should be retrievable
+	db.Set(calcBlockPartKey(height, index), cdc.MustMarshalBinaryBare(part1))
+	gotPart, _, panicErr := doFn(loadPart)
+	require.Nil(t, panicErr, "an existent and proper block should not panic")
+	require.Nil(t, res, "a properly saved block should return a proper block")
+	require.Equal(t, gotPart.(*types.Part).Hash(), part1.Hash(),
+		"expecting successful retrieval of previously saved block")
+}
+
+func TestLoadBlockMeta(t *testing.T) {
+	bs, db := freshBlockStore()
+	height := int64(10)
+	loadMeta := func() (interface{}, error) {
+		meta := bs.LoadBlockMeta(height)
+		return meta, nil
+	}
+
+	// Initially no contents.
+	// 1. Requesting for a non-existent blockMeta shouldn't fail
+	res, _, panicErr := doFn(loadMeta)
+	require.Nil(t, panicErr, "a non-existent blockMeta shouldn't cause a panic")
+	require.Nil(t, res, "a non-existent blockMeta should return nil")
+
+	// 2. Next save a corrupted blockMeta then try to load it
+	db.Set(calcBlockMetaKey(height), []byte("Tendermint-Meta"))
+	res, _, panicErr = doFn(loadMeta)
+	require.NotNil(t, panicErr, "expecting a non-nil panic")
+	require.Contains(t, panicErr.Error(), "Error reading block meta")
+
+	// 3. A good blockMeta serialized and saved to the DB should be retrievable
+	meta := &types.BlockMeta{}
+	db.Set(calcBlockMetaKey(height), cdc.MustMarshalBinaryBare(meta))
+	gotMeta, _, panicErr := doFn(loadMeta)
+	require.Nil(t, panicErr, "an existent and proper block should not panic")
+	require.Nil(t, res, "a properly saved blockMeta should return a proper blocMeta ")
+	require.Equal(t, cdc.MustMarshalBinaryBare(meta), cdc.MustMarshalBinaryBare(gotMeta),
+		"expecting successful retrieval of previously saved blockMeta")
+}
+
+func TestBlockFetchAtHeight(t *testing.T) {
+	state, bs := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	require.Equal(t, bs.Height(), int64(0), "initially the height should be zero")
+	block := makeBlock(bs.Height()+1, state)
+
+	partSet := block.MakePartSet(2)
+	seenCommit := &types.Commit{Precommits: []*types.Vote{{Height: 10,
+		Timestamp: time.Now().UTC()}}}
+
+	bs.SaveBlock(block, partSet, seenCommit)
+	require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed")
+
+	blockAtHeight := bs.LoadBlock(bs.Height())
+	bz1 := cdc.MustMarshalBinaryBare(block)
+	bz2 := cdc.MustMarshalBinaryBare(blockAtHeight)
+	require.Equal(t, bz1, bz2)
+	require.Equal(t, block.Hash(), blockAtHeight.Hash(),
+		"expecting a successful load of the last saved block")
+
+	blockAtHeightPlus1 := bs.LoadBlock(bs.Height() + 1)
+	require.Nil(t, blockAtHeightPlus1, "expecting an unsuccessful load of Height()+1")
+	blockAtHeightPlus2 := bs.LoadBlock(bs.Height() + 2)
+	require.Nil(t, blockAtHeightPlus2, "expecting an unsuccessful load of Height()+2")
+}
+
+func doFn(fn func() (interface{}, error)) (res interface{}, err error, panicErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			switch e := r.(type) {
+			case error:
+				panicErr = e
+			case string:
+				panicErr = fmt.Errorf("%s", e)
+			default:
+				if st, ok := r.(fmt.Stringer); ok {
+					panicErr = fmt.Errorf("%s", st)
+				} else {
+					panicErr = fmt.Errorf("%s", debug.Stack())
+				}
+			}
+		}
+	}()
+
+	res, err = fn()
+	return res, err, panicErr
+}
+
+func newBlock(hdr *types.Header, lastCommit *types.Commit) *types.Block {
+	return &types.Block{
+		Header:     hdr,
+		LastCommit: lastCommit,
+	}
+}

blockchain/wire.go

@@ -0,0 +1,13 @@
+package blockchain
+
+import (
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/go-crypto"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	RegisterBlockchainMessages(cdc)
+	crypto.RegisterAmino(cdc)
+}

circle.yml

@@ -1,33 +0,0 @@
----
-machine:
-  environment:
-    MACH_PREFIX: tendermint-test-mach
-    DOCKER_VERSION: 1.10.0
-    DOCKER_MACHINE_VERSION: 0.9.0
-    GOPATH: "$HOME/.go_project"
-    PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
-    PROJECT_PATH: "$PROJECT_PARENT_PATH/$CIRCLE_PROJECT_REPONAME"
-  hosts:
-    localhost: 127.0.0.1
-
-dependencies:
-  override:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | sudo bash -s -- $DOCKER_VERSION
-    - sudo start docker
-    - sudo curl -sSL -o /usr/bin/docker-machine "https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-`uname -s`-`uname -m`"; sudo chmod 0755 /usr/bin/docker-machine
-    - mkdir -p "$PROJECT_PARENT_PATH"
-    - ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH"
-  post:
-    - go version
-    - docker version
-    - docker-machine version
-
-test:
-  override:
-    - cd "$PROJECT_PATH" && set -o pipefail && make test_integrations 2>&1 | tee test_integrations.log:
-        timeout: 1800
-  post:
-    - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
-    - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
-    - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"
-    - cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker_logs.txt"

cmd/priv_val_server/main.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	crypto "github.com/tendermint/go-crypto"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/tendermint/privval"
+)
+
+func main() {
+	var (
+		addr        = flag.String("addr", ":26659", "Address of client to connect to")
+		chainID     = flag.String("chain-id", "mychain", "chain id")
+		privValPath = flag.String("priv", "", "priv val file path")
+
+		logger = log.NewTMLogger(
+			log.NewSyncWriter(os.Stdout),
+		).With("module", "priv_val")
+	)
+	flag.Parse()
+
+	logger.Info(
+		"Starting private validator",
+		"addr", *addr,
+		"chainID", *chainID,
+		"privPath", *privValPath,
+	)
+
+	pv := privval.LoadFilePV(*privValPath)
+
+	rs := privval.NewRemoteSigner(
+		logger,
+		*chainID,
+		*addr,
+		pv,
+		crypto.GenPrivKeyEd25519(),
+	)
+	err := rs.Start()
+	if err != nil {
+		panic(err)
+	}
+
+	cmn.TrapSignal(func() {
+		err := rs.Stop()
+		if err != nil {
+			panic(err)
+		}
+	})
+}

cmd/tendermint/commands/gen_node_key.go

@@ -0,0 +1,32 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/p2p"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+// GenNodeKeyCmd allows the generation of a node key. It prints node's ID to
+// the standard output.
+var GenNodeKeyCmd = &cobra.Command{
+	Use:   "gen_node_key",
+	Short: "Generate a node key for this node and print its ID",
+	RunE:  genNodeKey,
+}
+
+func genNodeKey(cmd *cobra.Command, args []string) error {
+	nodeKeyFile := config.NodeKeyFile()
+	if cmn.FileExists(nodeKeyFile) {
+		return fmt.Errorf("node key at %s already exists", nodeKeyFile)
+	}
+
+	nodeKey, err := p2p.LoadOrGenNodeKey(nodeKeyFile)
+	if err != nil {
+		return err
+	}
+	fmt.Println(nodeKey.ID())
+	return nil
+}

cmd/tendermint/commands/gen_validator.go

@@ -1,12 +1,11 @@
 package commands
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/spf13/cobra"
 
-	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/privval"
 )
 
 // GenValidatorCmd allows the generation of a keypair for a
@@ -18,8 +17,11 @@ var GenValidatorCmd = &cobra.Command{
 }
 
 func genValidator(cmd *cobra.Command, args []string) {
-	privValidator := types.GenPrivValidatorFS("")
-	privValidatorJSONBytes, _ := json.MarshalIndent(privValidator, "", "\t")
+	pv := privval.GenFilePV("")
+	jsbz, err := cdc.MarshalJSON(pv)
+	if err != nil {
+		panic(err)
+	}
 	fmt.Printf(`%v
-`, string(privValidatorJSONBytes))
+`, string(jsbz))
 }

cmd/tendermint/commands/init.go

@@ -1,10 +1,13 @@
 package commands
 
 import (
-	"os"
+	"time"
 
 	"github.com/spf13/cobra"
 
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 )
@@ -13,31 +16,55 @@ import (
 var InitFilesCmd = &cobra.Command{
 	Use:   "init",
 	Short: "Initialize Tendermint",
-	Run:   initFiles,
+	RunE:  initFiles,
 }
 
-func initFiles(cmd *cobra.Command, args []string) {
+func initFiles(cmd *cobra.Command, args []string) error {
+	return initFilesWithConfig(config)
+}
+
+func initFilesWithConfig(config *cfg.Config) error {
+	// private validator
 	privValFile := config.PrivValidatorFile()
-	if _, err := os.Stat(privValFile); os.IsNotExist(err) {
-		privValidator := types.GenPrivValidatorFS(privValFile)
-		privValidator.Save()
-
-		genFile := config.GenesisFile()
-
-		if _, err := os.Stat(genFile); os.IsNotExist(err) {
-			genDoc := types.GenesisDoc{
-				ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)),
-			}
-			genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{
-				PubKey: privValidator.GetPubKey(),
-				Power:  10,
-			}}
-
-			genDoc.SaveAs(genFile)
-		}
-
-		logger.Info("Initialized tendermint", "genesis", config.GenesisFile(), "priv_validator", config.PrivValidatorFile())
+	var pv *privval.FilePV
+	if cmn.FileExists(privValFile) {
+		pv = privval.LoadFilePV(privValFile)
+		logger.Info("Found private validator", "path", privValFile)
 	} else {
-		logger.Info("Already initialized", "priv_validator", config.PrivValidatorFile())
+		pv = privval.GenFilePV(privValFile)
+		pv.Save()
+		logger.Info("Generated private validator", "path", privValFile)
 	}
+
+	nodeKeyFile := config.NodeKeyFile()
+	if cmn.FileExists(nodeKeyFile) {
+		logger.Info("Found node key", "path", nodeKeyFile)
+	} else {
+		if _, err := p2p.LoadOrGenNodeKey(nodeKeyFile); err != nil {
+			return err
+		}
+		logger.Info("Generated node key", "path", nodeKeyFile)
+	}
+
+	// genesis file
+	genFile := config.GenesisFile()
+	if cmn.FileExists(genFile) {
+		logger.Info("Found genesis file", "path", genFile)
+	} else {
+		genDoc := types.GenesisDoc{
+			ChainID:     cmn.Fmt("test-chain-%v", cmn.RandStr(6)),
+			GenesisTime: time.Now(),
+		}
+		genDoc.Validators = []types.GenesisValidator{{
+			PubKey: pv.GetPubKey(),
+			Power:  10,
+		}}
+
+		if err := genDoc.SaveAs(genFile); err != nil {
+			return err
+		}
+		logger.Info("Generated genesis file", "path", genFile)
+	}
+
+	return nil
 }

cmd/tendermint/commands/lite.go

@@ -0,0 +1,87 @@
+package commands
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/spf13/cobra"
+
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/lite/proxy"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+)
+
+// LiteCmd represents the base command when called without any subcommands
+var LiteCmd = &cobra.Command{
+	Use:   "lite",
+	Short: "Run lite-client proxy server, verifying tendermint rpc",
+	Long: `This node will run a secure proxy to a tendermint rpc server.
+
+All calls that can be tracked back to a block header by a proof
+will be verified before passing them back to the caller. Other that
+that it will present the same interface as a full tendermint node,
+just with added trust and running locally.`,
+	RunE:         runProxy,
+	SilenceUsage: true,
+}
+
+var (
+	listenAddr string
+	nodeAddr   string
+	chainID    string
+	home       string
+)
+
+func init() {
+	LiteCmd.Flags().StringVar(&listenAddr, "laddr", "tcp://localhost:8888", "Serve the proxy on the given address")
+	LiteCmd.Flags().StringVar(&nodeAddr, "node", "tcp://localhost:26657", "Connect to a Tendermint node at this address")
+	LiteCmd.Flags().StringVar(&chainID, "chain-id", "tendermint", "Specify the Tendermint chain ID")
+	LiteCmd.Flags().StringVar(&home, "home-dir", ".tendermint-lite", "Specify the home directory")
+}
+
+func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) {
+	u, err := url.Parse(addr)
+	if err != nil {
+		return "", err
+	}
+	switch u.Scheme {
+	case "tcp", "unix":
+	case "":
+		u.Scheme = "tcp"
+	default:
+		return "", fmt.Errorf("unknown scheme %q, use either tcp or unix", u.Scheme)
+	}
+	return u.String(), nil
+}
+
+func runProxy(cmd *cobra.Command, args []string) error {
+	nodeAddr, err := ensureAddrHasSchemeOrDefaultToTCP(nodeAddr)
+	if err != nil {
+		return err
+	}
+	listenAddr, err := ensureAddrHasSchemeOrDefaultToTCP(listenAddr)
+	if err != nil {
+		return err
+	}
+
+	// First, connect a client
+	node := rpcclient.NewHTTP(nodeAddr, "/websocket")
+
+	cert, err := proxy.GetCertifier(chainID, home, nodeAddr)
+	if err != nil {
+		return err
+	}
+	sc := proxy.SecureClient(node, cert)
+
+	err = proxy.StartProxy(sc, listenAddr, logger)
+	if err != nil {
+		return err
+	}
+
+	cmn.TrapSignal(func() {
+		// TODO: close up shop
+	})
+
+	return nil
+}

cmd/tendermint/commands/probe_upnp.go

@@ -1,7 +1,6 @@
 package commands
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/spf13/cobra"
@@ -19,10 +18,10 @@ var ProbeUpnpCmd = &cobra.Command{
 func probeUpnp(cmd *cobra.Command, args []string) error {
 	capabilities, err := upnp.Probe(logger)
 	if err != nil {
-		fmt.Println("Probe failed: %v", err)
+		fmt.Println("Probe failed: ", err)
 	} else {
 		fmt.Println("Probe success!")
-		jsonBytes, err := json.Marshal(capabilities)
+		jsonBytes, err := cdc.MarshalJSON(capabilities)
 		if err != nil {
 			return err
 		}

cmd/tendermint/commands/reset_priv_validator.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tmlibs/log"
 )
 
@@ -13,46 +13,57 @@ import (
 // instance.
 var ResetAllCmd = &cobra.Command{
 	Use:   "unsafe_reset_all",
-	Short: "(unsafe) Remove all the data and WAL, reset this node's validator",
+	Short: "(unsafe) Remove all the data and WAL, reset this node's validator to genesis state",
 	Run:   resetAll,
 }
 
 // ResetPrivValidatorCmd resets the private validator files.
 var ResetPrivValidatorCmd = &cobra.Command{
 	Use:   "unsafe_reset_priv_validator",
-	Short: "(unsafe) Reset this node's validator",
+	Short: "(unsafe) Reset this node's validator to genesis state",
 	Run:   resetPrivValidator,
 }
 
-// ResetAll removes the privValidator files.
-// Exported so other CLI tools can use  it
-func ResetAll(dbDir, privValFile string, logger log.Logger) {
-	resetPrivValidatorFS(privValFile, logger)
-	os.RemoveAll(dbDir)
-	logger.Info("Removed all data", "dir", dbDir)
-}
-
 // XXX: this is totally unsafe.
 // it's only suitable for testnets.
 func resetAll(cmd *cobra.Command, args []string) {
-	ResetAll(config.DBDir(), config.PrivValidatorFile(), logger)
+	ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorFile(), logger)
 }
 
 // XXX: this is totally unsafe.
 // it's only suitable for testnets.
 func resetPrivValidator(cmd *cobra.Command, args []string) {
-	resetPrivValidatorFS(config.PrivValidatorFile(), logger)
+	resetFilePV(config.PrivValidatorFile(), logger)
 }
 
-func resetPrivValidatorFS(privValFile string, logger log.Logger) {
-	// Get PrivValidator
-	if _, err := os.Stat(privValFile); err == nil {
-		privValidator := types.LoadPrivValidatorFS(privValFile)
-		privValidator.Reset()
-		logger.Info("Reset PrivValidator", "file", privValFile)
+// ResetAll removes the privValidator and address book files plus all data.
+// Exported so other CLI tools can use it.
+func ResetAll(dbDir, addrBookFile, privValFile string, logger log.Logger) {
+	resetFilePV(privValFile, logger)
+	removeAddrBook(addrBookFile, logger)
+	if err := os.RemoveAll(dbDir); err == nil {
+		logger.Info("Removed all blockchain history", "dir", dbDir)
 	} else {
-		privValidator := types.GenPrivValidatorFS(privValFile)
-		privValidator.Save()
-		logger.Info("Generated PrivValidator", "file", privValFile)
+		logger.Error("Error removing all blockchain history", "dir", dbDir, "err", err)
+	}
+}
+
+func resetFilePV(privValFile string, logger log.Logger) {
+	if _, err := os.Stat(privValFile); err == nil {
+		pv := privval.LoadFilePV(privValFile)
+		pv.Reset()
+		logger.Info("Reset private validator file to genesis state", "file", privValFile)
+	} else {
+		pv := privval.GenFilePV(privValFile)
+		pv.Save()
+		logger.Info("Generated private validator file", "file", privValFile)
+	}
+}
+
+func removeAddrBook(addrBookFile string, logger log.Logger) {
+	if err := os.Remove(addrBookFile); err == nil {
+		logger.Info("Removed existing address book", "file", addrBookFile)
+	} else if !os.IsNotExist(err) {
+		logger.Info("Error removing address book", "file", addrBookFile, "err", err)
 	}
 }

cmd/tendermint/commands/root.go

@@ -14,11 +14,15 @@ import (
 
 var (
 	config = cfg.DefaultConfig()
-	logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main")
+	logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
 )
 
 func init() {
-	RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level")
+	registerFlagsRootCmd(RootCmd)
+}
+
+func registerFlagsRootCmd(cmd *cobra.Command) {
+	cmd.PersistentFlags().String("log_level", config.LogLevel, "Log level")
 }
 
 // ParseConfig retrieves the default environment configuration,
@@ -53,6 +57,7 @@ var RootCmd = &cobra.Command{
 		if viper.GetBool(cli.TraceFlag) {
 			logger = log.NewTracingLogger(logger)
 		}
+		logger = logger.With("module", "main")
 		return nil
 	},
 }

cmd/tendermint/commands/root_test.go

@@ -1,7 +1,10 @@
 package commands
 
 import (
+	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"testing"
 
@@ -12,6 +15,7 @@ import (
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tmlibs/cli"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 var (
@@ -22,84 +26,151 @@ const (
 	rootName = "root"
 )
 
-// isolate provides a clean setup and returns a copy of RootCmd you can
-// modify in the test cases.
-// NOTE: it unsets all TM* env variables.
-func isolate(cmds ...*cobra.Command) cli.Executable {
-	os.Unsetenv("TMHOME")
-	os.Unsetenv("TM_HOME")
-	os.Unsetenv("TMROOT")
-	os.Unsetenv("TM_ROOT")
+// clearConfig clears env vars, the given root dir, and resets viper.
+func clearConfig(dir string) {
+	if err := os.Unsetenv("TMHOME"); err != nil {
+		panic(err)
+	}
+	if err := os.Unsetenv("TM_HOME"); err != nil {
+		panic(err)
+	}
 
+	if err := os.RemoveAll(dir); err != nil {
+		panic(err)
+	}
 	viper.Reset()
 	config = cfg.DefaultConfig()
-	r := &cobra.Command{
-		Use:               rootName,
-		PersistentPreRunE: RootCmd.PersistentPreRunE,
-	}
-	r.AddCommand(cmds...)
-	wr := cli.PrepareBaseCmd(r, "TM", defaultRoot)
-	return wr
 }
 
-func TestRootConfig(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	// we pre-create a config file we can refer to in the rest of
-	// the test cases.
-	cvals := map[string]string{
-		"moniker":   "monkey",
-		"fast_sync": "false",
+// prepare new rootCmd
+func testRootCmd() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:               RootCmd.Use,
+		PersistentPreRunE: RootCmd.PersistentPreRunE,
+		Run:               func(cmd *cobra.Command, args []string) {},
 	}
-	// proper types of the above settings
-	cfast := false
-	conf, err := cli.WriteDemoConfig(cvals)
-	require.Nil(err)
+	registerFlagsRootCmd(rootCmd)
+	var l string
+	rootCmd.PersistentFlags().String("log", l, "Log")
+	return rootCmd
+}
 
+func testSetup(rootDir string, args []string, env map[string]string) error {
+	clearConfig(defaultRoot)
+
+	rootCmd := testRootCmd()
+	cmd := cli.PrepareBaseCmd(rootCmd, "TM", defaultRoot)
+
+	// run with the args and env
+	args = append([]string{rootCmd.Use}, args...)
+	return cli.RunWithArgs(cmd, args, env)
+}
+
+func TestRootHome(t *testing.T) {
+	newRoot := filepath.Join(defaultRoot, "something-else")
+	cases := []struct {
+		args []string
+		env  map[string]string
+		root string
+	}{
+		{nil, nil, defaultRoot},
+		{[]string{"--home", newRoot}, nil, newRoot},
+		{nil, map[string]string{"TMHOME": newRoot}, newRoot},
+	}
+
+	for i, tc := range cases {
+		idxString := strconv.Itoa(i)
+
+		err := testSetup(defaultRoot, tc.args, tc.env)
+		require.Nil(t, err, idxString)
+
+		assert.Equal(t, tc.root, config.RootDir, idxString)
+		assert.Equal(t, tc.root, config.P2P.RootDir, idxString)
+		assert.Equal(t, tc.root, config.Consensus.RootDir, idxString)
+		assert.Equal(t, tc.root, config.Mempool.RootDir, idxString)
+	}
+}
+
+func TestRootFlagsEnv(t *testing.T) {
+
+	// defaults
 	defaults := cfg.DefaultConfig()
-	dmax := defaults.P2P.MaxNumPeers
+	defaultLogLvl := defaults.LogLevel
 
 	cases := []struct {
 		args     []string
 		env      map[string]string
-		root     string
-		moniker  string
-		fastSync bool
-		maxPeer  int
+		logLevel string
 	}{
-		{nil, nil, defaultRoot, defaults.Moniker, defaults.FastSync, dmax},
-		// try multiple ways of setting root (two flags, cli vs. env)
-		{[]string{"--home", conf}, nil, conf, cvals["moniker"], cfast, dmax},
-		{nil, map[string]string{"TMROOT": conf}, conf, cvals["moniker"], cfast, dmax},
-		// check setting p2p subflags two different ways
-		{[]string{"--p2p.max_num_peers", "420"}, nil, defaultRoot, defaults.Moniker, defaults.FastSync, 420},
-		{nil, map[string]string{"TM_P2P_MAX_NUM_PEERS": "17"}, defaultRoot, defaults.Moniker, defaults.FastSync, 17},
-		// try to set env that have no flags attached...
-		{[]string{"--home", conf}, map[string]string{"TM_MONIKER": "funny"}, conf, "funny", cfast, dmax},
+		{[]string{"--log", "debug"}, nil, defaultLogLvl},                 // wrong flag
+		{[]string{"--log_level", "debug"}, nil, "debug"},                 // right flag
+		{nil, map[string]string{"TM_LOW": "debug"}, defaultLogLvl},       // wrong env flag
+		{nil, map[string]string{"MT_LOG_LEVEL": "debug"}, defaultLogLvl}, // wrong env prefix
+		{nil, map[string]string{"TM_LOG_LEVEL": "debug"}, "debug"},       // right env
 	}
 
-	for idx, tc := range cases {
-		i := strconv.Itoa(idx)
-		// test command that does nothing, except trigger unmarshalling in root
-		noop := &cobra.Command{
-			Use: "noop",
-			RunE: func(cmd *cobra.Command, args []string) error {
-				return nil
-			},
-		}
-		noop.Flags().Int("p2p.max_num_peers", defaults.P2P.MaxNumPeers, "")
-		cmd := isolate(noop)
+	for i, tc := range cases {
+		idxString := strconv.Itoa(i)
 
-		args := append([]string{rootName, noop.Use}, tc.args...)
-		err := cli.RunWithArgs(cmd, args, tc.env)
-		require.Nil(err, i)
-		assert.Equal(tc.root, config.RootDir, i)
-		assert.Equal(tc.root, config.P2P.RootDir, i)
-		assert.Equal(tc.root, config.Consensus.RootDir, i)
-		assert.Equal(tc.root, config.Mempool.RootDir, i)
-		assert.Equal(tc.moniker, config.Moniker, i)
-		assert.Equal(tc.fastSync, config.FastSync, i)
-		assert.Equal(tc.maxPeer, config.P2P.MaxNumPeers, i)
+		err := testSetup(defaultRoot, tc.args, tc.env)
+		require.Nil(t, err, idxString)
+
+		assert.Equal(t, tc.logLevel, config.LogLevel, idxString)
 	}
-
+}
+
+func TestRootConfig(t *testing.T) {
+
+	// write non-default config
+	nonDefaultLogLvl := "abc:debug"
+	cvals := map[string]string{
+		"log_level": nonDefaultLogLvl,
+	}
+
+	cases := []struct {
+		args []string
+		env  map[string]string
+
+		logLvl string
+	}{
+		{nil, nil, nonDefaultLogLvl},                                     // should load config
+		{[]string{"--log_level=abc:info"}, nil, "abc:info"},              // flag over rides
+		{nil, map[string]string{"TM_LOG_LEVEL": "abc:info"}, "abc:info"}, // env over rides
+	}
+
+	for i, tc := range cases {
+		idxString := strconv.Itoa(i)
+		clearConfig(defaultRoot)
+
+		// XXX: path must match cfg.defaultConfigPath
+		configFilePath := filepath.Join(defaultRoot, "config")
+		err := cmn.EnsureDir(configFilePath, 0700)
+		require.Nil(t, err)
+
+		// write the non-defaults to a different path
+		// TODO: support writing sub configs so we can test that too
+		err = WriteConfigVals(configFilePath, cvals)
+		require.Nil(t, err)
+
+		rootCmd := testRootCmd()
+		cmd := cli.PrepareBaseCmd(rootCmd, "TM", defaultRoot)
+
+		// run with the args and env
+		tc.args = append([]string{rootCmd.Use}, tc.args...)
+		err = cli.RunWithArgs(cmd, tc.args, tc.env)
+		require.Nil(t, err, idxString)
+
+		assert.Equal(t, tc.logLvl, config.LogLevel, idxString)
+	}
+}
+
+// WriteConfigVals writes a toml file with the given values.
+// It returns an error if writing was impossible.
+func WriteConfigVals(dir string, vals map[string]string) error {
+	data := ""
+	for k, v := range vals {
+		data = data + fmt.Sprintf("%s = \"%s\"\n", k, v)
+	}
+	cfile := filepath.Join(dir, "config.toml")
+	return ioutil.WriteFile(cfile, []byte(data), 0666)
 }

cmd/tendermint/commands/run_node.go

@@ -14,11 +14,14 @@ func AddNodeFlags(cmd *cobra.Command) {
 	// bind flags
 	cmd.Flags().String("moniker", config.Moniker, "Node Name")
 
+	// priv val flags
+	cmd.Flags().String("priv_validator_laddr", config.PrivValidatorListenAddr, "Socket address to listen on for connections from external priv_validator process")
+
 	// node flags
 	cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing")
 
 	// abci flags
-	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.")
+	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.")
 	cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)")
 
 	// rpc flags
@@ -28,16 +31,19 @@ func AddNodeFlags(cmd *cobra.Command) {
 
 	// p2p flags
 	cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)")
-	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes")
+	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma-delimited ID@host:port seed nodes")
+	cmd.Flags().String("p2p.persistent_peers", config.P2P.PersistentPeers, "Comma-delimited ID@host:port persistent peers")
 	cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration")
-	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)")
+	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable/disable Peer-Exchange")
+	cmd.Flags().Bool("p2p.seed_mode", config.P2P.SeedMode, "Enable/disable seed mode")
+	cmd.Flags().String("p2p.private_peer_ids", config.P2P.PrivatePeerIDs, "Comma-delimited private peer IDs")
 
 	// consensus flags
 	cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes")
 }
 
-// NewRunNodeCmd returns the command that allows the CLI to start a
-// node. It can be used with a custom PrivValidator and in-process ABCI application.
+// NewRunNodeCmd returns the command that allows the CLI to start a node.
+// It can be used with a custom PrivValidator and in-process ABCI application.
 func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "node",
@@ -49,11 +55,10 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command {
 				return fmt.Errorf("Failed to create node: %v", err)
 			}
 
-			if _, err := n.Start(); err != nil {
+			if err := n.Start(); err != nil {
 				return fmt.Errorf("Failed to start node: %v", err)
-			} else {
-				logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
 			}
+			logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
 
 			// Trap signal, run forever.
 			n.RunForever()

cmd/tendermint/commands/show_node_id.go

@@ -0,0 +1,27 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/p2p"
+)
+
+// ShowNodeIDCmd dumps node's ID to the standard output.
+var ShowNodeIDCmd = &cobra.Command{
+	Use:   "show_node_id",
+	Short: "Show this node's ID",
+	RunE:  showNodeID,
+}
+
+func showNodeID(cmd *cobra.Command, args []string) error {
+
+	nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile())
+	if err != nil {
+		return err
+	}
+	fmt.Println(nodeKey.ID())
+
+	return nil
+}

cmd/tendermint/commands/show_validator.go

@@ -5,8 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/tendermint/go-wire/data"
-	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/privval"
 )
 
 // ShowValidatorCmd adds capabilities for showing the validator info.
@@ -17,7 +16,7 @@ var ShowValidatorCmd = &cobra.Command{
 }
 
 func showValidator(cmd *cobra.Command, args []string) {
-	privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile())
-	pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey)
+	privValidator := privval.LoadOrGenFilePV(config.PrivValidatorFile())
+	pubKeyJSONBytes, _ := cdc.MarshalJSON(privValidator.GetPubKey())
 	fmt.Println(string(pubKeyJSONBytes))
 }

cmd/tendermint/commands/testnet.go

@@ -2,57 +2,114 @@ package commands
 
 import (
 	"fmt"
-	"path"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 )
 
-//flags
 var (
-	nValidators int
-	dataDir     string
+	nValidators    int
+	nNonValidators int
+	outputDir      string
+	nodeDirPrefix  string
+
+	populatePersistentPeers bool
+	hostnamePrefix          string
+	startingIPAddress       string
+	p2pPort                 int
+)
+
+const (
+	nodeDirPerm = 0755
 )
 
 func init() {
-	TestnetFilesCmd.Flags().IntVar(&nValidators, "n", 4,
+	TestnetFilesCmd.Flags().IntVar(&nValidators, "v", 4,
 		"Number of validators to initialize the testnet with")
-	TestnetFilesCmd.Flags().StringVar(&dataDir, "dir", "mytestnet",
+	TestnetFilesCmd.Flags().IntVar(&nNonValidators, "n", 0,
+		"Number of non-validators to initialize the testnet with")
+	TestnetFilesCmd.Flags().StringVar(&outputDir, "o", "./mytestnet",
 		"Directory to store initialization data for the testnet")
+	TestnetFilesCmd.Flags().StringVar(&nodeDirPrefix, "node-dir-prefix", "node",
+		"Prefix the directory name for each node with (node results in node0, node1, ...)")
+
+	TestnetFilesCmd.Flags().BoolVar(&populatePersistentPeers, "populate-persistent-peers", true,
+		"Update config of each node with the list of persistent peers build using either hostname-prefix or starting-ip-address")
+	TestnetFilesCmd.Flags().StringVar(&hostnamePrefix, "hostname-prefix", "node",
+		"Hostname prefix (node results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
+	TestnetFilesCmd.Flags().StringVar(&startingIPAddress, "starting-ip-address", "",
+		"Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:26656, ID1@192.168.0.2:26656, ...)")
+	TestnetFilesCmd.Flags().IntVar(&p2pPort, "p2p-port", 26656,
+		"P2P Port")
 }
 
-// TestnetFilesCmd allows initialisation of files for a
-// Tendermint testnet.
+// TestnetFilesCmd allows initialisation of files for a Tendermint testnet.
 var TestnetFilesCmd = &cobra.Command{
 	Use:   "testnet",
 	Short: "Initialize files for a Tendermint testnet",
-	Run:   testnetFiles,
+	Long: `testnet will create "v" + "n" number of directories and populate each with
+necessary files (private validator, genesis, config, etc.).
+
+Note, strict routability for addresses is turned off in the config file.
+
+Optionally, it will fill in persistent_peers list in config file using either hostnames or IPs.
+
+Example:
+
+	tendermint testnet --v 4 --o ./output --populate-persistent-peers --starting-ip-address 192.168.10.2
+	`,
+	RunE: testnetFiles,
 }
 
-func testnetFiles(cmd *cobra.Command, args []string) {
-
+func testnetFiles(cmd *cobra.Command, args []string) error {
+	config := cfg.DefaultConfig()
 	genVals := make([]types.GenesisValidator, nValidators)
 
-	// Initialize core dir and priv_validator.json's
 	for i := 0; i < nValidators; i++ {
-		mach := cmn.Fmt("mach%d", i)
-		err := initMachCoreDirectory(dataDir, mach)
+		nodeDirName := cmn.Fmt("%s%d", nodeDirPrefix, i)
+		nodeDir := filepath.Join(outputDir, nodeDirName)
+		config.SetRoot(nodeDir)
+
+		err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm)
 		if err != nil {
-			cmn.Exit(err.Error())
+			_ = os.RemoveAll(outputDir)
+			return err
 		}
-		// Read priv_validator.json to populate vals
-		privValFile := path.Join(dataDir, mach, "priv_validator.json")
-		privVal := types.LoadPrivValidatorFS(privValFile)
+
+		initFilesWithConfig(config)
+
+		pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator)
+		pv := privval.LoadFilePV(pvFile)
 		genVals[i] = types.GenesisValidator{
-			PubKey: privVal.GetPubKey(),
+			PubKey: pv.GetPubKey(),
 			Power:  1,
-			Name:   mach,
+			Name:   nodeDirName,
 		}
 	}
 
+	for i := 0; i < nNonValidators; i++ {
+		nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i+nValidators))
+		config.SetRoot(nodeDir)
+
+		err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm)
+		if err != nil {
+			_ = os.RemoveAll(outputDir)
+			return err
+		}
+
+		initFilesWithConfig(config)
+	}
+
 	// Generate genesis doc from generated validators
 	genDoc := &types.GenesisDoc{
 		GenesisTime: time.Now(),
@@ -61,32 +118,66 @@ func testnetFiles(cmd *cobra.Command, args []string) {
 	}
 
 	// Write genesis file.
-	for i := 0; i < nValidators; i++ {
-		mach := cmn.Fmt("mach%d", i)
-		genDoc.SaveAs(path.Join(dataDir, mach, "genesis.json"))
+	for i := 0; i < nValidators+nNonValidators; i++ {
+		nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i))
+		if err := genDoc.SaveAs(filepath.Join(nodeDir, config.BaseConfig.Genesis)); err != nil {
+			_ = os.RemoveAll(outputDir)
+			return err
+		}
 	}
 
-	fmt.Println(cmn.Fmt("Successfully initialized %v node directories", nValidators))
-}
-
-// Initialize per-machine core directory
-func initMachCoreDirectory(base, mach string) error {
-	dir := path.Join(base, mach)
-	err := cmn.EnsureDir(dir, 0777)
-	if err != nil {
-		return err
+	if populatePersistentPeers {
+		err := populatePersistentPeersInConfigAndWriteIt(config)
+		if err != nil {
+			_ = os.RemoveAll(outputDir)
+			return err
+		}
 	}
 
-	// Create priv_validator.json file if not present
-	ensurePrivValidator(path.Join(dir, "priv_validator.json"))
+	fmt.Printf("Successfully initialized %v node directories\n", nValidators+nNonValidators)
 	return nil
-
 }
 
-func ensurePrivValidator(file string) {
-	if cmn.FileExists(file) {
-		return
+func hostnameOrIP(i int) string {
+	if startingIPAddress != "" {
+		ip := net.ParseIP(startingIPAddress)
+		ip = ip.To4()
+		if ip == nil {
+			fmt.Printf("%v: non ipv4 address\n", startingIPAddress)
+			os.Exit(1)
+		}
+
+		for j := 0; j < i; j++ {
+			ip[3]++
+		}
+		return ip.String()
 	}
-	privValidator := types.GenPrivValidatorFS(file)
-	privValidator.Save()
+
+	return fmt.Sprintf("%s%d", hostnamePrefix, i)
+}
+
+func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error {
+	persistentPeers := make([]string, nValidators+nNonValidators)
+	for i := 0; i < nValidators+nNonValidators; i++ {
+		nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i))
+		config.SetRoot(nodeDir)
+		nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile())
+		if err != nil {
+			return err
+		}
+		persistentPeers[i] = p2p.IDAddressString(nodeKey.ID(), fmt.Sprintf("%s:%d", hostnameOrIP(i), p2pPort))
+	}
+	persistentPeersList := strings.Join(persistentPeers, ",")
+
+	for i := 0; i < nValidators+nNonValidators; i++ {
+		nodeDir := filepath.Join(outputDir, cmn.Fmt("%s%d", nodeDirPrefix, i))
+		config.SetRoot(nodeDir)
+		config.P2P.PersistentPeers = persistentPeersList
+		config.P2P.AddrBookStrict = false
+
+		// overwrite default config
+		cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config)
+	}
+
+	return nil
 }

cmd/tendermint/commands/wire.go

@@ -0,0 +1,12 @@
+package commands
+
+import (
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/go-crypto"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	crypto.RegisterAmino(cdc)
+}

cmd/tendermint/main.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/tendermint/tmlibs/cli"
 
 	cmd "github.com/tendermint/tendermint/cmd/tendermint/commands"
+	cfg "github.com/tendermint/tendermint/config"
 	nm "github.com/tendermint/tendermint/node"
 )
 
@@ -15,12 +17,15 @@ func main() {
 		cmd.GenValidatorCmd,
 		cmd.InitFilesCmd,
 		cmd.ProbeUpnpCmd,
+		cmd.LiteCmd,
 		cmd.ReplayCmd,
 		cmd.ReplayConsoleCmd,
 		cmd.ResetAllCmd,
 		cmd.ResetPrivValidatorCmd,
 		cmd.ShowValidatorCmd,
 		cmd.TestnetFilesCmd,
+		cmd.ShowNodeIDCmd,
+		cmd.GenNodeKeyCmd,
 		cmd.VersionCmd)
 
 	// NOTE:
@@ -36,6 +41,8 @@ func main() {
 	// Create & start node
 	rootCmd.AddCommand(cmd.NewRunNodeCmd(nodeFunc))
 
-	cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint"))
-	cmd.Execute()
+	cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv(filepath.Join("$HOME", cfg.DefaultTendermintDir)))
+	if err := cmd.Execute(); err != nil {
+		panic(err)
+	}
 }

codecov.yml

@@ -0,0 +1,23 @@
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+
+  status:
+    project:
+      default:
+        threshold: 1%
+    patch: on
+    changes: off
+
+comment:
+  layout: "diff, files"
+  behavior: default
+  require_changes: no
+  require_base: no
+  require_head: yes
+
+ignore:
+  - "docs"
+  - "DOCKER"
+  - "scripts"

config/config.go

@@ -2,10 +2,43 @@ package config
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 )
 
+const (
+	// FuzzModeDrop is a mode in which we randomly drop reads/writes, connections or sleep
+	FuzzModeDrop = iota
+	// FuzzModeDelay is a mode in which we randomly sleep
+	FuzzModeDelay
+)
+
+// NOTE: Most of the structs & relevant comments + the
+// default configuration options were used to manually
+// generate the config.toml. Please reflect any changes
+// made here in the defaultConfigTemplate constant in
+// config/toml.go
+// NOTE: tmlibs/cli must know to look in the config dir!
+var (
+	DefaultTendermintDir = ".tendermint"
+	defaultConfigDir     = "config"
+	defaultDataDir       = "data"
+
+	defaultConfigFileName  = "config.toml"
+	defaultGenesisJSONName = "genesis.json"
+
+	defaultPrivValName  = "priv_validator.json"
+	defaultNodeKeyName  = "node_key.json"
+	defaultAddrBookName = "addrbook.json"
+
+	defaultConfigFilePath  = filepath.Join(defaultConfigDir, defaultConfigFileName)
+	defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName)
+	defaultPrivValPath     = filepath.Join(defaultConfigDir, defaultPrivValName)
+	defaultNodeKeyPath     = filepath.Join(defaultConfigDir, defaultNodeKeyName)
+	defaultAddrBookPath    = filepath.Join(defaultConfigDir, defaultAddrBookName)
+)
+
 // Config defines the top level configuration for a Tendermint node
 type Config struct {
 	// Top level options use an anonymous struct
@@ -16,6 +49,7 @@ type Config struct {
 	P2P       *P2PConfig       `mapstructure:"p2p"`
 	Mempool   *MempoolConfig   `mapstructure:"mempool"`
 	Consensus *ConsensusConfig `mapstructure:"consensus"`
+	TxIndex   *TxIndexConfig   `mapstructure:"tx_index"`
 }
 
 // DefaultConfig returns a default configuration for a Tendermint node
@@ -26,6 +60,7 @@ func DefaultConfig() *Config {
 		P2P:        DefaultP2PConfig(),
 		Mempool:    DefaultMempoolConfig(),
 		Consensus:  DefaultConsensusConfig(),
+		TxIndex:    DefaultTxIndexConfig(),
 	}
 }
 
@@ -35,8 +70,9 @@ func TestConfig() *Config {
 		BaseConfig: TestBaseConfig(),
 		RPC:        TestRPCConfig(),
 		P2P:        TestP2PConfig(),
-		Mempool:    DefaultMempoolConfig(),
+		Mempool:    TestMempoolConfig(),
 		Consensus:  TestConsensusConfig(),
+		TxIndex:    TestTxIndexConfig(),
 	}
 }
 
@@ -55,22 +91,30 @@ func (cfg *Config) SetRoot(root string) *Config {
 
 // BaseConfig defines the base configuration for a Tendermint node
 type BaseConfig struct {
+
+	// chainID is unexposed and immutable but here for convenience
+	chainID string
+
 	// The root directory for all data.
 	// This should be set in viper so it can unmarshal into this struct
 	RootDir string `mapstructure:"home"`
 
-	// The ID of the chain to join (should be signed with every transaction and vote)
-	ChainID string `mapstructure:"chain_id"`
-
-	// A JSON file containing the initial validator set and other meta data
+	// Path to the JSON file containing the initial validator set and other meta data
 	Genesis string `mapstructure:"genesis_file"`
 
-	// A JSON file containing the private key to use as a validator in the consensus protocol
+	// Path to the JSON file containing the private key to use as a validator in the consensus protocol
 	PrivValidator string `mapstructure:"priv_validator_file"`
 
+	// A JSON file containing the private key to use for p2p authenticated encryption
+	NodeKey string `mapstructure:"node_key_file"`
+
 	// A custom human readable name for this node
 	Moniker string `mapstructure:"moniker"`
 
+	// TCP or UNIX socket address for Tendermint to listen on for
+	// connections from an external PrivValidator process
+	PrivValidatorListenAddr string `mapstructure:"priv_validator_laddr"`
+
 	// TCP or UNIX socket address of the ABCI application,
 	// or the name of an ABCI application compiled in with the Tendermint binary
 	ProxyApp string `mapstructure:"proxy_app"`
@@ -93,9 +137,6 @@ type BaseConfig struct {
 	// so the app can decide if we should keep the connection or not
 	FilterPeers bool `mapstructure:"filter_peers"` // false
 
-	// What indexer to use for transactions
-	TxIndex string `mapstructure:"tx_index"`
-
 	// Database backend: leveldb | memdb
 	DBBackend string `mapstructure:"db_backend"`
 
@@ -106,16 +147,16 @@ type BaseConfig struct {
 // DefaultBaseConfig returns a default base configuration for a Tendermint node
 func DefaultBaseConfig() BaseConfig {
 	return BaseConfig{
-		Genesis:           "genesis.json",
-		PrivValidator:     "priv_validator.json",
-		Moniker:           "anonymous",
-		ProxyApp:          "tcp://127.0.0.1:46658",
+		Genesis:           defaultGenesisJSONPath,
+		PrivValidator:     defaultPrivValPath,
+		NodeKey:           defaultNodeKeyPath,
+		Moniker:           defaultMoniker,
+		ProxyApp:          "tcp://127.0.0.1:26658",
 		ABCI:              "socket",
 		LogLevel:          DefaultPackageLogLevels(),
 		ProfListenAddress: "",
 		FastSync:          true,
 		FilterPeers:       false,
-		TxIndex:           "kv",
 		DBBackend:         "leveldb",
 		DBPath:            "data",
 	}
@@ -123,27 +164,36 @@ func DefaultBaseConfig() BaseConfig {
 
 // TestBaseConfig returns a base configuration for testing a Tendermint node
 func TestBaseConfig() BaseConfig {
-	conf := DefaultBaseConfig()
-	conf.ChainID = "tendermint_test"
-	conf.ProxyApp = "dummy"
-	conf.FastSync = false
-	conf.DBBackend = "memdb"
-	return conf
+	cfg := DefaultBaseConfig()
+	cfg.chainID = "tendermint_test"
+	cfg.ProxyApp = "kvstore"
+	cfg.FastSync = false
+	cfg.DBBackend = "memdb"
+	return cfg
+}
+
+func (cfg BaseConfig) ChainID() string {
+	return cfg.chainID
 }
 
 // GenesisFile returns the full path to the genesis.json file
-func (b BaseConfig) GenesisFile() string {
-	return rootify(b.Genesis, b.RootDir)
+func (cfg BaseConfig) GenesisFile() string {
+	return rootify(cfg.Genesis, cfg.RootDir)
 }
 
 // PrivValidatorFile returns the full path to the priv_validator.json file
-func (b BaseConfig) PrivValidatorFile() string {
-	return rootify(b.PrivValidator, b.RootDir)
+func (cfg BaseConfig) PrivValidatorFile() string {
+	return rootify(cfg.PrivValidator, cfg.RootDir)
+}
+
+// NodeKeyFile returns the full path to the node_key.json file
+func (cfg BaseConfig) NodeKeyFile() string {
+	return rootify(cfg.NodeKey, cfg.RootDir)
 }
 
 // DBDir returns the full path to the database directory
-func (b BaseConfig) DBDir() string {
-	return rootify(b.DBPath, b.RootDir)
+func (cfg BaseConfig) DBDir() string {
+	return rootify(cfg.DBPath, cfg.RootDir)
 }
 
 // DefaultLogLevel returns a default log level of "error"
@@ -151,9 +201,10 @@ func DefaultLogLevel() string {
 	return "error"
 }
 
-// DefaultPackageLogLevels returns a default log level setting so all packages log at "error", while the `state` package logs at "info"
+// DefaultPackageLogLevels returns a default log level setting so all packages
+// log at "error", while the `state` and `main` packages log at "info"
 func DefaultPackageLogLevels() string {
-	return fmt.Sprintf("state:info,*:%s", DefaultLogLevel())
+	return fmt.Sprintf("main:info,state:info,*:%s", DefaultLogLevel())
 }
 
 //-----------------------------------------------------------------------------
@@ -170,14 +221,14 @@ type RPCConfig struct {
 	// NOTE: This server only supports /broadcast_tx_commit
 	GRPCListenAddress string `mapstructure:"grpc_laddr"`
 
-	// Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
+	// Activate unsafe RPC commands like /dial_persistent_peers and /unsafe_flush_mempool
 	Unsafe bool `mapstructure:"unsafe"`
 }
 
 // DefaultRPCConfig returns a default configuration for the RPC server
 func DefaultRPCConfig() *RPCConfig {
 	return &RPCConfig{
-		ListenAddress:     "tcp://0.0.0.0:46657",
+		ListenAddress:     "tcp://0.0.0.0:26657",
 		GRPCListenAddress: "",
 		Unsafe:            false,
 	}
@@ -185,11 +236,11 @@ func DefaultRPCConfig() *RPCConfig {
 
 // TestRPCConfig returns a configuration for testing the RPC server
 func TestRPCConfig() *RPCConfig {
-	conf := DefaultRPCConfig()
-	conf.ListenAddress = "tcp://0.0.0.0:36657"
-	conf.GRPCListenAddress = "tcp://0.0.0.0:36658"
-	conf.Unsafe = true
-	return conf
+	cfg := DefaultRPCConfig()
+	cfg.ListenAddress = "tcp://0.0.0.0:36657"
+	cfg.GRPCListenAddress = "tcp://0.0.0.0:36658"
+	cfg.Unsafe = true
+	return cfg
 }
 
 //-----------------------------------------------------------------------------
@@ -203,8 +254,13 @@ type P2PConfig struct {
 	ListenAddress string `mapstructure:"laddr"`
 
 	// Comma separated list of seed nodes to connect to
+	// We only use these if we can’t connect to peers in the addrbook
 	Seeds string `mapstructure:"seeds"`
 
+	// Comma separated list of nodes to keep persistent connections to
+	// Do not add private peers to this list if you don't want them advertised
+	PersistentPeers string `mapstructure:"persistent_peers"`
+
 	// Skip UPNP port forwarding
 	SkipUPNP bool `mapstructure:"skip_upnp"`
 
@@ -214,9 +270,6 @@ type P2PConfig struct {
 	// Set true for strict address routability rules
 	AddrBookStrict bool `mapstructure:"addr_book_strict"`
 
-	// Set true to enable the peer-exchange reactor
-	PexReactor bool `mapstructure:"pex"`
-
 	// Maximum number of peers to connect to
 	MaxNumPeers int `mapstructure:"max_num_peers"`
 
@@ -224,40 +277,97 @@ type P2PConfig struct {
 	FlushThrottleTimeout int `mapstructure:"flush_throttle_timeout"`
 
 	// Maximum size of a message packet payload, in bytes
-	MaxMsgPacketPayloadSize int `mapstructure:"max_msg_packet_payload_size"`
+	MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"`
 
 	// Rate at which packets can be sent, in bytes/second
 	SendRate int64 `mapstructure:"send_rate"`
 
 	// Rate at which packets can be received, in bytes/second
 	RecvRate int64 `mapstructure:"recv_rate"`
+
+	// Set true to enable the peer-exchange reactor
+	PexReactor bool `mapstructure:"pex"`
+
+	// Seed mode, in which node constantly crawls the network and looks for
+	// peers. If another node asks it for addresses, it responds and disconnects.
+	//
+	// Does not work if the peer-exchange reactor is disabled.
+	SeedMode bool `mapstructure:"seed_mode"`
+
+	// Comma separated list of peer IDs to keep private (will not be gossiped to
+	// other peers)
+	PrivatePeerIDs string `mapstructure:"private_peer_ids"`
+
+	// Toggle to disable guard against peers connecting from the same ip.
+	AllowDuplicateIP bool `mapstructure:"allow_duplicate_ip"`
+
+	// Peer connection configuration.
+	HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
+	DialTimeout      time.Duration `mapstructure:"dial_timeout"`
+
+	// Testing params.
+	// Force dial to fail
+	TestDialFail bool `mapstructure:"test_dial_fail"`
+	// FUzz connection
+	TestFuzz       bool            `mapstructure:"test_fuzz"`
+	TestFuzzConfig *FuzzConnConfig `mapstructure:"test_fuzz_config"`
 }
 
 // DefaultP2PConfig returns a default configuration for the peer-to-peer layer
 func DefaultP2PConfig() *P2PConfig {
 	return &P2PConfig{
-		ListenAddress:           "tcp://0.0.0.0:46656",
-		AddrBook:                "addrbook.json",
+		ListenAddress:           "tcp://0.0.0.0:26656",
+		AddrBook:                defaultAddrBookPath,
 		AddrBookStrict:          true,
 		MaxNumPeers:             50,
 		FlushThrottleTimeout:    100,
-		MaxMsgPacketPayloadSize: 1024,   // 1 kB
+		MaxPacketMsgPayloadSize: 1024,   // 1 kB
 		SendRate:                512000, // 500 kB/s
 		RecvRate:                512000, // 500 kB/s
+		PexReactor:              true,
+		SeedMode:                false,
+		AllowDuplicateIP:        true, // so non-breaking yet
+		HandshakeTimeout:        20 * time.Second,
+		DialTimeout:             3 * time.Second,
+		TestDialFail:            false,
+		TestFuzz:                false,
+		TestFuzzConfig:          DefaultFuzzConnConfig(),
 	}
 }
 
 // TestP2PConfig returns a configuration for testing the peer-to-peer layer
 func TestP2PConfig() *P2PConfig {
-	conf := DefaultP2PConfig()
-	conf.ListenAddress = "tcp://0.0.0.0:36656"
-	conf.SkipUPNP = true
-	return conf
+	cfg := DefaultP2PConfig()
+	cfg.ListenAddress = "tcp://0.0.0.0:36656"
+	cfg.SkipUPNP = true
+	cfg.FlushThrottleTimeout = 10
+	cfg.AllowDuplicateIP = true
+	return cfg
 }
 
-// AddrBookFile returns the full path to the address bool
-func (p *P2PConfig) AddrBookFile() string {
-	return rootify(p.AddrBook, p.RootDir)
+// AddrBookFile returns the full path to the address book
+func (cfg *P2PConfig) AddrBookFile() string {
+	return rootify(cfg.AddrBook, cfg.RootDir)
+}
+
+// FuzzConnConfig is a FuzzedConnection configuration.
+type FuzzConnConfig struct {
+	Mode         int
+	MaxDelay     time.Duration
+	ProbDropRW   float64
+	ProbDropConn float64
+	ProbSleep    float64
+}
+
+// DefaultFuzzConnConfig returns the default config.
+func DefaultFuzzConnConfig() *FuzzConnConfig {
+	return &FuzzConnConfig{
+		Mode:         FuzzModeDrop,
+		MaxDelay:     3 * time.Second,
+		ProbDropRW:   0.2,
+		ProbDropConn: 0.00,
+		ProbSleep:    0.00,
+	}
 }
 
 //-----------------------------------------------------------------------------
@@ -270,6 +380,8 @@ type MempoolConfig struct {
 	RecheckEmpty bool   `mapstructure:"recheck_empty"`
 	Broadcast    bool   `mapstructure:"broadcast"`
 	WalPath      string `mapstructure:"wal_dir"`
+	Size         int    `mapstructure:"size"`
+	CacheSize    int    `mapstructure:"cache_size"`
 }
 
 // DefaultMempoolConfig returns a default configuration for the Tendermint mempool
@@ -278,13 +390,22 @@ func DefaultMempoolConfig() *MempoolConfig {
 		Recheck:      true,
 		RecheckEmpty: true,
 		Broadcast:    true,
-		WalPath:      "data/mempool.wal",
+		WalPath:      filepath.Join(defaultDataDir, "mempool.wal"),
+		Size:         100000,
+		CacheSize:    100000,
 	}
 }
 
+// TestMempoolConfig returns a configuration for testing the Tendermint mempool
+func TestMempoolConfig() *MempoolConfig {
+	cfg := DefaultMempoolConfig()
+	cfg.CacheSize = 1000
+	return cfg
+}
+
 // WalDir returns the full path to the mempool's write-ahead log
-func (m *MempoolConfig) WalDir() string {
-	return rootify(m.WalPath, m.RootDir)
+func (cfg *MempoolConfig) WalDir() string {
+	return rootify(cfg.WalPath, cfg.RootDir)
 }
 
 //-----------------------------------------------------------------------------
@@ -293,12 +414,11 @@ func (m *MempoolConfig) WalDir() string {
 // ConsensusConfig defines the confuguration for the Tendermint consensus service,
 // including timeouts and details about the WAL and the block structure.
 type ConsensusConfig struct {
-	RootDir  string `mapstructure:"home"`
-	WalPath  string `mapstructure:"wal_file"`
-	WalLight bool   `mapstructure:"wal_light"`
-	walFile  string // overrides WalPath if set
+	RootDir string `mapstructure:"home"`
+	WalPath string `mapstructure:"wal_file"`
+	walFile string // overrides WalPath if set
 
-	// All timeouts are in ms
+	// All timeouts are in milliseconds
 	TimeoutPropose        int `mapstructure:"timeout_propose"`
 	TimeoutProposeDelta   int `mapstructure:"timeout_propose_delta"`
 	TimeoutPrevote        int `mapstructure:"timeout_prevote"`
@@ -318,11 +438,48 @@ type ConsensusConfig struct {
 	CreateEmptyBlocks         bool `mapstructure:"create_empty_blocks"`
 	CreateEmptyBlocksInterval int  `mapstructure:"create_empty_blocks_interval"`
 
-	// Reactor sleep duration parameters are in ms
+	// Reactor sleep duration parameters are in milliseconds
 	PeerGossipSleepDuration     int `mapstructure:"peer_gossip_sleep_duration"`
 	PeerQueryMaj23SleepDuration int `mapstructure:"peer_query_maj23_sleep_duration"`
 }
 
+// DefaultConsensusConfig returns a default configuration for the consensus service
+func DefaultConsensusConfig() *ConsensusConfig {
+	return &ConsensusConfig{
+		WalPath:                     filepath.Join(defaultDataDir, "cs.wal", "wal"),
+		TimeoutPropose:              3000,
+		TimeoutProposeDelta:         500,
+		TimeoutPrevote:              1000,
+		TimeoutPrevoteDelta:         500,
+		TimeoutPrecommit:            1000,
+		TimeoutPrecommitDelta:       500,
+		TimeoutCommit:               1000,
+		SkipTimeoutCommit:           false,
+		MaxBlockSizeTxs:             10000,
+		MaxBlockSizeBytes:           1, // TODO
+		CreateEmptyBlocks:           true,
+		CreateEmptyBlocksInterval:   0,
+		PeerGossipSleepDuration:     100,
+		PeerQueryMaj23SleepDuration: 2000,
+	}
+}
+
+// TestConsensusConfig returns a configuration for testing the consensus service
+func TestConsensusConfig() *ConsensusConfig {
+	cfg := DefaultConsensusConfig()
+	cfg.TimeoutPropose = 100
+	cfg.TimeoutProposeDelta = 1
+	cfg.TimeoutPrevote = 10
+	cfg.TimeoutPrevoteDelta = 1
+	cfg.TimeoutPrecommit = 10
+	cfg.TimeoutPrecommitDelta = 1
+	cfg.TimeoutCommit = 10
+	cfg.SkipTimeoutCommit = true
+	cfg.PeerGossipSleepDuration = 5
+	cfg.PeerQueryMaj23SleepDuration = 250
+	return cfg
+}
+
 // WaitForTxs returns true if the consensus should wait for transactions before entering the propose step
 func (cfg *ConsensusConfig) WaitForTxs() bool {
 	return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0
@@ -363,53 +520,57 @@ func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration {
 	return time.Duration(cfg.PeerQueryMaj23SleepDuration) * time.Millisecond
 }
 
-// DefaultConsensusConfig returns a default configuration for the consensus service
-func DefaultConsensusConfig() *ConsensusConfig {
-	return &ConsensusConfig{
-		WalPath:                     "data/cs.wal/wal",
-		WalLight:                    false,
-		TimeoutPropose:              3000,
-		TimeoutProposeDelta:         500,
-		TimeoutPrevote:              1000,
-		TimeoutPrevoteDelta:         500,
-		TimeoutPrecommit:            1000,
-		TimeoutPrecommitDelta:       500,
-		TimeoutCommit:               1000,
-		SkipTimeoutCommit:           false,
-		MaxBlockSizeTxs:             10000,
-		MaxBlockSizeBytes:           1, // TODO
-		CreateEmptyBlocks:           true,
-		CreateEmptyBlocksInterval:   0,
-		PeerGossipSleepDuration:     100,
-		PeerQueryMaj23SleepDuration: 2000,
-	}
-}
-
-// TestConsensusConfig returns a configuration for testing the consensus service
-func TestConsensusConfig() *ConsensusConfig {
-	config := DefaultConsensusConfig()
-	config.TimeoutPropose = 2000
-	config.TimeoutProposeDelta = 1
-	config.TimeoutPrevote = 10
-	config.TimeoutPrevoteDelta = 1
-	config.TimeoutPrecommit = 10
-	config.TimeoutPrecommitDelta = 1
-	config.TimeoutCommit = 10
-	config.SkipTimeoutCommit = true
-	return config
-}
-
 // WalFile returns the full path to the write-ahead log file
-func (c *ConsensusConfig) WalFile() string {
-	if c.walFile != "" {
-		return c.walFile
+func (cfg *ConsensusConfig) WalFile() string {
+	if cfg.walFile != "" {
+		return cfg.walFile
 	}
-	return rootify(c.WalPath, c.RootDir)
+	return rootify(cfg.WalPath, cfg.RootDir)
 }
 
 // SetWalFile sets the path to the write-ahead log file
-func (c *ConsensusConfig) SetWalFile(walFile string) {
-	c.walFile = walFile
+func (cfg *ConsensusConfig) SetWalFile(walFile string) {
+	cfg.walFile = walFile
+}
+
+//-----------------------------------------------------------------------------
+// TxIndexConfig
+
+// TxIndexConfig defines the confuguration for the transaction
+// indexer, including tags to index.
+type TxIndexConfig struct {
+	// What indexer to use for transactions
+	//
+	// Options:
+	//   1) "null" (default)
+	//   2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
+	Indexer string `mapstructure:"indexer"`
+
+	// Comma-separated list of tags to index (by default the only tag is tx hash)
+	//
+	// It's recommended to index only a subset of tags due to possible memory
+	// bloat. This is, of course, depends on the indexer's DB and the volume of
+	// transactions.
+	IndexTags string `mapstructure:"index_tags"`
+
+	// When set to true, tells indexer to index all tags. Note this may be not
+	// desirable (see the comment above). IndexTags has a precedence over
+	// IndexAllTags (i.e. when given both, IndexTags will be indexed).
+	IndexAllTags bool `mapstructure:"index_all_tags"`
+}
+
+// DefaultTxIndexConfig returns a default configuration for the transaction indexer.
+func DefaultTxIndexConfig() *TxIndexConfig {
+	return &TxIndexConfig{
+		Indexer:      "kv",
+		IndexTags:    "",
+		IndexAllTags: false,
+	}
+}
+
+// TestTxIndexConfig returns a default configuration for the transaction indexer.
+func TestTxIndexConfig() *TxIndexConfig {
+	return DefaultTxIndexConfig()
 }
 
 //-----------------------------------------------------------------------------
@@ -422,3 +583,18 @@ func rootify(path, root string) string {
 	}
 	return filepath.Join(root, path)
 }
+
+//-----------------------------------------------------------------------------
+// Moniker
+
+var defaultMoniker = getDefaultMoniker()
+
+// getDefaultMoniker returns a default moniker, which is the host name. If runtime
+// fails to get the host name, "anonymous" will be returned.
+func getDefaultMoniker() string {
+	moniker, err := os.Hostname()
+	if err != nil {
+		moniker = "anonymous"
+	}
+	return moniker
+}

config/toml.go

@@ -1,50 +1,238 @@
 package config
 
 import (
+	"bytes"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
+	"text/template"
 
 	cmn "github.com/tendermint/tmlibs/common"
 )
 
-/****** these are for production settings ***********/
+var configTemplate *template.Template
 
-func EnsureRoot(rootDir string) {
-	cmn.EnsureDir(rootDir, 0700)
-	cmn.EnsureDir(rootDir+"/data", 0700)
-
-	configFilePath := path.Join(rootDir, "config.toml")
-
-	// Write default config file if missing.
-	if !cmn.FileExists(configFilePath) {
-		// Ask user for moniker
-		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
-		cmn.MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
+func init() {
+	var err error
+	if configTemplate, err = template.New("configFileTemplate").Parse(defaultConfigTemplate); err != nil {
+		panic(err)
 	}
 }
 
-var defaultConfigTmpl = `# This is a TOML config file.
+/****** these are for production settings ***********/
+
+// EnsureRoot creates the root, config, and data directories if they don't exist,
+// and panics if it fails.
+func EnsureRoot(rootDir string) {
+	if err := cmn.EnsureDir(rootDir, 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+
+	configFilePath := filepath.Join(rootDir, defaultConfigFilePath)
+
+	// Write default config file if missing.
+	if !cmn.FileExists(configFilePath) {
+		writeDefaultConfigFile(configFilePath)
+	}
+}
+
+// XXX: this func should probably be called by cmd/tendermint/commands/init.go
+// alongside the writing of the genesis.json and priv_validator.json
+func writeDefaultConfigFile(configFilePath string) {
+	WriteConfigFile(configFilePath, DefaultConfig())
+}
+
+// WriteConfigFile renders config using the template and writes it to configFilePath.
+func WriteConfigFile(configFilePath string, config *Config) {
+	var buffer bytes.Buffer
+
+	if err := configTemplate.Execute(&buffer, config); err != nil {
+		panic(err)
+	}
+
+	cmn.MustWriteFile(configFilePath, buffer.Bytes(), 0644)
+}
+
+// Note: any changes to the comments/variables/mapstructure
+// must be reflected in the appropriate struct in config/config.go
+const defaultConfigTemplate = `# This is a TOML config file.
 # For more information, see https://github.com/toml-lang/toml
 
-proxy_app = "tcp://127.0.0.1:46658"
-moniker = "__MONIKER__"
-fast_sync = true
-db_backend = "leveldb"
-log_level = "state:info,*:error"
+##### main base config options #####
 
+# TCP or UNIX socket address of the ABCI application,
+# or the name of an ABCI application compiled in with the Tendermint binary
+proxy_app = "{{ .BaseConfig.ProxyApp }}"
+
+# A custom human readable name for this node
+moniker = "{{ .BaseConfig.Moniker }}"
+
+# If this node is many blocks behind the tip of the chain, FastSync
+# allows them to catchup quickly by downloading blocks in parallel
+# and verifying their commits
+fast_sync = {{ .BaseConfig.FastSync }}
+
+# Database backend: leveldb | memdb
+db_backend = "{{ .BaseConfig.DBBackend }}"
+
+# Database directory
+db_path = "{{ js .BaseConfig.DBPath }}"
+
+# Output level for logging, including package level options
+log_level = "{{ .BaseConfig.LogLevel }}"
+
+##### additional base config options #####
+
+# Path to the JSON file containing the initial validator set and other meta data
+genesis_file = "{{ js .BaseConfig.Genesis }}"
+
+# Path to the JSON file containing the private key to use as a validator in the consensus protocol
+priv_validator_file = "{{ js .BaseConfig.PrivValidator }}"
+
+# Path to the JSON file containing the private key to use for node authentication in the p2p protocol
+node_key_file = "{{ js .BaseConfig.NodeKey}}"
+
+# Mechanism to connect to the ABCI application: socket | grpc
+abci = "{{ .BaseConfig.ABCI }}"
+
+# TCP or UNIX socket address for the profiling server to listen on
+prof_laddr = "{{ .BaseConfig.ProfListenAddress }}"
+
+# If true, query the ABCI app on connecting to a new peer
+# so the app can decide if we should keep the connection or not
+filter_peers = {{ .BaseConfig.FilterPeers }}
+
+##### advanced configuration options #####
+
+##### rpc server configuration options #####
 [rpc]
-laddr = "tcp://0.0.0.0:46657"
 
+# TCP or UNIX socket address for the RPC server to listen on
+laddr = "{{ .RPC.ListenAddress }}"
+
+# TCP or UNIX socket address for the gRPC server to listen on
+# NOTE: This server only supports /broadcast_tx_commit
+grpc_laddr = "{{ .RPC.GRPCListenAddress }}"
+
+# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
+unsafe = {{ .RPC.Unsafe }}
+
+##### peer to peer configuration options #####
 [p2p]
-laddr = "tcp://0.0.0.0:46656"
-seeds = ""
-`
 
-func defaultConfig(moniker string) string {
-	return strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
-}
+# Address to listen for incoming connections
+laddr = "{{ .P2P.ListenAddress }}"
+
+# Comma separated list of seed nodes to connect to
+seeds = "{{ .P2P.Seeds }}"
+
+# Comma separated list of nodes to keep persistent connections to
+# Do not add private peers to this list if you don't want them advertised
+persistent_peers = "{{ .P2P.PersistentPeers }}"
+
+# Path to address book
+addr_book_file = "{{ js .P2P.AddrBook }}"
+
+# Set true for strict address routability rules
+addr_book_strict = {{ .P2P.AddrBookStrict }}
+
+# Time to wait before flushing messages out on the connection, in ms
+flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }}
+
+# Maximum number of peers to connect to
+max_num_peers = {{ .P2P.MaxNumPeers }}
+
+# Maximum size of a message packet payload, in bytes
+max_packet_msg_payload_size = {{ .P2P.MaxPacketMsgPayloadSize }}
+
+# Rate at which packets can be sent, in bytes/second
+send_rate = {{ .P2P.SendRate }}
+
+# Rate at which packets can be received, in bytes/second
+recv_rate = {{ .P2P.RecvRate }}
+
+# Set true to enable the peer-exchange reactor
+pex = {{ .P2P.PexReactor }}
+
+# Seed mode, in which node constantly crawls the network and looks for
+# peers. If another node asks it for addresses, it responds and disconnects.
+#
+# Does not work if the peer-exchange reactor is disabled.
+seed_mode = {{ .P2P.SeedMode }}
+
+# Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
+private_peer_ids = "{{ .P2P.PrivatePeerIDs }}"
+
+##### mempool configuration options #####
+[mempool]
+
+recheck = {{ .Mempool.Recheck }}
+recheck_empty = {{ .Mempool.RecheckEmpty }}
+broadcast = {{ .Mempool.Broadcast }}
+wal_dir = "{{ js .Mempool.WalPath }}"
+
+# size of the mempool
+size = {{ .Mempool.Size }}
+
+# size of the cache (used to filter transactions we saw earlier)
+cache_size = {{ .Mempool.CacheSize }}
+
+##### consensus configuration options #####
+[consensus]
+
+wal_file = "{{ js .Consensus.WalPath }}"
+
+# All timeouts are in milliseconds
+timeout_propose = {{ .Consensus.TimeoutPropose }}
+timeout_propose_delta = {{ .Consensus.TimeoutProposeDelta }}
+timeout_prevote = {{ .Consensus.TimeoutPrevote }}
+timeout_prevote_delta = {{ .Consensus.TimeoutPrevoteDelta }}
+timeout_precommit = {{ .Consensus.TimeoutPrecommit }}
+timeout_precommit_delta = {{ .Consensus.TimeoutPrecommitDelta }}
+timeout_commit = {{ .Consensus.TimeoutCommit }}
+
+# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)
+skip_timeout_commit = {{ .Consensus.SkipTimeoutCommit }}
+
+# BlockSize
+max_block_size_txs = {{ .Consensus.MaxBlockSizeTxs }}
+max_block_size_bytes = {{ .Consensus.MaxBlockSizeBytes }}
+
+# EmptyBlocks mode and possible interval between empty blocks in seconds
+create_empty_blocks = {{ .Consensus.CreateEmptyBlocks }}
+create_empty_blocks_interval = {{ .Consensus.CreateEmptyBlocksInterval }}
+
+# Reactor sleep duration parameters are in milliseconds
+peer_gossip_sleep_duration = {{ .Consensus.PeerGossipSleepDuration }}
+peer_query_maj23_sleep_duration = {{ .Consensus.PeerQueryMaj23SleepDuration }}
+
+##### transactions indexer configuration options #####
+[tx_index]
+
+# What indexer to use for transactions
+#
+# Options:
+#   1) "null" (default)
+#   2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
+indexer = "{{ .TxIndex.Indexer }}"
+
+# Comma-separated list of tags to index (by default the only tag is tx hash)
+#
+# It's recommended to index only a subset of tags due to possible memory
+# bloat. This is, of course, depends on the indexer's DB and the volume of
+# transactions.
+index_tags = "{{ .TxIndex.IndexTags }}"
+
+# When set to true, tells indexer to index all tags. Note this may be not
+# desirable (see the comment above). IndexTags has a precedence over
+# IndexAllTags (i.e. when given both, IndexTags will be indexed).
+index_all_tags = {{ .TxIndex.IndexAllTags }}
+`
 
 /****** these are for test settings ***********/
 
@@ -53,30 +241,35 @@ func ResetTestRoot(testName string) *Config {
 	rootDir = filepath.Join(rootDir, testName)
 	// Remove ~/.tendermint_test_bak
 	if cmn.FileExists(rootDir + "_bak") {
-		err := os.RemoveAll(rootDir + "_bak")
-		if err != nil {
+		if err := os.RemoveAll(rootDir + "_bak"); err != nil {
 			cmn.PanicSanity(err.Error())
 		}
 	}
 	// Move ~/.tendermint_test to ~/.tendermint_test_bak
 	if cmn.FileExists(rootDir) {
-		err := os.Rename(rootDir, rootDir+"_bak")
-		if err != nil {
+		if err := os.Rename(rootDir, rootDir+"_bak"); err != nil {
 			cmn.PanicSanity(err.Error())
 		}
 	}
 	// Create new dir
-	cmn.EnsureDir(rootDir, 0700)
-	cmn.EnsureDir(rootDir+"/data", 0700)
+	if err := cmn.EnsureDir(rootDir, 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
 
-	configFilePath := path.Join(rootDir, "config.toml")
-	genesisFilePath := path.Join(rootDir, "genesis.json")
-	privFilePath := path.Join(rootDir, "priv_validator.json")
+	baseConfig := DefaultBaseConfig()
+	configFilePath := filepath.Join(rootDir, defaultConfigFilePath)
+	genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis)
+	privFilePath := filepath.Join(rootDir, baseConfig.PrivValidator)
 
 	// Write default config file if missing.
 	if !cmn.FileExists(configFilePath) {
-		// Ask user for moniker
-		cmn.MustWriteFile(configFilePath, []byte(testConfig("anonymous")), 0644)
+		writeDefaultConfigFile(configFilePath)
 	}
 	if !cmn.FileExists(genesisFilePath) {
 		cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644)
@@ -88,36 +281,14 @@ func ResetTestRoot(testName string) *Config {
 	return config
 }
 
-var testConfigTmpl = `# This is a TOML config file.
-# For more information, see https://github.com/toml-lang/toml
-
-proxy_app = "dummy"
-moniker = "__MONIKER__"
-fast_sync = false
-db_backend = "memdb"
-log_level = "info"
-
-[rpc]
-laddr = "tcp://0.0.0.0:36657"
-
-[p2p]
-laddr = "tcp://0.0.0.0:36656"
-seeds = ""
-`
-
-func testConfig(moniker string) (testConfig string) {
-	testConfig = strings.Replace(testConfigTmpl, "__MONIKER__", moniker, -1)
-	return
-}
-
 var testGenesis = `{
   "genesis_time": "0001-01-01T00:00:00.000Z",
   "chain_id": "tendermint_test",
   "validators": [
     {
       "pub_key": {
-        "type": "ed25519",
-        "data":"3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
+        "type": "AC26791624DE60",
+        "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="
       },
       "power": 10,
       "name": ""
@@ -127,14 +298,14 @@ var testGenesis = `{
 }`
 
 var testPrivValidator = `{
-  "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456",
+  "address": "849CB2C877F87A20925F35D00AE6688342D25B47",
   "pub_key": {
-    "type": "ed25519",
-    "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
+    "type": "AC26791624DE60",
+    "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="
   },
   "priv_key": {
-    "type": "ed25519",
-    "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
+    "type": "954568A3288910",
+    "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="
   },
   "last_height": 0,
   "last_round": 0,

config/toml_test.go

@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -19,26 +20,29 @@ func ensureFiles(t *testing.T, rootDir string, files ...string) {
 }
 
 func TestEnsureRoot(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
+	require := require.New(t)
 
 	// setup temp dir for test
 	tmpDir, err := ioutil.TempDir("", "config-test")
 	require.Nil(err)
-	defer os.RemoveAll(tmpDir)
+	defer os.RemoveAll(tmpDir) // nolint: errcheck
 
 	// create root dir
 	EnsureRoot(tmpDir)
 
 	// make sure config is set properly
-	data, err := ioutil.ReadFile(filepath.Join(tmpDir, "config.toml"))
+	data, err := ioutil.ReadFile(filepath.Join(tmpDir, defaultConfigFilePath))
 	require.Nil(err)
-	assert.Equal([]byte(defaultConfig("anonymous")), data)
+
+	if !checkConfig(string(data)) {
+		t.Fatalf("config file missing some information")
+	}
 
 	ensureFiles(t, tmpDir, "data")
 }
 
 func TestEnsureTestRoot(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
+	require := require.New(t)
 
 	testName := "ensureTestRoot"
 
@@ -47,11 +51,44 @@ func TestEnsureTestRoot(t *testing.T) {
 	rootDir := cfg.RootDir
 
 	// make sure config is set properly
-	data, err := ioutil.ReadFile(filepath.Join(rootDir, "config.toml"))
+	data, err := ioutil.ReadFile(filepath.Join(rootDir, defaultConfigFilePath))
 	require.Nil(err)
-	assert.Equal([]byte(testConfig("anonymous")), data)
+
+	if !checkConfig(string(data)) {
+		t.Fatalf("config file missing some information")
+	}
 
 	// TODO: make sure the cfg returned and testconfig are the same!
-
-	ensureFiles(t, rootDir, "data", "genesis.json", "priv_validator.json")
+	baseConfig := DefaultBaseConfig()
+	ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidator)
+}
+
+func checkConfig(configFile string) bool {
+	var valid bool
+
+	// list of words we expect in the config
+	var elems = []string{
+		"moniker",
+		"seeds",
+		"proxy_app",
+		"fast_sync",
+		"create_empty_blocks",
+		"peer",
+		"timeout",
+		"broadcast",
+		"send",
+		"addr",
+		"wal",
+		"propose",
+		"max",
+		"genesis",
+	}
+	for _, e := range elems {
+		if !strings.Contains(configFile, e) {
+			valid = false
+		} else {
+			valid = true
+		}
+	}
+	return valid
 }

consensus/README.md

@@ -1,18 +1 @@
-# The core consensus algorithm.
-
-* state.go - The state machine as detailed in the whitepaper
-* reactor.go - A reactor that connects the state machine to the gossip network
-
-# Go-routine summary
-
-The reactor runs 2 go-routines for each added peer: gossipDataRoutine and gossipVotesRoutine.
-
-The consensus state runs two persistent go-routines: timeoutRoutine and receiveRoutine.
-Go-routines are also started to trigger timeouts and to avoid blocking when the internalMsgQueue is really backed up.
-
-# Replay/WAL
-
-A write-ahead log is used to record all messages processed by the receiveRoutine,
-which amounts to all inputs to the consensus state machine:
-messages from peers, messages from ourselves, and timeouts.
-They can be played back deterministically at startup or using the replay console. 
+See the [consensus spec](https://github.com/tendermint/tendermint/tree/master/docs/spec/consensus) and the [reactor consensus spec](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/consensus) for more information.

consensus/byzantine_test.go

@@ -1,16 +1,15 @@
 package consensus
 
 import (
+	"context"
 	"sync"
 	"testing"
 	"time"
 
-	crypto "github.com/tendermint/go-crypto"
-	data "github.com/tendermint/go-wire/data"
+	"github.com/stretchr/testify/require"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
-	"github.com/tendermint/tmlibs/events"
 )
 
 func init() {
@@ -28,11 +27,13 @@ func init() {
 // Heal partition and ensure A sees the commit
 func TestByzantine(t *testing.T) {
 	N := 4
-	logger := consensusLogger()
+	logger := consensusLogger().With("test", "byzantine")
 	css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter)
 
 	// give the byzantine validator a normal ticker
-	css[0].SetTimeoutTicker(NewTimeoutTicker())
+	ticker := NewTimeoutTicker()
+	ticker.SetLogger(css[0].Logger)
+	css[0].SetTimeoutTicker(ticker)
 
 	switches := make([]*p2p.Switch, N)
 	p2pLogger := logger.With("module", "p2p")
@@ -41,7 +42,44 @@ func TestByzantine(t *testing.T) {
 		switches[i].SetLogger(p2pLogger.With("validator", i))
 	}
 
+	eventChans := make([]chan interface{}, N)
 	reactors := make([]p2p.Reactor, N)
+	for i := 0; i < N; i++ {
+		// make first val byzantine
+		if i == 0 {
+			// NOTE: Now, test validators are MockPV, which by default doesn't
+			// do any safety checks.
+			css[i].privValidator.(*types.MockPV).DisableChecks()
+			css[i].decideProposal = func(j int) func(int64, int) {
+				return func(height int64, round int) {
+					byzantineDecideProposalFunc(t, height, round, css[j], switches[j])
+				}
+			}(i)
+			css[i].doPrevote = func(height int64, round int) {}
+		}
+
+		eventBus := css[i].eventBus
+		eventBus.SetLogger(logger.With("module", "events", "validator", i))
+
+		eventChans[i] = make(chan interface{}, 1)
+		err := eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		require.NoError(t, err)
+
+		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
+		conR.SetLogger(logger.With("validator", i))
+		conR.SetEventBus(eventBus)
+
+		var conRI p2p.Reactor // nolint: gotype, gosimple
+		conRI = conR
+
+		// make first val byzantine
+		if i == 0 {
+			conRI = NewByzantineReactor(conR)
+		}
+
+		reactors[i] = conRI
+	}
+
 	defer func() {
 		for _, r := range reactors {
 			if rr, ok := r.(*ByzantineReactor); ok {
@@ -51,40 +89,6 @@ func TestByzantine(t *testing.T) {
 			}
 		}
 	}()
-	eventChans := make([]chan interface{}, N)
-	eventLogger := logger.With("module", "events")
-	for i := 0; i < N; i++ {
-		if i == 0 {
-			css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator)
-			// make byzantine
-			css[i].decideProposal = func(j int) func(int, int) {
-				return func(height, round int) {
-					byzantineDecideProposalFunc(t, height, round, css[j], switches[j])
-				}
-			}(i)
-			css[i].doPrevote = func(height, round int) {}
-		}
-
-		eventSwitch := events.NewEventSwitch()
-		eventSwitch.SetLogger(eventLogger.With("validator", i))
-		_, err := eventSwitch.Start()
-		if err != nil {
-			t.Fatalf("Failed to start switch: %v", err)
-		}
-		eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1)
-
-		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
-		conR.SetLogger(logger.With("validator", i))
-		conR.SetEventSwitch(eventSwitch)
-
-		var conRI p2p.Reactor
-		conRI = conR
-
-		if i == 0 {
-			conRI = NewByzantineReactor(conR)
-		}
-		reactors[i] = conRI
-	}
 
 	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
 		// ignore new switch s, we already made ours
@@ -98,32 +102,35 @@ func TestByzantine(t *testing.T) {
 		p2p.Connect2Switches(sws, i, j)
 	})
 
-	// start the state machines
-	byzR := reactors[0].(*ByzantineReactor)
-	s := byzR.reactor.conS.GetState()
-	byzR.reactor.SwitchToConsensus(s)
+	// start the non-byz state machines.
+	// note these must be started before the byz
 	for i := 1; i < N; i++ {
 		cr := reactors[i].(*ConsensusReactor)
-		cr.SwitchToConsensus(cr.conS.GetState())
+		cr.SwitchToConsensus(cr.conS.GetState(), 0)
 	}
 
+	// start the byzantine state machine
+	byzR := reactors[0].(*ByzantineReactor)
+	s := byzR.reactor.conS.GetState()
+	byzR.reactor.SwitchToConsensus(s, 0)
+
 	// byz proposer sends one block to peers[0]
 	// and the other block to peers[1] and peers[2].
 	// note peers and switches order don't match.
 	peers := switches[0].Peers().List()
+
+	// partition A
 	ind0 := getSwitchIndex(switches, peers[0])
+
+	// partition B
 	ind1 := getSwitchIndex(switches, peers[1])
 	ind2 := getSwitchIndex(switches, peers[2])
-
-	// connect the 2 peers in the larger partition
 	p2p.Connect2Switches(switches, ind1, ind2)
 
-	// wait for someone in the big partition to make a block
+	// wait for someone in the big partition (B) to make a block
 	<-eventChans[ind2]
 
 	t.Log("A block has been committed. Healing partition")
-
-	// connect the partitions
 	p2p.Connect2Switches(switches, ind0, ind1)
 	p2p.Connect2Switches(switches, ind0, ind2)
 
@@ -159,7 +166,7 @@ func TestByzantine(t *testing.T) {
 //-------------------------------
 // byzantine consensus functions
 
-func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusState, sw *p2p.Switch) {
+func byzantineDecideProposalFunc(t *testing.T, height int64, round int, cs *ConsensusState, sw *p2p.Switch) {
 	// byzantine user should create two proposals and try to split the vote.
 	// Avoid sending on internalMsgQueue and running consensus state.
 
@@ -167,13 +174,17 @@ func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusS
 	block1, blockParts1 := cs.createProposalBlock()
 	polRound, polBlockID := cs.Votes.POLInfo()
 	proposal1 := types.NewProposal(height, round, blockParts1.Header(), polRound, polBlockID)
-	cs.privValidator.SignProposal(cs.state.ChainID, proposal1) // byzantine doesnt err
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal1); err != nil {
+		t.Error(err)
+	}
 
 	// Create a new proposal block from state/txs from the mempool.
 	block2, blockParts2 := cs.createProposalBlock()
 	polRound, polBlockID = cs.Votes.POLInfo()
 	proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID)
-	cs.privValidator.SignProposal(cs.state.ChainID, proposal2) // byzantine doesnt err
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal2); err != nil {
+		t.Error(err)
+	}
 
 	block1Hash := block1.Hash()
 	block2Hash := block2.Hash()
@@ -190,10 +201,10 @@ func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusS
 	}
 }
 
-func sendProposalAndParts(height, round int, cs *ConsensusState, peer p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) {
+func sendProposalAndParts(height int64, round int, cs *ConsensusState, peer p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) {
 	// proposal
 	msg := &ProposalMessage{Proposal: proposal}
-	peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
+	peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg))
 
 	// parts
 	for i := 0; i < parts.Total(); i++ {
@@ -203,7 +214,7 @@ func sendProposalAndParts(height, round int, cs *ConsensusState, peer p2p.Peer,
 			Round:  round,  // This tells peer that this part applies to us.
 			Part:   part,
 		}
-		peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
+		peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg))
 	}
 
 	// votes
@@ -212,8 +223,8 @@ func sendProposalAndParts(height, round int, cs *ConsensusState, peer p2p.Peer,
 	precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, parts.Header())
 	cs.mtx.Unlock()
 
-	peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}})
-	peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}})
+	peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(&VoteMessage{prevote}))
+	peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(&VoteMessage{precommit}))
 }
 
 //----------------------------------------
@@ -254,47 +265,3 @@ func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
 func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) {
 	br.reactor.Receive(chID, peer, msgBytes)
 }
-
-//----------------------------------------
-// byzantine privValidator
-
-type ByzantinePrivValidator struct {
-	types.Signer
-
-	pv types.PrivValidator
-}
-
-// Return a priv validator that will sign anything
-func NewByzantinePrivValidator(pv types.PrivValidator) *ByzantinePrivValidator {
-	return &ByzantinePrivValidator{
-		Signer: pv.(*types.PrivValidatorFS).Signer,
-		pv:     pv,
-	}
-}
-
-func (privVal *ByzantinePrivValidator) GetAddress() data.Bytes {
-	return privVal.pv.GetAddress()
-}
-
-func (privVal *ByzantinePrivValidator) GetPubKey() crypto.PubKey {
-	return privVal.pv.GetPubKey()
-}
-
-func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) (err error) {
-	vote.Signature, err = privVal.Sign(types.SignBytes(chainID, vote))
-	return err
-}
-
-func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) (err error) {
-	proposal.Signature, err = privVal.Sign(types.SignBytes(chainID, proposal))
-	return nil
-}
-
-func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) (err error) {
-	heartbeat.Signature, err = privVal.Sign(types.SignBytes(chainID, heartbeat))
-	return nil
-}
-
-func (privVal *ByzantinePrivValidator) String() string {
-	return cmn.Fmt("PrivValidator{%X}", privVal.GetAddress())
-}

consensus/common.go

@@ -1,35 +0,0 @@
-package consensus
-
-import (
-	"github.com/tendermint/tendermint/types"
-)
-
-// XXX: WARNING: these functions can halt the consensus as firing events is synchronous.
-// Make sure to read off the channels, and in the case of subscribeToEventRespond, to write back on it
-
-// NOTE: if chanCap=0, this blocks on the event being consumed
-func subscribeToEvent(evsw types.EventSwitch, receiver, eventID string, chanCap int) chan interface{} {
-	// listen for event
-	ch := make(chan interface{}, chanCap)
-	types.AddListenerForEvent(evsw, receiver, eventID, func(data types.TMEventData) {
-		ch <- data
-	})
-	return ch
-}
-
-// NOTE: this blocks on receiving a response after the event is consumed
-func subscribeToEventRespond(evsw types.EventSwitch, receiver, eventID string) chan interface{} {
-	// listen for event
-	ch := make(chan interface{})
-	types.AddListenerForEvent(evsw, receiver, eventID, func(data types.TMEventData) {
-		ch <- data
-		<-ch
-	})
-	return ch
-}
-
-func discardFromChan(ch chan interface{}, n int) {
-	for i := 0; i < n; i++ {
-		<-ch
-	}
-}

consensus/common_test.go

@@ -2,6 +2,7 @@ package consensus
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -18,6 +19,7 @@ import (
 	cstypes "github.com/tendermint/tendermint/consensus/types"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
@@ -25,14 +27,18 @@ import (
 	"github.com/tendermint/tmlibs/log"
 
 	"github.com/tendermint/abci/example/counter"
-	"github.com/tendermint/abci/example/dummy"
+	"github.com/tendermint/abci/example/kvstore"
 
 	"github.com/go-kit/kit/log/term"
 )
 
+const (
+	testSubscriber = "test-client"
+)
+
 // genesis, chain_id, priv_val
-var config *cfg.Config // NOTE: must be reset for each _test.go file
-var ensureTimeout = time.Second * 2
+var config *cfg.Config              // NOTE: must be reset for each _test.go file
+var ensureTimeout = time.Second * 1 // must be in seconds because CreateEmptyBlocksInterval is
 
 func ensureDir(dir string, mode os.FileMode) {
 	if err := cmn.EnsureDir(dir, mode); err != nil {
@@ -45,16 +51,16 @@ func ResetConfig(name string) *cfg.Config {
 }
 
 //-------------------------------------------------------------------------------
-// validator stub (a dummy consensus peer we control)
+// validator stub (a kvstore consensus peer we control)
 
 type validatorStub struct {
 	Index  int // Validator index. NOTE: we don't assume validator set changes.
-	Height int
+	Height int64
 	Round  int
 	types.PrivValidator
 }
 
-var testMinPower = 10
+var testMinPower int64 = 10
 
 func NewValidatorStub(privValidator types.PrivValidator, valIndex int) *validatorStub {
 	return &validatorStub{
@@ -69,10 +75,11 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS
 		ValidatorAddress: vs.PrivValidator.GetAddress(),
 		Height:           vs.Height,
 		Round:            vs.Round,
+		Timestamp:        time.Now().UTC(),
 		Type:             voteType,
 		BlockID:          types.BlockID{hash, header},
 	}
-	err := vs.PrivValidator.SignVote(config.ChainID, vote)
+	err := vs.PrivValidator.SignVote(config.ChainID(), vote)
 	return vote, err
 }
 
@@ -95,26 +102,26 @@ func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*v
 
 func incrementHeight(vss ...*validatorStub) {
 	for _, vs := range vss {
-		vs.Height += 1
+		vs.Height++
 	}
 }
 
 func incrementRound(vss ...*validatorStub) {
 	for _, vs := range vss {
-		vs.Round += 1
+		vs.Round++
 	}
 }
 
 //-------------------------------------------------------------------------------
 // Functions for transitioning the consensus state
 
-func startTestRound(cs *ConsensusState, height, round int) {
+func startTestRound(cs *ConsensusState, height int64, round int) {
 	cs.enterNewRound(height, round)
 	cs.startRoutines(0)
 }
 
 // Create proposal block from cs1 but sign it with vs
-func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) (proposal *types.Proposal, block *types.Block) {
+func decideProposal(cs1 *ConsensusState, vs *validatorStub, height int64, round int) (proposal *types.Proposal, block *types.Block) {
 	block, blockParts := cs1.createProposalBlock()
 	if block == nil { // on error
 		panic("error creating proposal block")
@@ -123,7 +130,7 @@ func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) (
 	// Make proposal
 	polRound, polBlockID := cs1.Votes.POLInfo()
 	proposal = types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID)
-	if err := vs.SignProposal(config.ChainID, proposal); err != nil {
+	if err := vs.SignProposal(cs1.state.ChainID, proposal); err != nil {
 		panic(err)
 	}
 	return
@@ -208,12 +215,15 @@ func validatePrevoteAndPrecommit(t *testing.T, cs *ConsensusState, thisRound, lo
 
 // genesis
 func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} {
-	voteCh0 := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1)
+	voteCh0 := make(chan interface{})
+	err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryVote, voteCh0)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote))
+	}
 	voteCh := make(chan interface{})
 	go func() {
-		for {
-			v := <-voteCh0
-			vote := v.(types.TMEventData).Unwrap().(types.EventDataVote)
+		for v := range voteCh0 {
+			vote := v.(types.EventDataVote)
 			// we only fire for our own votes
 			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
 				voteCh <- v
@@ -226,13 +236,17 @@ func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} {
 //-------------------------------------------------------------------------------
 // consensus states
 
-func newConsensusState(state *sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState {
+func newConsensusState(state sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState {
 	return newConsensusStateWithConfig(config, state, pv, app)
 }
 
-func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState {
-	// Get BlockStore
+func newConsensusStateWithConfig(thisConfig *cfg.Config, state sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState {
 	blockDB := dbm.NewMemDB()
+	return newConsensusStateWithConfigAndBlockStore(thisConfig, state, pv, app, blockDB)
+}
+
+func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.State, pv types.PrivValidator, app abci.Application, blockDB dbm.DB) *ConsensusState {
+	// Get BlockStore
 	blockStore := bc.NewBlockStore(blockDB)
 
 	// one for mempool, one for consensus
@@ -247,36 +261,31 @@ func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv typ
 		mempool.EnableTxsAvailable()
 	}
 
-	// Make ConsensusReactor
-	cs := NewConsensusState(thisConfig.Consensus, state, proxyAppConnCon, blockStore, mempool)
-	cs.SetLogger(log.TestingLogger())
+	// mock the evidence pool
+	evpool := sm.MockEvidencePool{}
+
+	// Make ConsensusState
+	stateDB := dbm.NewMemDB()
+	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+	cs := NewConsensusState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+	cs.SetLogger(log.TestingLogger().With("module", "consensus"))
 	cs.SetPrivValidator(pv)
 
-	evsw := types.NewEventSwitch()
-	evsw.SetLogger(log.TestingLogger().With("module", "events"))
-	cs.SetEventSwitch(evsw)
-	evsw.Start()
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+	eventBus.Start()
+	cs.SetEventBus(eventBus)
 	return cs
 }
 
-func loadPrivValidator(config *cfg.Config) *types.PrivValidatorFS {
+func loadPrivValidator(config *cfg.Config) *privval.FilePV {
 	privValidatorFile := config.PrivValidatorFile()
 	ensureDir(path.Dir(privValidatorFile), 0700)
-	privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile)
+	privValidator := privval.LoadOrGenFilePV(privValidatorFile)
 	privValidator.Reset()
 	return privValidator
 }
 
-func fixedConsensusStateDummy() *ConsensusState {
-	stateDB := dbm.NewMemDB()
-	state, _ := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
-	state.SetLogger(log.TestingLogger().With("module", "state"))
-	privValidator := loadPrivValidator(config)
-	cs := newConsensusState(state, privValidator, dummy.NewDummyApplication())
-	cs.SetLogger(log.TestingLogger())
-	return cs
-}
-
 func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 	// Get State
 	state, privVals := randGenesisState(nValidators, false, 10)
@@ -284,7 +293,6 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 	vss := make([]*validatorStub, nValidators)
 
 	cs := newConsensusState(state, privVals[0], counter.NewCounterApplication(true))
-	cs.SetLogger(log.TestingLogger())
 
 	for i := 0; i < nValidators; i++ {
 		vss[i] = NewValidatorStub(privVals[i], i)
@@ -297,7 +305,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 
 //-------------------------------------------------------------------------------
 
-func ensureNoNewStep(stepCh chan interface{}) {
+func ensureNoNewStep(stepCh <-chan interface{}) {
 	timer := time.NewTimer(ensureTimeout)
 	select {
 	case <-timer.C:
@@ -307,7 +315,7 @@ func ensureNoNewStep(stepCh chan interface{}) {
 	}
 }
 
-func ensureNewStep(stepCh chan interface{}) {
+func ensureNewStep(stepCh <-chan interface{}) {
 	timer := time.NewTimer(ensureTimeout)
 	select {
 	case <-timer.C:
@@ -330,39 +338,40 @@ func consensusLogger() log.Logger {
 			}
 		}
 		return term.FgBgColor{}
-	})
+	}).With("module", "consensus")
 }
 
 func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application, configOpts ...func(*cfg.Config)) []*ConsensusState {
-	genDoc, privVals := randGenesisDoc(nValidators, false, 10)
+	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
 	css := make([]*ConsensusState, nValidators)
 	logger := consensusLogger()
 	for i := 0; i < nValidators; i++ {
-		db := dbm.NewMemDB() // each state needs its own db
-		state, _ := sm.MakeGenesisState(db, genDoc)
-		state.SetLogger(logger.With("module", "state", "validator", i))
-		state.Save()
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
 		thisConfig := ResetConfig(cmn.Fmt("%s_%d", testName, i))
 		for _, opt := range configOpts {
 			opt(thisConfig)
 		}
 		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc())
-		css[i].SetLogger(logger.With("validator", i))
+		app := appFunc()
+		vals := types.TM2PB.Validators(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], app)
 		css[i].SetTimeoutTicker(tickerFunc())
+		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
 	}
 	return css
 }
 
 // nPeers = nValidators + nNotValidator
 func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application) []*ConsensusState {
-	genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower))
+	genDoc, privVals := randGenesisDoc(nValidators, false, testMinPower)
 	css := make([]*ConsensusState, nPeers)
+	logger := consensusLogger()
 	for i := 0; i < nPeers; i++ {
-		db := dbm.NewMemDB() // each state needs its own db
-		state, _ := sm.MakeGenesisState(db, genDoc)
-		state.SetLogger(log.TestingLogger().With("module", "state"))
-		state.Save()
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
 		thisConfig := ResetConfig(cmn.Fmt("%s_%d", testName, i))
 		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
 		var privVal types.PrivValidator
@@ -370,19 +379,23 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 			privVal = privVals[i]
 		} else {
 			_, tempFilePath := cmn.Tempfile("priv_validator_")
-			privVal = types.GenPrivValidatorFS(tempFilePath)
+			privVal = privval.GenFilePV(tempFilePath)
 		}
 
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc())
-		css[i].SetLogger(log.TestingLogger())
+		app := appFunc()
+		vals := types.TM2PB.Validators(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, app)
 		css[i].SetTimeoutTicker(tickerFunc())
+		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
 	}
 	return css
 }
 
 func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
 	for i, s := range switches {
-		if bytes.Equal(peer.NodeInfo().PubKey.Address(), s.NodeInfo().PubKey.Address()) {
+		if peer.NodeInfo().ID == s.NodeInfo().ID {
 			return i
 		}
 	}
@@ -393,9 +406,9 @@ func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
 //-------------------------------------------------------------------------------
 // genesis
 
-func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []*types.PrivValidatorFS) {
+func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
 	validators := make([]types.GenesisValidator, numValidators)
-	privValidators := make([]*types.PrivValidatorFS, numValidators)
+	privValidators := make([]types.PrivValidator, numValidators)
 	for i := 0; i < numValidators; i++ {
 		val, privVal := types.RandValidator(randPower, minPower)
 		validators[i] = types.GenesisValidator{
@@ -405,19 +418,19 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G
 		privValidators[i] = privVal
 	}
 	sort.Sort(types.PrivValidatorsByAddress(privValidators))
+
 	return &types.GenesisDoc{
 		GenesisTime: time.Now(),
-		ChainID:     config.ChainID,
+		ChainID:     config.ChainID(),
 		Validators:  validators,
 	}, privValidators
 }
 
-func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidatorFS) {
+func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []types.PrivValidator) {
 	genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower)
+	s0, _ := sm.MakeGenesisState(genDoc)
 	db := dbm.NewMemDB()
-	s0, _ := sm.MakeGenesisState(db, genDoc)
-	s0.SetLogger(log.TestingLogger().With("module", "state"))
-	s0.Save()
+	sm.SaveState(db, s0)
 	return s0, privValidators
 }
 
@@ -443,12 +456,12 @@ type mockTicker struct {
 	fired    bool
 }
 
-func (m *mockTicker) Start() (bool, error) {
-	return true, nil
+func (m *mockTicker) Start() error {
+	return nil
 }
 
-func (m *mockTicker) Stop() bool {
-	return true
+func (m *mockTicker) Stop() error {
+	return nil
 }
 
 func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) {
@@ -476,7 +489,7 @@ func newCounter() abci.Application {
 	return counter.NewCounterApplication(true)
 }
 
-func newPersistentDummy() abci.Application {
-	dir, _ := ioutil.TempDir("/tmp", "persistent-dummy")
-	return dummy.NewPersistentDummyApplication(dir)
+func newPersistentKVStore() abci.Application {
+	dir, _ := ioutil.TempDir("/tmp", "persistent-kvstore")
+	return kvstore.NewPersistentKVStoreApplication(dir)
 }

consensus/mempool_test.go

@@ -2,46 +2,49 @@ package consensus
 
 import (
 	"encoding/binary"
+	"fmt"
 	"testing"
 	"time"
 
-	abci "github.com/tendermint/abci/types"
-	"github.com/tendermint/tendermint/types"
+	"github.com/stretchr/testify/assert"
 
+	"github.com/tendermint/abci/example/code"
+	abci "github.com/tendermint/abci/types"
 	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/types"
 )
 
 func init() {
 	config = ResetConfig("consensus_mempool_test")
 }
 
-func TestNoProgressUntilTxsAvailable(t *testing.T) {
+func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
 	config := ResetConfig("consensus_mempool_txs_available_test")
 	config.Consensus.CreateEmptyBlocks = false
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
 	cs.mempool.EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	startTestRound(cs, height, round)
 
 	ensureNewStep(newBlockCh) // first block gets committed
 	ensureNoNewStep(newBlockCh)
-	deliverTxsRange(cs, 0, 2)
+	deliverTxsRange(cs, 0, 1)
 	ensureNewStep(newBlockCh) // commit txs
 	ensureNewStep(newBlockCh) // commit updated app hash
 	ensureNoNewStep(newBlockCh)
-
 }
 
-func TestProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
+func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
 	config := ResetConfig("consensus_mempool_txs_available_test")
 	config.Consensus.CreateEmptyBlocksInterval = int(ensureTimeout.Seconds())
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
 	cs.mempool.EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	startTestRound(cs, height, round)
 
 	ensureNewStep(newBlockCh)   // first block gets committed
@@ -49,16 +52,16 @@ func TestProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
 	ensureNewStep(newBlockCh)   // until the CreateEmptyBlocksInterval has passed
 }
 
-func TestProgressInHigherRound(t *testing.T) {
+func TestMempoolProgressInHigherRound(t *testing.T) {
 	config := ResetConfig("consensus_mempool_txs_available_test")
 	config.Consensus.CreateEmptyBlocks = false
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
 	cs.mempool.EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
-	newRoundCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewRound(), 1)
-	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
 	cs.setProposal = func(proposal *types.Proposal) error {
 		if cs.Height == 2 && cs.Round == 0 {
 			// dont set the proposal in round 0 so we timeout and
@@ -73,7 +76,7 @@ func TestProgressInHigherRound(t *testing.T) {
 	ensureNewStep(newRoundCh) // first round at first height
 	ensureNewStep(newBlockCh) // first block gets committed
 	ensureNewStep(newRoundCh) // first round at next height
-	deliverTxsRange(cs, 0, 2) // we deliver txs, but dont set a proposal so we get the next round
+	deliverTxsRange(cs, 0, 1) // we deliver txs, but dont set a proposal so we get the next round
 	<-timeoutCh
 	ensureNewStep(newRoundCh) // wait for the next round
 	ensureNewStep(newBlockCh) // now we can commit the block
@@ -91,29 +94,29 @@ func deliverTxsRange(cs *ConsensusState, start, end int) {
 	}
 }
 
-func TestTxConcurrentWithCommit(t *testing.T) {
-
+func TestMempoolTxConcurrentWithCommit(t *testing.T) {
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusState(state, privVals[0], NewCounterApplication())
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 
 	NTxs := 10000
 	go deliverTxsRange(cs, 0, NTxs)
 
 	startTestRound(cs, height, round)
-	ticker := time.NewTicker(time.Second * 20)
 	for nTxs := 0; nTxs < NTxs; {
+		ticker := time.NewTicker(time.Second * 30)
 		select {
 		case b := <-newBlockCh:
-			nTxs += b.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block.Header.NumTxs
+			evt := b.(types.EventDataNewBlock)
+			nTxs += int(evt.Block.Header.NumTxs)
 		case <-ticker.C:
 			panic("Timed out waiting to commit blocks with transactions")
 		}
 	}
 }
 
-func TestRmBadTx(t *testing.T) {
+func TestMempoolRmBadTx(t *testing.T) {
 	state, privVals := randGenesisState(1, false, 10)
 	app := NewCounterApplication()
 	cs := newConsensusState(state, privVals[0], app)
@@ -121,41 +124,44 @@ func TestRmBadTx(t *testing.T) {
 	// increment the counter by 1
 	txBytes := make([]byte, 8)
 	binary.BigEndian.PutUint64(txBytes, uint64(0))
-	app.DeliverTx(txBytes)
-	app.Commit()
 
-	ch := make(chan struct{})
-	cbCh := make(chan struct{})
+	resDeliver := app.DeliverTx(txBytes)
+	assert.False(t, resDeliver.IsErr(), cmn.Fmt("expected no error. got %v", resDeliver))
+
+	resCommit := app.Commit()
+	assert.True(t, len(resCommit.Data) > 0)
+
+	emptyMempoolCh := make(chan struct{})
+	checkTxRespCh := make(chan struct{})
 	go func() {
 		// Try to send the tx through the mempool.
 		// CheckTx should not err, but the app should return a bad abci code
 		// and the tx should get removed from the pool
 		err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) {
-			if r.GetCheckTx().Code != abci.CodeType_BadNonce {
+			if r.GetCheckTx().Code != code.CodeTypeBadNonce {
 				t.Fatalf("expected checktx to return bad nonce, got %v", r)
 			}
-			cbCh <- struct{}{}
+			checkTxRespCh <- struct{}{}
 		})
 		if err != nil {
-			t.Fatal("Error after CheckTx: %v", err)
+			t.Fatalf("Error after CheckTx: %v", err)
 		}
 
 		// check for the tx
 		for {
-			time.Sleep(time.Second)
 			txs := cs.mempool.Reap(1)
 			if len(txs) == 0 {
-				ch <- struct{}{}
+				emptyMempoolCh <- struct{}{}
 				return
 			}
-
+			time.Sleep(10 * time.Millisecond)
 		}
 	}()
 
 	// Wait until the tx returns
 	ticker := time.After(time.Second * 5)
 	select {
-	case <-cbCh:
+	case <-checkTxRespCh:
 		// success
 	case <-ticker:
 		t.Fatalf("Timed out waiting for tx to return")
@@ -164,7 +170,7 @@ func TestRmBadTx(t *testing.T) {
 	// Wait until the tx is removed
 	ticker = time.After(time.Second * 5)
 	select {
-	case <-ch:
+	case <-emptyMempoolCh:
 		// success
 	case <-ticker:
 		t.Fatalf("Timed out waiting for tx to be removed")
@@ -187,33 +193,40 @@ func (app *CounterApplication) Info(req abci.RequestInfo) abci.ResponseInfo {
 	return abci.ResponseInfo{Data: cmn.Fmt("txs:%v", app.txCount)}
 }
 
-func (app *CounterApplication) DeliverTx(tx []byte) abci.Result {
-	return runTx(tx, &app.txCount)
+func (app *CounterApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx {
+	txValue := txAsUint64(tx)
+	if txValue != uint64(app.txCount) {
+		return abci.ResponseDeliverTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)}
+	}
+	app.txCount++
+	return abci.ResponseDeliverTx{Code: code.CodeTypeOK}
 }
 
-func (app *CounterApplication) CheckTx(tx []byte) abci.Result {
-	return runTx(tx, &app.mempoolTxCount)
+func (app *CounterApplication) CheckTx(tx []byte) abci.ResponseCheckTx {
+	txValue := txAsUint64(tx)
+	if txValue != uint64(app.mempoolTxCount) {
+		return abci.ResponseCheckTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)}
+	}
+	app.mempoolTxCount++
+	return abci.ResponseCheckTx{Code: code.CodeTypeOK}
 }
 
-func runTx(tx []byte, countPtr *int) abci.Result {
-	count := *countPtr
+func txAsUint64(tx []byte) uint64 {
 	tx8 := make([]byte, 8)
 	copy(tx8[len(tx8)-len(tx):], tx)
-	txValue := binary.BigEndian.Uint64(tx8)
-	if txValue != uint64(count) {
-		return abci.ErrBadNonce.AppendLog(cmn.Fmt("Invalid nonce. Expected %v, got %v", count, txValue))
-	}
-	*countPtr += 1
-	return abci.OK
+	return binary.BigEndian.Uint64(tx8)
 }
 
-func (app *CounterApplication) Commit() abci.Result {
+func (app *CounterApplication) Commit() abci.ResponseCommit {
 	app.mempoolTxCount = app.txCount
 	if app.txCount == 0 {
-		return abci.OK
-	} else {
-		hash := make([]byte, 8)
-		binary.BigEndian.PutUint64(hash, uint64(app.txCount))
-		return abci.NewResultOK(hash, "")
+		return abci.ResponseCommit{}
 	}
+	hash := make([]byte, 8)
+	binary.BigEndian.PutUint64(hash, uint64(app.txCount))
+	return abci.ResponseCommit{Data: hash}
 }

consensus/reactor_test.go

@@ -1,17 +1,26 @@
 package consensus
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
 	"sync"
 	"testing"
 	"time"
 
-	"github.com/tendermint/abci/example/dummy"
-	"github.com/tendermint/tmlibs/events"
+	"github.com/tendermint/abci/example/kvstore"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
+	p2pdummy "github.com/tendermint/tendermint/p2p/dummy"
 	"github.com/tendermint/tendermint/types"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func init() {
@@ -21,60 +30,64 @@ func init() {
 //----------------------------------------------
 // in-process testnets
 
-func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEventRespond bool) ([]*ConsensusReactor, []chan interface{}) {
+func startConsensusNet(t *testing.T, css []*ConsensusState, N int) ([]*ConsensusReactor, []chan interface{}, []*types.EventBus) {
 	reactors := make([]*ConsensusReactor, N)
 	eventChans := make([]chan interface{}, N)
-	logger := consensusLogger()
+	eventBuses := make([]*types.EventBus, N)
 	for i := 0; i < N; i++ {
+		/*logger, err := tmflags.ParseLogLevel("consensus:info,*:error", logger, "info")
+		if err != nil {	t.Fatal(err)}*/
 		reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states
-		reactors[i].SetLogger(logger.With("validator", i))
+		reactors[i].SetLogger(css[i].Logger)
 
-		eventSwitch := events.NewEventSwitch()
-		eventSwitch.SetLogger(logger.With("module", "events", "validator", i))
-		_, err := eventSwitch.Start()
-		if err != nil {
-			t.Fatalf("Failed to start switch: %v", err)
-		}
+		// eventBus is already started with the cs
+		eventBuses[i] = css[i].eventBus
+		reactors[i].SetEventBus(eventBuses[i])
 
-		reactors[i].SetEventSwitch(eventSwitch)
-		if subscribeEventRespond {
-			eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock())
-		} else {
-			eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1)
-		}
+		eventChans[i] = make(chan interface{}, 1)
+		err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		require.NoError(t, err)
 	}
 	// make connected switches and start all reactors
 	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
 		s.AddReactor("CONSENSUS", reactors[i])
+		s.SetLogger(reactors[i].conS.Logger.With("module", "p2p"))
 		return s
 	}, p2p.Connect2Switches)
 
 	// now that everyone is connected,  start the state machines
 	// If we started the state machines before everyone was connected,
 	// we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors
+	// TODO: is this still true with new pubsub?
 	for i := 0; i < N; i++ {
 		s := reactors[i].conS.GetState()
-		reactors[i].SwitchToConsensus(s)
+		reactors[i].SwitchToConsensus(s, 0)
 	}
-	return reactors, eventChans
+	return reactors, eventChans, eventBuses
 }
 
-func stopConsensusNet(reactors []*ConsensusReactor) {
-	for _, r := range reactors {
+func stopConsensusNet(logger log.Logger, reactors []*ConsensusReactor, eventBuses []*types.EventBus) {
+	logger.Info("stopConsensusNet", "n", len(reactors))
+	for i, r := range reactors {
+		logger.Info("stopConsensusNet: Stopping ConsensusReactor", "i", i)
 		r.Switch.Stop()
 	}
+	for i, b := range eventBuses {
+		logger.Info("stopConsensusNet: Stopping eventBus", "i", i)
+		b.Stop()
+	}
+	logger.Info("stopConsensusNet: DONE", "n", len(reactors))
 }
 
 // Ensure a testnet makes blocks
-func TestReactor(t *testing.T) {
+func TestReactorBasic(t *testing.T) {
 	N := 4
 	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
-	reactors, eventChans := startConsensusNet(t, css, N, false)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 	// wait till everyone makes the first new block
-	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
+	timeoutWaitGroup(t, N, func(j int) {
 		<-eventChans[j]
-		wg.Done()
 	}, css)
 }
 
@@ -85,36 +98,146 @@ func TestReactorProposalHeartbeats(t *testing.T) {
 		func(c *cfg.Config) {
 			c.Consensus.CreateEmptyBlocks = false
 		})
-	reactors, eventChans := startConsensusNet(t, css, N, false)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 	heartbeatChans := make([]chan interface{}, N)
+	var err error
 	for i := 0; i < N; i++ {
-		heartbeatChans[i] = subscribeToEvent(css[i].evsw, "tester", types.EventStringProposalHeartbeat(), 1)
+		heartbeatChans[i] = make(chan interface{}, 1)
+		err = eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryProposalHeartbeat, heartbeatChans[i])
+		require.NoError(t, err)
 	}
 	// wait till everyone sends a proposal heartbeat
-	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
+	timeoutWaitGroup(t, N, func(j int) {
 		<-heartbeatChans[j]
-		wg.Done()
 	}, css)
 
 	// send a tx
-	css[3].mempool.CheckTx([]byte{1, 2, 3}, nil)
+	if err := css[3].mempool.CheckTx([]byte{1, 2, 3}, nil); err != nil {
+		//t.Fatal(err)
+	}
 
 	// wait till everyone makes the first new block
-	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
+	timeoutWaitGroup(t, N, func(j int) {
 		<-eventChans[j]
-		wg.Done()
 	}, css)
 }
 
+// Test we record block parts from other peers
+func TestReactorRecordsBlockParts(t *testing.T) {
+	// create dummy peer
+	peer := p2pdummy.NewPeer()
+	ps := NewPeerState(peer).SetLogger(log.TestingLogger())
+	peer.Set(types.PeerStateKey, ps)
+
+	// create reactor
+	css := randConsensusNet(1, "consensus_reactor_records_block_parts_test", newMockTickerFunc(true), newPersistentKVStore)
+	reactor := NewConsensusReactor(css[0], false) // so we dont start the consensus states
+	reactor.SetEventBus(css[0].eventBus)
+	reactor.SetLogger(log.TestingLogger())
+	sw := p2p.MakeSwitch(cfg.DefaultP2PConfig(), 1, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { return sw })
+	reactor.SetSwitch(sw)
+	err := reactor.Start()
+	require.NoError(t, err)
+	defer reactor.Stop()
+
+	// 1) new block part
+	parts := types.NewPartSetFromData(cmn.RandBytes(100), 10)
+	msg := &BlockPartMessage{
+		Height: 2,
+		Round:  0,
+		Part:   parts.GetPart(0),
+	}
+	bz, err := cdc.MarshalBinaryBare(msg)
+	require.NoError(t, err)
+
+	reactor.Receive(DataChannel, peer, bz)
+	require.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should have increased by 1")
+
+	// 2) block part with the same height, but different round
+	msg.Round = 1
+
+	bz, err = cdc.MarshalBinaryBare(msg)
+	require.NoError(t, err)
+
+	reactor.Receive(DataChannel, peer, bz)
+	require.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should stay the same")
+
+	// 3) block part from earlier height
+	msg.Height = 1
+	msg.Round = 0
+
+	bz, err = cdc.MarshalBinaryBare(msg)
+	require.NoError(t, err)
+
+	reactor.Receive(DataChannel, peer, bz)
+	require.Equal(t, 1, ps.BlockPartsSent(), "number of block parts sent should stay the same")
+}
+
+// Test we record votes from other peers
+func TestReactorRecordsVotes(t *testing.T) {
+	// create dummy peer
+	peer := p2pdummy.NewPeer()
+	ps := NewPeerState(peer).SetLogger(log.TestingLogger())
+	peer.Set(types.PeerStateKey, ps)
+
+	// create reactor
+	css := randConsensusNet(1, "consensus_reactor_records_votes_test", newMockTickerFunc(true), newPersistentKVStore)
+	reactor := NewConsensusReactor(css[0], false) // so we dont start the consensus states
+	reactor.SetEventBus(css[0].eventBus)
+	reactor.SetLogger(log.TestingLogger())
+	sw := p2p.MakeSwitch(cfg.DefaultP2PConfig(), 1, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { return sw })
+	reactor.SetSwitch(sw)
+	err := reactor.Start()
+	require.NoError(t, err)
+	defer reactor.Stop()
+	_, val := css[0].state.Validators.GetByIndex(0)
+
+	// 1) new vote
+	vote := &types.Vote{
+		ValidatorIndex:   0,
+		ValidatorAddress: val.Address,
+		Height:           2,
+		Round:            0,
+		Timestamp:        time.Now().UTC(),
+		Type:             types.VoteTypePrevote,
+		BlockID:          types.BlockID{},
+	}
+	bz, err := cdc.MarshalBinaryBare(&VoteMessage{vote})
+	require.NoError(t, err)
+
+	reactor.Receive(VoteChannel, peer, bz)
+	assert.Equal(t, 1, ps.VotesSent(), "number of votes sent should have increased by 1")
+
+	// 2) vote with the same height, but different round
+	vote.Round = 1
+
+	bz, err = cdc.MarshalBinaryBare(&VoteMessage{vote})
+	require.NoError(t, err)
+
+	reactor.Receive(VoteChannel, peer, bz)
+	assert.Equal(t, 1, ps.VotesSent(), "number of votes sent should stay the same")
+
+	// 3) vote from earlier height
+	vote.Height = 1
+	vote.Round = 0
+
+	bz, err = cdc.MarshalBinaryBare(&VoteMessage{vote})
+	require.NoError(t, err)
+
+	reactor.Receive(VoteChannel, peer, bz)
+	assert.Equal(t, 1, ps.VotesSent(), "number of votes sent should stay the same")
+}
+
 //-------------------------------------------------------------
 // ensure we can make blocks despite cycling a validator set
 
-func TestVotingPowerChange(t *testing.T) {
+func TestReactorVotingPowerChange(t *testing.T) {
 	nVals := 4
-	css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentDummy)
-	reactors, eventChans := startConsensusNet(t, css, nVals, true)
-	defer stopConsensusNet(reactors)
+	logger := log.TestingLogger()
+	css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentKVStore)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, nVals)
+	defer stopConsensusNet(logger, reactors, eventBuses)
 
 	// map of active validators
 	activeVals := make(map[string]struct{})
@@ -123,21 +246,20 @@ func TestVotingPowerChange(t *testing.T) {
 	}
 
 	// wait till everyone makes block 1
-	timeoutWaitGroup(t, nVals, func(wg *sync.WaitGroup, j int) {
+	timeoutWaitGroup(t, nVals, func(j int) {
 		<-eventChans[j]
-		eventChans[j] <- struct{}{}
-		wg.Done()
 	}, css)
 
 	//---------------------------------------------------------------------------
-	t.Log("---------------------------- Testing changing the voting power of one validator a few times")
+	logger.Debug("---------------------------- Testing changing the voting power of one validator a few times")
 
 	val1PubKey := css[0].privValidator.GetPubKey()
-	updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25)
+	val1PubKeyABCI := types.TM2PB.PubKey(val1PubKey)
+	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25)
 	previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, eventChans, css, updateValidatorTx)
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
 
@@ -145,11 +267,11 @@ func TestVotingPowerChange(t *testing.T) {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
 	}
 
-	updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 2)
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2)
 	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, eventChans, css, updateValidatorTx)
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
 
@@ -157,11 +279,11 @@ func TestVotingPowerChange(t *testing.T) {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
 	}
 
-	updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 100)
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26)
 	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, eventChans, css, updateValidatorTx)
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
 	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
 
@@ -170,12 +292,15 @@ func TestVotingPowerChange(t *testing.T) {
 	}
 }
 
-func TestValidatorSetChanges(t *testing.T) {
+func TestReactorValidatorSetChanges(t *testing.T) {
 	nPeers := 7
 	nVals := 4
-	css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy)
-	reactors, eventChans := startConsensusNet(t, css, nPeers, true)
-	defer stopConsensusNet(reactors)
+	css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentKVStore)
+
+	logger := log.TestingLogger()
+
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, nPeers)
+	defer stopConsensusNet(logger, reactors, eventBuses)
 
 	// map of active validators
 	activeVals := make(map[string]struct{})
@@ -184,17 +309,16 @@ func TestValidatorSetChanges(t *testing.T) {
 	}
 
 	// wait till everyone makes block 1
-	timeoutWaitGroup(t, nPeers, func(wg *sync.WaitGroup, j int) {
+	timeoutWaitGroup(t, nPeers, func(j int) {
 		<-eventChans[j]
-		eventChans[j] <- struct{}{}
-		wg.Done()
 	}, css)
 
 	//---------------------------------------------------------------------------
-	t.Log("---------------------------- Testing adding one validator")
+	logger.Info("---------------------------- Testing adding one validator")
 
 	newValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
-	newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower))
+	valPubKey1ABCI := types.TM2PB.PubKey(newValidatorPubKey1)
+	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
 
 	// wait till everyone makes block 2
 	// ensure the commit includes all validators
@@ -203,7 +327,7 @@ func TestValidatorSetChanges(t *testing.T) {
 
 	// wait till everyone makes block 3.
 	// it includes the commit for block 2, which is by the original validator set
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, newValidatorTx1)
 
 	// wait till everyone makes block 4.
 	// it includes the commit for block 3, which is by the original validator set
@@ -214,52 +338,55 @@ func TestValidatorSetChanges(t *testing.T) {
 
 	// wait till everyone makes block 5
 	// it includes the commit for block 4, which should have the updated validator set
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 
 	//---------------------------------------------------------------------------
-	t.Log("---------------------------- Testing changing the voting power of one validator")
+	logger.Info("---------------------------- Testing changing the voting power of one validator")
 
 	updateValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
-	updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25)
+	updatePubKey1ABCI := types.TM2PB.PubKey(updateValidatorPubKey1)
+	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
 	previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower()
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 
 	if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
 		t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].GetRoundState().LastValidators.TotalVotingPower())
 	}
 
 	//---------------------------------------------------------------------------
-	t.Log("---------------------------- Testing adding two validators at once")
+	logger.Info("---------------------------- Testing adding two validators at once")
 
 	newValidatorPubKey2 := css[nVals+1].privValidator.GetPubKey()
-	newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower))
+	newVal2ABCI := types.TM2PB.PubKey(newValidatorPubKey2)
+	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
 
 	newValidatorPubKey3 := css[nVals+2].privValidator.GetPubKey()
-	newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower))
+	newVal3ABCI := types.TM2PB.PubKey(newValidatorPubKey3)
+	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 	activeVals[string(newValidatorPubKey2.Address())] = struct{}{}
 	activeVals[string(newValidatorPubKey3.Address())] = struct{}{}
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 
 	//---------------------------------------------------------------------------
-	t.Log("---------------------------- Testing removing two validators at once")
+	logger.Info("---------------------------- Testing removing two validators at once")
 
-	removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0)
-	removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0)
+	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
+	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 	delete(activeVals, string(newValidatorPubKey2.Address()))
 	delete(activeVals, string(newValidatorPubKey3.Address()))
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 }
 
 // Check we can make blocks with skip_timeout_commit=false
@@ -271,31 +398,86 @@ func TestReactorWithTimeoutCommit(t *testing.T) {
 		css[i].config.SkipTimeoutCommit = false
 	}
 
-	reactors, eventChans := startConsensusNet(t, css, N-1, false)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N-1)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 
 	// wait till everyone makes the first new block
-	timeoutWaitGroup(t, N-1, func(wg *sync.WaitGroup, j int) {
+	timeoutWaitGroup(t, N-1, func(j int) {
 		<-eventChans[j]
-		wg.Done()
 	}, css)
 }
 
 func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
-	timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) {
-		newBlockI := <-eventChans[j]
-		newBlock := newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block
-		t.Logf("[WARN] Got block height=%v validator=%v", newBlock.Height, j)
-		err := validateBlock(newBlock, activeVals)
-		if err != nil {
-			t.Fatal(err)
+	timeoutWaitGroup(t, n, func(j int) {
+		css[j].Logger.Debug("waitForAndValidateBlock")
+		newBlockI, ok := <-eventChans[j]
+		if !ok {
+			return
 		}
+		newBlock := newBlockI.(types.EventDataNewBlock).Block
+		css[j].Logger.Debug("waitForAndValidateBlock: Got block", "height", newBlock.Height)
+		err := validateBlock(newBlock, activeVals)
+		assert.Nil(t, err)
 		for _, tx := range txs {
 			css[j].mempool.CheckTx(tx, nil)
+			assert.Nil(t, err)
+		}
+	}, css)
+}
+
+func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
+	timeoutWaitGroup(t, n, func(j int) {
+		ntxs := 0
+	BLOCK_TX_LOOP:
+		for {
+			css[j].Logger.Debug("waitForAndValidateBlockWithTx", "ntxs", ntxs)
+			newBlockI, ok := <-eventChans[j]
+			if !ok {
+				return
+			}
+			newBlock := newBlockI.(types.EventDataNewBlock).Block
+			css[j].Logger.Debug("waitForAndValidateBlockWithTx: Got block", "height", newBlock.Height)
+			err := validateBlock(newBlock, activeVals)
+			assert.Nil(t, err)
+
+			// check that txs match the txs we're waiting for.
+			// note they could be spread over multiple blocks,
+			// but they should be in order.
+			for _, tx := range newBlock.Data.Txs {
+				assert.EqualValues(t, txs[ntxs], tx)
+				ntxs++
+			}
+
+			if ntxs == len(txs) {
+				break BLOCK_TX_LOOP
+			}
 		}
 
-		eventChans[j] <- struct{}{}
-		wg.Done()
+	}, css)
+}
+
+func waitForBlockWithUpdatedValsAndValidateIt(t *testing.T, n int, updatedVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState) {
+	timeoutWaitGroup(t, n, func(j int) {
+
+		var newBlock *types.Block
+	LOOP:
+		for {
+			css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt")
+			newBlockI, ok := <-eventChans[j]
+			if !ok {
+				return
+			}
+			newBlock = newBlockI.(types.EventDataNewBlock).Block
+			if newBlock.LastCommit.Size() == len(updatedVals) {
+				css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt: Got block", "height", newBlock.Height)
+				break LOOP
+			} else {
+				css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt: Got block with no new validators. Skipping", "height", newBlock.Height)
+			}
+		}
+
+		err := validateBlock(newBlock, updatedVals)
+		assert.Nil(t, err)
 	}, css)
 }
 
@@ -313,11 +495,14 @@ func validateBlock(block *types.Block, activeVals map[string]struct{}) error {
 	return nil
 }
 
-func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int), css []*ConsensusState) {
+func timeoutWaitGroup(t *testing.T, n int, f func(int), css []*ConsensusState) {
 	wg := new(sync.WaitGroup)
 	wg.Add(n)
 	for i := 0; i < n; i++ {
-		go f(wg, i)
+		go func(j int) {
+			f(j)
+			wg.Done()
+		}(i)
 	}
 
 	done := make(chan struct{})
@@ -326,15 +511,28 @@ func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int), css []*
 		close(done)
 	}()
 
+	// we're running many nodes in-process, possibly in in a virtual machine,
+	// and spewing debug messages - making a block could take a while,
+	timeout := time.Second * 300
+
 	select {
 	case <-done:
-	case <-time.After(time.Second * 10):
+	case <-time.After(timeout):
 		for i, cs := range css {
-			fmt.Println("#################")
-			fmt.Println("Validator", i)
-			fmt.Println(cs.GetRoundState())
-			fmt.Println("")
+			t.Log("#################")
+			t.Log("Validator", i)
+			t.Log(cs.GetRoundState())
+			t.Log("")
 		}
+		os.Stdout.Write([]byte("pprof.Lookup('goroutine'):\n"))
+		pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		capture()
 		panic("Timed out waiting for all validators to commit a block")
 	}
 }
+
+func capture() {
+	trace := make([]byte, 10240000)
+	count := runtime.Stack(trace, true)
+	fmt.Printf("Stack of %d bytes: %s\n", count, trace)
+}

consensus/replay.go

@@ -2,18 +2,18 @@ package consensus
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"reflect"
-	"strconv"
-	"strings"
+	//"strconv"
+	//"strings"
 	"time"
 
 	abci "github.com/tendermint/abci/types"
-	wire "github.com/tendermint/go-wire"
-	auto "github.com/tendermint/tmlibs/autofile"
+	//auto "github.com/tendermint/tmlibs/autofile"
 	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
 
 	"github.com/tendermint/tendermint/proxy"
@@ -22,31 +22,30 @@ import (
 	"github.com/tendermint/tendermint/version"
 )
 
+var crc32c = crc32.MakeTable(crc32.Castagnoli)
+
 // Functionality to replay blocks and messages on recovery from a crash.
-// There are two general failure scenarios: failure during consensus, and failure while applying the block.
-// The former is handled by the WAL, the latter by the proxyApp Handshake on restart,
-// which ultimately hands off the work to the WAL.
+// There are two general failure scenarios:
+//
+//  1. failure during consensus
+//  2. failure while applying the block
+//
+// The former is handled by the WAL, the latter by the proxyApp Handshake on
+// restart, which ultimately hands off the work to the WAL.
 
 //-----------------------------------------
-// recover from failure during consensus
-// by replaying messages from the WAL
+// 1. Recover from failure during consensus
+// (by replaying messages from the WAL)
+//-----------------------------------------
 
-// Unmarshal and apply a single message to the consensus state
-// as if it were received in receiveRoutine
-// Lines that start with "#" are ignored.
-// NOTE: receiveRoutine should not be running
-func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan interface{}) error {
-	// Skip over empty and meta lines
-	if len(msgBytes) == 0 || msgBytes[0] == '#' {
+// Unmarshal and apply a single message to the consensus state as if it were
+// received in receiveRoutine.  Lines that start with "#" are ignored.
+// NOTE: receiveRoutine should not be running.
+func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan interface{}) error {
+	// Skip meta messages which exist for demarcating boundaries.
+	if _, ok := msg.Msg.(EndHeightMessage); ok {
 		return nil
 	}
-	var err error
-	var msg TimedWALMessage
-	wire.ReadJSON(&msg, msgBytes, &err)
-	if err != nil {
-		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
-		return fmt.Errorf("Error reading json data: %v", err)
-	}
 
 	// for logging
 	switch m := msg.Msg.(type) {
@@ -66,21 +65,21 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte
 			}
 		}
 	case msgInfo:
-		peerKey := m.PeerKey
-		if peerKey == "" {
-			peerKey = "local"
+		peerID := m.PeerID
+		if peerID == "" {
+			peerID = "local"
 		}
 		switch msg := m.Msg.(type) {
 		case *ProposalMessage:
 			p := msg.Proposal
 			cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
-				p.BlockPartsHeader, "pol", p.POLRound, "peer", peerKey)
+				p.BlockPartsHeader, "pol", p.POLRound, "peer", peerID)
 		case *BlockPartMessage:
-			cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerKey)
+			cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID)
 		case *VoteMessage:
 			v := msg.Vote
 			cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
-				"blockID", v.BlockID, "peer", peerKey)
+				"blockID", v.BlockID, "peer", peerID)
 		}
 
 		cs.handleMsg(m)
@@ -93,53 +92,67 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte
 	return nil
 }
 
-// replay only those messages since the last block.
-// timeoutRoutine should run concurrently to read off tickChan
-func (cs *ConsensusState) catchupReplay(csHeight int) error {
+// Replay only those messages since the last block.  `timeoutRoutine` should
+// run concurrently to read off tickChan.
+func (cs *ConsensusState) catchupReplay(csHeight int64) error {
 
-	// set replayMode
+	// Set replayMode to true so we don't log signing errors.
 	cs.replayMode = true
 	defer func() { cs.replayMode = false }()
 
-	// Ensure that ENDHEIGHT for this height doesn't exist
-	// NOTE: This is just a sanity check. As far as we know things work fine without it,
-	// and Handshake could reuse ConsensusState if it weren't for this check (since we can crash after writing ENDHEIGHT).
-	gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight))
+	// Ensure that #ENDHEIGHT for this height doesn't exist.
+	// NOTE: This is just a sanity check. As far as we know things work fine
+	// without it, and Handshake could reuse ConsensusState if it weren't for
+	// this check (since we can crash after writing #ENDHEIGHT).
+	//
+	// Ignore data corruption errors since this is a sanity check.
+	gr, found, err := cs.wal.SearchForEndHeight(csHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
+	if err != nil {
+		return err
+	}
 	if gr != nil {
-		gr.Close()
+		if err := gr.Close(); err != nil {
+			return err
+		}
 	}
 	if found {
-		return errors.New(cmn.Fmt("WAL should not contain #ENDHEIGHT %d.", csHeight))
+		return fmt.Errorf("WAL should not contain #ENDHEIGHT %d", csHeight)
 	}
 
-	// Search for last height marker
-	gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1))
+	// Search for last height marker.
+	//
+	// Ignore data corruption errors in previous heights because we only care about last height
+	gr, found, err = cs.wal.SearchForEndHeight(csHeight-1, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
 	if err == io.EOF {
 		cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", csHeight-1)
 	} else if err != nil {
 		return err
-	} else {
-		defer gr.Close()
 	}
 	if !found {
-		return errors.New(cmn.Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1))
+		return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, csHeight-1)
 	}
+	defer gr.Close() // nolint: errcheck
 
 	cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight)
 
+	var msg *TimedWALMessage
+	dec := WALDecoder{gr}
+
 	for {
-		line, err := gr.ReadLine()
-		if err != nil {
-			if err == io.EOF {
-				break
-			} else {
-				return err
-			}
+		msg, err = dec.Decode()
+		if err == io.EOF {
+			break
+		} else if IsDataCorruptionError(err) {
+			cs.Logger.Debug("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight)
+			panic(fmt.Sprintf("data has been corrupted (%v) in last height %d of consensus WAL", err, csHeight))
+		} else if err != nil {
+			return err
 		}
+
 		// NOTE: since the priv key is set when the msgs are received
 		// it will attempt to eg double sign but we can just ignore it
 		// since the votes will be replayed and we'll get to the next step
-		if err := cs.readReplayMessage([]byte(line), nil); err != nil {
+		if err := cs.readReplayMessage(msg, nil); err != nil {
 			return err
 		}
 	}
@@ -151,7 +164,8 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 
 // Parses marker lines of the form:
 // #ENDHEIGHT: 12345
-func makeHeightSearchFunc(height int) auto.SearchFunc {
+/*
+func makeHeightSearchFunc(height int64) auto.SearchFunc {
 	return func(line string) (int, error) {
 		line = strings.TrimRight(line, "\n")
 		parts := strings.Split(line, " ")
@@ -170,23 +184,35 @@ func makeHeightSearchFunc(height int) auto.SearchFunc {
 			return -1, nil
 		}
 	}
-}
+}*/
 
-//----------------------------------------------
-// Recover from failure during block processing
-// by handshaking with the app to figure out where
-// we were last and using the WAL to recover there
+//---------------------------------------------------
+// 2. Recover from failure while applying the block.
+// (by handshaking with the app to figure out where
+// we were last, and using the WAL to recover there.)
+//---------------------------------------------------
 
 type Handshaker struct {
-	state  *sm.State
-	store  types.BlockStore
-	logger log.Logger
+	stateDB      dbm.DB
+	initialState sm.State
+	store        sm.BlockStore
+	genDoc       *types.GenesisDoc
+	logger       log.Logger
 
 	nBlocks int // number of blocks applied to the state
 }
 
-func NewHandshaker(state *sm.State, store types.BlockStore) *Handshaker {
-	return &Handshaker{state, store, log.NewNopLogger(), 0}
+func NewHandshaker(stateDB dbm.DB, state sm.State,
+	store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker {
+
+	return &Handshaker{
+		stateDB:      stateDB,
+		initialState: state,
+		store:        store,
+		genDoc:       genDoc,
+		logger:       log.NewNopLogger(),
+		nBlocks:      0,
+	}
 }
 
 func (h *Handshaker) SetLogger(l log.Logger) {
@@ -199,26 +225,31 @@ func (h *Handshaker) NBlocks() int {
 
 // TODO: retry the handshake/replay if it fails ?
 func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
-	// handshake is done via info request on the query conn
+
+	// Handshake is done via ABCI Info on the query conn.
 	res, err := proxyApp.Query().InfoSync(abci.RequestInfo{version.Version})
 	if err != nil {
-		return errors.New(cmn.Fmt("Error calling Info: %v", err))
+		return fmt.Errorf("Error calling Info: %v", err)
 	}
 
-	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
+	blockHeight := int64(res.LastBlockHeight)
+	if blockHeight < 0 {
+		return fmt.Errorf("Got a negative last block height (%d) from the app", blockHeight)
+	}
 	appHash := res.LastBlockAppHash
 
 	h.logger.Info("ABCI Handshake", "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
 
-	// TODO: check version
+	// TODO: check app version.
 
-	// replay blocks up to the latest in the blockstore
-	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
+	// Replay blocks up to the latest in the blockstore.
+	_, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp)
 	if err != nil {
-		return errors.New(cmn.Fmt("Error on replay: %v", err))
+		return fmt.Errorf("Error on replay: %v", err)
 	}
 
-	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced", "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
+	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced",
+		"appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
 
 	// TODO: (on restart) replay mempool
 
@@ -226,22 +257,48 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 }
 
 // Replay all blocks since appBlockHeight and ensure the result matches the current state.
-// Returns the final AppHash or an error
-func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
+// Returns the final AppHash or an error.
+func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight int64, proxyApp proxy.AppConns) ([]byte, error) {
 
 	storeBlockHeight := h.store.Height()
-	stateBlockHeight := h.state.LastBlockHeight
+	stateBlockHeight := state.LastBlockHeight
 	h.logger.Info("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)
 
 	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain
 	if appBlockHeight == 0 {
-		validators := types.TM2PB.Validators(h.state.Validators)
-		proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators})
+		validators := types.TM2PB.Validators(state.Validators)
+		csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams)
+		req := abci.RequestInitChain{
+			Time:            h.genDoc.GenesisTime.Unix(), // TODO
+			ChainId:         h.genDoc.ChainID,
+			ConsensusParams: csParams,
+			Validators:      validators,
+			AppStateBytes:   h.genDoc.AppStateJSON,
+		}
+		res, err := proxyApp.Consensus().InitChainSync(req)
+		if err != nil {
+			return nil, err
+		}
+
+		// if the app returned validators
+		// or consensus params, update the state
+		// with the them
+		if len(res.Validators) > 0 {
+			vals, err := types.PB2TM.Validators(res.Validators)
+			if err != nil {
+				return nil, err
+			}
+			state.Validators = types.NewValidatorSet(vals)
+		}
+		if res.ConsensusParams != nil {
+			state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams)
+		}
+		sm.SaveState(h.stateDB, state)
 	}
 
 	// First handle edge cases and constraints on the storeBlockHeight
 	if storeBlockHeight == 0 {
-		return appHash, h.checkAppHash(appHash)
+		return appHash, checkAppHash(state, appHash)
 
 	} else if storeBlockHeight < appBlockHeight {
 		// the app should never be ahead of the store (but this is under app's control)
@@ -256,6 +313,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 		cmn.PanicSanity(cmn.Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
 	}
 
+	var err error
 	// Now either store is equal to state, or one ahead.
 	// For each, consider all cases of where the app could be, given app <= store
 	if storeBlockHeight == stateBlockHeight {
@@ -263,11 +321,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 		// Either the app is asking for replay, or we're all synced up.
 		if appBlockHeight < storeBlockHeight {
 			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
-			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)
+			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false)
 
 		} else if appBlockHeight == storeBlockHeight {
 			// We're good!
-			return appHash, h.checkAppHash(appHash)
+			return appHash, checkAppHash(state, appHash)
 		}
 
 	} else if storeBlockHeight == stateBlockHeight+1 {
@@ -276,22 +334,27 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 		if appBlockHeight < stateBlockHeight {
 			// the app is further behind than it should be, so replay blocks
 			// but leave the last block to go through the WAL
-			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)
+			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true)
 
 		} else if appBlockHeight == stateBlockHeight {
 			// We haven't run Commit (both the state and app are one block behind),
 			// so replayBlock with the real app.
 			// NOTE: We could instead use the cs.WAL on cs.Start,
-			// but we'd have to allow the WAL to replay a block that wrote it's ENDHEIGHT
+			// but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT
 			h.logger.Info("Replay last block using real app")
-			return h.replayBlock(storeBlockHeight, proxyApp.Consensus())
+			state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
+			return state.AppHash, err
 
 		} else if appBlockHeight == storeBlockHeight {
 			// We ran Commit, but didn't save the state, so replayBlock with mock app
-			abciResponses := h.state.LoadABCIResponses()
+			abciResponses, err := sm.LoadABCIResponses(h.stateDB, storeBlockHeight)
+			if err != nil {
+				return nil, err
+			}
 			mockApp := newMockProxyApp(appHash, abciResponses)
 			h.logger.Info("Replay last block using mock app")
-			return h.replayBlock(storeBlockHeight, mockApp)
+			state, err = h.replayBlock(state, storeBlockHeight, mockApp)
+			return state.AppHash, err
 		}
 
 	}
@@ -300,13 +363,14 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 	return nil, nil
 }
 
-func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) {
+func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int64, mutateState bool) ([]byte, error) {
 	// App is further behind than it should be, so we need to replay blocks.
 	// We replay all blocks from appBlockHeight+1.
 	//
 	// Note that we don't have an old version of the state,
 	// so we by-pass state validation/mutation using sm.ExecCommitBlock.
 	// This also means we won't be saving validator sets if they change during this period.
+	// TODO: Load the historical information to fix this and just use state.ApplyBlock
 	//
 	// If mutateState == true, the final block is replayed with h.replayBlock()
 
@@ -314,48 +378,52 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store
 	var err error
 	finalBlock := storeBlockHeight
 	if mutateState {
-		finalBlock -= 1
+		finalBlock--
 	}
 	for i := appBlockHeight + 1; i <= finalBlock; i++ {
 		h.logger.Info("Applying block", "height", i)
 		block := h.store.LoadBlock(i)
-		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger)
+		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, state.LastValidators, h.stateDB)
 		if err != nil {
 			return nil, err
 		}
 
-		h.nBlocks += 1
+		h.nBlocks++
 	}
 
 	if mutateState {
 		// sync the final block
-		return h.replayBlock(storeBlockHeight, proxyApp.Consensus())
+		state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
+		if err != nil {
+			return nil, err
+		}
+		appHash = state.AppHash
 	}
 
-	return appHash, h.checkAppHash(appHash)
+	return appHash, checkAppHash(state, appHash)
 }
 
 // ApplyBlock on the proxyApp with the last block.
-func (h *Handshaker) replayBlock(height int, proxyApp proxy.AppConnConsensus) ([]byte, error) {
-	mempool := types.MockMempool{}
-
-	var eventCache types.Fireable // nil
+func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) {
 	block := h.store.LoadBlock(height)
 	meta := h.store.LoadBlockMeta(height)
 
-	if err := h.state.ApplyBlock(eventCache, proxyApp, block, meta.BlockID.PartsHeader, mempool); err != nil {
-		return nil, err
+	blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, sm.MockMempool{}, sm.MockEvidencePool{})
+
+	var err error
+	state, err = blockExec.ApplyBlock(state, meta.BlockID, block)
+	if err != nil {
+		return sm.State{}, err
 	}
 
-	h.nBlocks += 1
+	h.nBlocks++
 
-	return h.state.AppHash, nil
+	return state, nil
 }
 
-func (h *Handshaker) checkAppHash(appHash []byte) error {
-	if !bytes.Equal(h.state.AppHash, appHash) {
-		panic(errors.New(cmn.Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)).Error())
-		return nil
+func checkAppHash(state sm.State, appHash []byte) error {
+	if !bytes.Equal(state.AppHash, appHash) {
+		panic(fmt.Errorf("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, state.AppHash).Error())
 	}
 	return nil
 }
@@ -370,7 +438,10 @@ func newMockProxyApp(appHash []byte, abciResponses *sm.ABCIResponses) proxy.AppC
 		abciResponses: abciResponses,
 	})
 	cli, _ := clientCreator.NewABCIClient()
-	cli.Start()
+	err := cli.Start()
+	if err != nil {
+		panic(err)
+	}
 	return proxy.NewAppConnConsensus(cli)
 }
 
@@ -382,21 +453,17 @@ type mockProxyApp struct {
 	abciResponses *sm.ABCIResponses
 }
 
-func (mock *mockProxyApp) DeliverTx(tx []byte) abci.Result {
+func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx {
 	r := mock.abciResponses.DeliverTx[mock.txCount]
-	mock.txCount += 1
-	return abci.Result{
-		r.Code,
-		r.Data,
-		r.Log,
-	}
+	mock.txCount++
+	return *r
 }
 
-func (mock *mockProxyApp) EndBlock(height uint64) abci.ResponseEndBlock {
+func (mock *mockProxyApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
 	mock.txCount = 0
-	return mock.abciResponses.EndBlock
+	return *mock.abciResponses.EndBlock
 }
 
-func (mock *mockProxyApp) Commit() abci.Result {
-	return abci.NewResultOK(mock.appHash, "")
+func (mock *mockProxyApp) Commit() abci.ResponseCommit {
+	return abci.ResponseCommit{Data: mock.appHash}
 }

consensus/replay_file.go

@@ -2,12 +2,15 @@ package consensus
 
 import (
 	"bufio"
-	"errors"
+	"context"
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/proxy"
@@ -15,11 +18,18 @@ import (
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+)
+
+const (
+	// event bus subscriber
+	subscriber = "replay-file"
 )
 
 //--------------------------------------------------------
 // replay messages interactively or all at once
 
+// replay the wal file
 func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) {
 	consensusState := newConsensusStateForReplay(config, csConfig)
 
@@ -41,31 +51,46 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 	cs.startForReplay()
 
 	// ensure all new step events are regenerated as expected
-	newStepCh := subscribeToEvent(cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+	newStepCh := make(chan interface{}, 1)
+
+	ctx := context.Background()
+	err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, newStepCh)
+	if err != nil {
+		return errors.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)
+	}
+	defer cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep)
 
 	// just open the file for reading, no need to use wal
-	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
+	fp, err := os.OpenFile(file, os.O_RDONLY, 0600)
 	if err != nil {
 		return err
 	}
 
 	pb := newPlayback(file, fp, cs, cs.state.Copy())
-	defer pb.fp.Close()
+	defer pb.fp.Close() // nolint: errcheck
 
 	var nextN int // apply N msgs in a row
-	for pb.scanner.Scan() {
+	var msg *TimedWALMessage
+	for {
 		if nextN == 0 && console {
 			nextN = pb.replayConsoleLoop()
 		}
 
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {
 			return err
 		}
 
 		if nextN > 0 {
-			nextN -= 1
+			nextN--
 		}
-		pb.count += 1
+		pb.count++
 	}
 	return nil
 }
@@ -76,51 +101,60 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 type playback struct {
 	cs *ConsensusState
 
-	fp      *os.File
-	scanner *bufio.Scanner
-	count   int // how many lines/msgs into the file are we
+	fp    *os.File
+	dec   *WALDecoder
+	count int // how many lines/msgs into the file are we
 
 	// replays can be reset to beginning
-	fileName     string    // so we can close/reopen the file
-	genesisState *sm.State // so the replay session knows where to restart from
+	fileName     string   // so we can close/reopen the file
+	genesisState sm.State // so the replay session knows where to restart from
 }
 
-func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm.State) *playback {
+func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState sm.State) *playback {
 	return &playback{
 		cs:           cs,
 		fp:           fp,
 		fileName:     fileName,
 		genesisState: genState,
-		scanner:      bufio.NewScanner(fp),
+		dec:          NewWALDecoder(fp),
 	}
 }
 
 // go back count steps by resetting the state and running (pb.count - count) steps
 func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
-
 	pb.cs.Stop()
 	pb.cs.Wait()
 
-	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool)
-	newCS.SetEventSwitch(pb.cs.evsw)
+	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec,
+		pb.cs.blockStore, pb.cs.mempool, pb.cs.evpool)
+	newCS.SetEventBus(pb.cs.eventBus)
 	newCS.startForReplay()
 
-	pb.fp.Close()
-	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0666)
+	if err := pb.fp.Close(); err != nil {
+		return err
+	}
+	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0600)
 	if err != nil {
 		return err
 	}
 	pb.fp = fp
-	pb.scanner = bufio.NewScanner(fp)
+	pb.dec = NewWALDecoder(fp)
 	count = pb.count - count
 	fmt.Printf("Reseting from %d to %d\n", pb.count, count)
 	pb.count = 0
 	pb.cs = newCS
-	for i := 0; pb.scanner.Scan() && i < count; i++ {
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+	var msg *TimedWALMessage
+	for i := 0; i < count; i++ {
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
 			return err
 		}
-		pb.count += 1
+		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {
+			return err
+		}
+		pb.count++
 	}
 	return nil
 }
@@ -164,13 +198,12 @@ func (pb *playback) replayConsoleLoop() int {
 
 			if len(tokens) == 1 {
 				return 0
+			}
+			i, err := strconv.Atoi(tokens[1])
+			if err != nil {
+				fmt.Println("next takes an integer argument")
 			} else {
-				i, err := strconv.Atoi(tokens[1])
-				if err != nil {
-					fmt.Println("next takes an integer argument")
-				} else {
-					return i
-				}
+				return i
 			}
 
 		case "back":
@@ -180,10 +213,20 @@ func (pb *playback) replayConsoleLoop() int {
 			// NOTE: "back" is not supported in the state machine design,
 			// so we restart and replay up to
 
+			ctx := context.Background()
 			// ensure all new step events are regenerated as expected
-			newStepCh := subscribeToEvent(pb.cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+			newStepCh := make(chan interface{}, 1)
+
+			err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, newStepCh)
+			if err != nil {
+				cmn.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep))
+			}
+			defer pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep)
+
 			if len(tokens) == 1 {
-				pb.replayReset(1, newStepCh)
+				if err := pb.replayReset(1, newStepCh); err != nil {
+					pb.cs.Logger.Error("Replay reset error", "err", err)
+				}
 			} else {
 				i, err := strconv.Atoi(tokens[1])
 				if err != nil {
@@ -191,7 +234,9 @@ func (pb *playback) replayConsoleLoop() int {
 				} else if i > pb.count {
 					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
 				} else {
-					pb.replayReset(i, newStepCh)
+					if err := pb.replayReset(i, newStepCh); err != nil {
+						pb.cs.Logger.Error("Replay reset error", "err", err)
+					}
 				}
 			}
 
@@ -218,7 +263,7 @@ func (pb *playback) replayConsoleLoop() int {
 				case "locked_block":
 					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
 				case "votes":
-					fmt.Println(rs.Votes.StringIndented("    "))
+					fmt.Println(rs.Votes.StringIndented("  "))
 
 				default:
 					fmt.Println("Unknown option", tokens[1])
@@ -235,33 +280,42 @@ func (pb *playback) replayConsoleLoop() int {
 
 // convenience for replay mode
 func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *ConsensusState {
+	dbType := dbm.DBBackendType(config.DBBackend)
 	// Get BlockStore
-	blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir())
+	blockStoreDB := dbm.NewDB("blockstore", dbType, config.DBDir())
 	blockStore := bc.NewBlockStore(blockStoreDB)
 
 	// Get State
-	stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir())
-	state, err := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
+	stateDB := dbm.NewDB("state", dbType, config.DBDir())
+	gdoc, err := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		cmn.Exit(err.Error())
+	}
+	state, err := sm.MakeGenesisState(gdoc)
 	if err != nil {
 		cmn.Exit(err.Error())
 	}
 
 	// Create proxyAppConn connection (consensus, mempool, query)
 	clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
-	proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore))
-	_, err = proxyApp.Start()
+	proxyApp := proxy.NewAppConns(clientCreator,
+		NewHandshaker(stateDB, state, blockStore, gdoc))
+	err = proxyApp.Start()
 	if err != nil {
 		cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err))
 	}
 
-	// Make event switch
-	eventSwitch := types.NewEventSwitch()
-	if _, err := eventSwitch.Start(); err != nil {
-		cmn.Exit(cmn.Fmt("Failed to start event switch: %v", err))
+	eventBus := types.NewEventBus()
+	if err := eventBus.Start(); err != nil {
+		cmn.Exit(cmn.Fmt("Failed to start event bus: %v", err))
 	}
 
-	consensusState := NewConsensusState(csConfig, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{})
+	mempool, evpool := sm.MockMempool{}, sm.MockEvidencePool{}
+	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 
-	consensusState.SetEventSwitch(eventSwitch)
+	consensusState := NewConsensusState(csConfig, state.Copy(), blockExec,
+		blockStore, mempool, evpool)
+
+	consensusState.SetEventBus(eventBus)
 	return consensusState
 }

consensus/replay_test.go

@@ -2,32 +2,39 @@ package consensus
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
-	"strings"
+	"runtime"
 	"testing"
 	"time"
 
-	"github.com/tendermint/abci/example/dummy"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/abci/types"
 	crypto "github.com/tendermint/go-crypto"
-	wire "github.com/tendermint/go-wire"
+	auto "github.com/tendermint/tmlibs/autofile"
 	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
 
 	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 	"github.com/tendermint/tmlibs/log"
 )
 
+var consensusReplayConfig *cfg.Config
+
 func init() {
-	config = ResetConfig("consensus_replay_test")
+	consensusReplayConfig = ResetConfig("consensus_replay_test")
 }
 
 // These tests ensure we can always recover from failure at any part of the consensus process.
@@ -38,14 +45,6 @@ func init() {
 // the `Handshake Tests` are for failures in applying the block.
 // With the help of the WAL, we can recover from it all!
 
-// NOTE: Files in this dir are generated by running the `build.sh` therein.
-// It's a simple way to generate wals for a single block, or multiple blocks, with random transactions,
-// and different part sizes. The output is not deterministic, and the stepChanges may need to be adjusted
-// after running it (eg. sometimes small_block2 will have 5 block parts, sometimes 6).
-// It should only have to be re-run if there is some breaking change to the consensus data structures (eg. blocks, votes)
-// or to the behaviour of the app (eg. computes app hash differently)
-var data_dir = path.Join(cmn.GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
-
 //------------------------------------------------------------------------------------------
 // WAL Tests
 
@@ -53,221 +52,220 @@ var data_dir = path.Join(cmn.GoPath, "src/github.com/tendermint/tendermint/conse
 // and which ones we need the wal for - then we'd also be able to only flush the
 // wal writer when we need to, instead of with every message.
 
-// the priv validator changes step at these lines for a block with 1 val and 1 part
-var baseStepChanges = []int{3, 6, 8}
+func startNewConsensusStateAndWaitForBlock(t *testing.T, lastBlockHeight int64, blockDB dbm.DB, stateDB dbm.DB) {
+	logger := log.TestingLogger()
+	state, _ := sm.LoadStateFromDBOrGenesisFile(stateDB, consensusReplayConfig.GenesisFile())
+	privValidator := loadPrivValidator(consensusReplayConfig)
+	cs := newConsensusStateWithConfigAndBlockStore(consensusReplayConfig, state, privValidator, kvstore.NewKVStoreApplication(), blockDB)
+	cs.SetLogger(logger)
 
-// test recovery from each line in each testCase
-var testCases = []*testCase{
-	newTestCase("empty_block", baseStepChanges),   // empty block (has 1 block part)
-	newTestCase("small_block1", baseStepChanges),  // small block with txs in 1 block part
-	newTestCase("small_block2", []int{3, 11, 13}), // small block with txs across 6 smaller block parts
-}
+	bytes, _ := ioutil.ReadFile(cs.config.WalFile())
+	// fmt.Printf("====== WAL: \n\r%s\n", bytes)
+	t.Logf("====== WAL: \n\r%X\n", bytes)
 
-type testCase struct {
-	name    string
-	log     string       //full cs wal
-	stepMap map[int]int8 // map lines of log to privval step
+	err := cs.Start()
+	require.NoError(t, err)
+	defer cs.Stop()
 
-	proposeLine   int
-	prevoteLine   int
-	precommitLine int
-}
-
-func newTestCase(name string, stepChanges []int) *testCase {
-	if len(stepChanges) != 3 {
-		panic(cmn.Fmt("a full wal has 3 step changes! Got array %v", stepChanges))
-	}
-	return &testCase{
-		name:    name,
-		log:     readWAL(path.Join(data_dir, name+".cswal")),
-		stepMap: newMapFromChanges(stepChanges),
-
-		proposeLine:   stepChanges[0],
-		prevoteLine:   stepChanges[1],
-		precommitLine: stepChanges[2],
-	}
-}
-
-func newMapFromChanges(changes []int) map[int]int8 {
-	changes = append(changes, changes[2]+1) // so we add the last step change to the map
-	m := make(map[int]int8)
-	var count int
-	for changeNum, nextChange := range changes {
-		for ; count < nextChange; count++ {
-			m[count] = int8(changeNum)
-		}
-	}
-	return m
-}
-
-func readWAL(p string) string {
-	b, err := ioutil.ReadFile(p)
-	if err != nil {
-		panic(err)
-	}
-	return string(b)
-}
-
-func writeWAL(walMsgs string) string {
-	tempDir := os.TempDir()
-	walDir := path.Join(tempDir, "/wal"+cmn.RandStr(12))
-	walFile := path.Join(walDir, "wal")
-	// Create WAL directory
-	err := cmn.EnsureDir(walDir, 0700)
-	if err != nil {
-		panic(err)
-	}
-	// Write the needed WAL to file
-	err = cmn.WriteFile(walFile, []byte(walMsgs), 0600)
-	if err != nil {
-		panic(err)
-	}
-	return walFile
-}
-
-func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
-	after := time.After(time.Second * 10)
+	// This is just a signal that we haven't halted; its not something contained
+	// in the WAL itself. Assuming the consensus state is running, replay of any
+	// WAL, including the empty one, should eventually be followed by a new
+	// block, or else something is wrong.
+	newBlockCh := make(chan interface{}, 1)
+	err = cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, newBlockCh)
+	require.NoError(t, err)
 	select {
 	case <-newBlockCh:
-	case <-after:
-		panic(cmn.Fmt("Timed out waiting for new block for case '%s' line %d", thisCase.name, i))
+	case <-time.After(60 * time.Second):
+		t.Fatalf("Timed out waiting for new block (see trace above)")
 	}
 }
 
-func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{},
-	thisCase *testCase, i int) {
+func sendTxs(cs *ConsensusState, ctx context.Context) {
+	for i := 0; i < 256; i++ {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			tx := []byte{byte(i)}
+			cs.mempool.CheckTx(tx, nil)
+			i++
+		}
+	}
+}
 
-	cs.config.SetWalFile(walFile)
-	started, err := cs.Start()
-	if err != nil {
-		t.Fatalf("Cannot start consensus: %v", err)
+// TestWALCrash uses crashing WAL to test we can recover from any WAL failure.
+func TestWALCrash(t *testing.T) {
+	testCases := []struct {
+		name         string
+		initFn       func(dbm.DB, *ConsensusState, context.Context)
+		heightToStop int64
+	}{
+		{"empty block",
+			func(stateDB dbm.DB, cs *ConsensusState, ctx context.Context) {},
+			1},
+		{"block with a smaller part size",
+			func(stateDB dbm.DB, cs *ConsensusState, ctx context.Context) {
+				// XXX: is there a better way to change BlockPartSizeBytes?
+				cs.state.ConsensusParams.BlockPartSizeBytes = 512
+				sm.SaveState(stateDB, cs.state)
+				go sendTxs(cs, ctx)
+			},
+			1},
+		{"many non-empty blocks",
+			func(stateDB dbm.DB, cs *ConsensusState, ctx context.Context) {
+				go sendTxs(cs, ctx)
+			},
+			3},
 	}
-	if !started {
-		t.Error("Consensus did not start")
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			crashWALandCheckLiveness(t, tc.initFn, tc.heightToStop)
+		})
 	}
-	// Wait to make a new block.
-	// This is just a signal that we haven't halted; its not something contained in the WAL itself.
-	// Assuming the consensus state is running, replay of any WAL, including the empty one,
-	// should eventually be followed by a new block, or else something is wrong
-	waitForBlock(newBlockCh, thisCase, i)
-	cs.evsw.Stop()
-	cs.Stop()
+}
+
+func crashWALandCheckLiveness(t *testing.T, initFn func(dbm.DB, *ConsensusState, context.Context), heightToStop int64) {
+	walPaniced := make(chan error)
+	crashingWal := &crashingWAL{panicCh: walPaniced, heightToStop: heightToStop}
+
+	i := 1
 LOOP:
 	for {
+		// fmt.Printf("====== LOOP %d\n", i)
+		t.Logf("====== LOOP %d\n", i)
+
+		// create consensus state from a clean slate
+		logger := log.NewNopLogger()
+		stateDB := dbm.NewMemDB()
+		state, _ := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile())
+		privValidator := loadPrivValidator(consensusReplayConfig)
+		blockDB := dbm.NewMemDB()
+		cs := newConsensusStateWithConfigAndBlockStore(consensusReplayConfig, state, privValidator, kvstore.NewKVStoreApplication(), blockDB)
+		cs.SetLogger(logger)
+
+		// start sending transactions
+		ctx, cancel := context.WithCancel(context.Background())
+		initFn(stateDB, cs, ctx)
+
+		// clean up WAL file from the previous iteration
+		walFile := cs.config.WalFile()
+		os.Remove(walFile)
+
+		// set crashing WAL
+		csWal, err := cs.OpenWAL(walFile)
+		require.NoError(t, err)
+		crashingWal.next = csWal
+		// reset the message counter
+		crashingWal.msgIndex = 1
+		cs.wal = crashingWal
+
+		// start consensus state
+		err = cs.Start()
+		require.NoError(t, err)
+
+		i++
+
 		select {
-		case <-newBlockCh:
-		default:
-			break LOOP
-		}
-	}
-	cs.Wait()
-}
+		case err := <-walPaniced:
+			t.Logf("WAL paniced: %v", err)
 
-func toPV(pv types.PrivValidator) *types.PrivValidatorFS {
-	return pv.(*types.PrivValidatorFS)
-}
+			// make sure we can make blocks after a crash
+			startNewConsensusStateAndWaitForBlock(t, cs.Height, blockDB, stateDB)
 
-func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) {
-	t.Log("-------------------------------------")
-	t.Logf("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter)
+			// stop consensus state and transactions sender (initFn)
+			cs.Stop()
+			cancel()
 
-	lineStep := nLines
-	if crashAfter {
-		lineStep -= 1
-	}
-
-	split := strings.Split(thisCase.log, "\n")
-	lastMsg := split[nLines]
-
-	// we write those lines up to (not including) one with the signature
-	walFile := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
-
-	cs := fixedConsensusStateDummy()
-
-	// set the last step according to when we crashed vs the wal
-	toPV(cs.privValidator).LastHeight = 1 // first block
-	toPV(cs.privValidator).LastStep = thisCase.stepMap[lineStep]
-
-	t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).LastStep)
-
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
-
-	return cs, newBlockCh, lastMsg, walFile
-}
-
-func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
-	var err error
-	var msg TimedWALMessage
-	wire.ReadJSON(&msg, []byte(walMsg), &err)
-	if err != nil {
-		t.Fatalf("Error reading json data: %v", err)
-	}
-	return msg
-}
-
-//-----------------------------------------------
-// Test the log at every iteration, and set the privVal last step
-// as if the log was written after signing, before the crash
-
-func TestWALCrashAfterWrite(t *testing.T) {
-	for _, thisCase := range testCases {
-		split := strings.Split(thisCase.log, "\n")
-		for i := 0; i < len(split)-1; i++ {
-			cs, newBlockCh, _, walFile := setupReplayTest(t, thisCase, i+1, true)
-			runReplayTest(t, cs, walFile, newBlockCh, thisCase, i+1)
+			// if we reached the required height, exit
+			if _, ok := err.(ReachedHeightToStopError); ok {
+				break LOOP
+			}
+		case <-time.After(10 * time.Second):
+			t.Fatal("WAL did not panic for 10 seconds (check the log)")
 		}
 	}
 }
 
-//-----------------------------------------------
-// Test the log as if we crashed after signing but before writing.
-// This relies on privValidator.LastSignature being set
+// crashingWAL is a WAL which crashes or rather simulates a crash during Save
+// (before and after). It remembers a message for which we last panicked
+// (lastPanicedForMsgIndex), so we don't panic for it in subsequent iterations.
+type crashingWAL struct {
+	next         WAL
+	panicCh      chan error
+	heightToStop int64
 
-func TestWALCrashBeforeWritePropose(t *testing.T) {
-	for _, thisCase := range testCases {
-		lineNum := thisCase.proposeLine
-		// setup replay test where last message is a proposal
-		cs, newBlockCh, proposalMsg, walFile := setupReplayTest(t, thisCase, lineNum, false)
-		msg := readTimedWALMessage(t, proposalMsg)
-		proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage)
-		// Set LastSig
-		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal)
-		toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature
-		runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
+	msgIndex               int // current message index
+	lastPanicedForMsgIndex int // last message for which we panicked
+}
+
+// WALWriteError indicates a WAL crash.
+type WALWriteError struct {
+	msg string
+}
+
+func (e WALWriteError) Error() string {
+	return e.msg
+}
+
+// ReachedHeightToStopError indicates we've reached the required consensus
+// height and may exit.
+type ReachedHeightToStopError struct {
+	height int64
+}
+
+func (e ReachedHeightToStopError) Error() string {
+	return fmt.Sprintf("reached height to stop %d", e.height)
+}
+
+// Write simulate WAL's crashing by sending an error to the panicCh and then
+// exiting the cs.receiveRoutine.
+func (w *crashingWAL) Write(m WALMessage) {
+	if endMsg, ok := m.(EndHeightMessage); ok {
+		if endMsg.Height == w.heightToStop {
+			w.panicCh <- ReachedHeightToStopError{endMsg.Height}
+			runtime.Goexit()
+		} else {
+			w.next.Write(m)
+		}
+		return
+	}
+
+	if w.msgIndex > w.lastPanicedForMsgIndex {
+		w.lastPanicedForMsgIndex = w.msgIndex
+		_, file, line, _ := runtime.Caller(1)
+		w.panicCh <- WALWriteError{fmt.Sprintf("failed to write %T to WAL (fileline: %s:%d)", m, file, line)}
+		runtime.Goexit()
+	} else {
+		w.msgIndex++
+		w.next.Write(m)
 	}
 }
 
-func TestWALCrashBeforeWritePrevote(t *testing.T) {
-	for _, thisCase := range testCases {
-		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal())
-	}
+func (w *crashingWAL) WriteSync(m WALMessage) {
+	w.Write(m)
 }
 
-func TestWALCrashBeforeWritePrecommit(t *testing.T) {
-	for _, thisCase := range testCases {
-		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka())
-	}
+func (w *crashingWAL) Group() *auto.Group { return w.next.Group() }
+func (w *crashingWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+	return w.next.SearchForEndHeight(height, options)
 }
 
-func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) {
-	// setup replay test where last message is a vote
-	cs, newBlockCh, voteMsg, walFile := setupReplayTest(t, thisCase, lineNum, false)
-	types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) {
-		msg := readTimedWALMessage(t, voteMsg)
-		vote := msg.Msg.(msgInfo).Msg.(*VoteMessage)
-		// Set LastSig
-		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote)
-		toPV(cs.privValidator).LastSignature = vote.Vote.Signature
-	})
-	runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
-}
+func (w *crashingWAL) Start() error { return w.next.Start() }
+func (w *crashingWAL) Stop() error  { return w.next.Stop() }
+func (w *crashingWAL) Wait()        { w.next.Wait() }
 
 //------------------------------------------------------------------------------------------
 // Handshake Tests
 
+const (
+	NUM_BLOCKS = 6
+)
+
 var (
-	NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal
-	mempool    = types.MockMempool{}
+	mempool = sm.MockMempool{}
+	evpool  = sm.MockEvidencePool{}
 )
 
 //---------------------------------------
@@ -306,57 +304,76 @@ func TestHandshakeReplayNone(t *testing.T) {
 	}
 }
 
+func tempWALWithData(data []byte) string {
+	walFile, err := ioutil.TempFile("", "wal")
+	if err != nil {
+		panic(fmt.Errorf("failed to create temp WAL file: %v", err))
+	}
+	_, err = walFile.Write(data)
+	if err != nil {
+		panic(fmt.Errorf("failed to write to temp WAL file: %v", err))
+	}
+	if err := walFile.Close(); err != nil {
+		panic(fmt.Errorf("failed to close temp WAL file: %v", err))
+	}
+	return walFile.Name()
+}
+
 // Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
 func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	config := ResetConfig("proxy_test_")
 
-	// copy the many_blocks file
-	walBody, err := cmn.ReadFile(path.Join(data_dir, "many_blocks.cswal"))
+	walBody, err := WALWithNBlocks(NUM_BLOCKS)
 	if err != nil {
 		t.Fatal(err)
 	}
-	walFile := writeWAL(string(walBody))
+	walFile := tempWALWithData(walBody)
 	config.Consensus.SetWalFile(walFile)
 
-	privVal := types.LoadPrivValidatorFS(config.PrivValidatorFile())
+	privVal := privval.LoadFilePV(config.PrivValidatorFile())
 
-	wal, err := NewWAL(walFile, false)
+	wal, err := NewWAL(walFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 	wal.SetLogger(log.TestingLogger())
-	if _, err := wal.Start(); err != nil {
+	if err := wal.Start(); err != nil {
 		t.Fatal(err)
 	}
+	defer wal.Stop()
+
 	chain, commits, err := makeBlockchainFromWAL(wal)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
 
-	state, store := stateAndStore(config, privVal.GetPubKey())
+	stateDB, state, store := stateAndStore(config, privVal.GetPubKey())
 	store.chain = chain
 	store.commits = commits
 
 	// run the chain through state.ApplyBlock to build up the tendermint state
-	latestAppHash := buildTMStateFromChain(config, state, chain, mode)
+	state = buildTMStateFromChain(config, stateDB, state, chain, mode)
+	latestAppHash := state.AppHash
 
 	// make a new client creator
-	dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "2"))
-	clientCreator2 := proxy.NewLocalClientCreator(dummyApp)
+	kvstoreApp := kvstore.NewPersistentKVStoreApplication(path.Join(config.DBDir(), "2"))
+	clientCreator2 := proxy.NewLocalClientCreator(kvstoreApp)
 	if nBlocks > 0 {
 		// run nBlocks against a new client to build up the app state.
 		// use a throwaway tendermint state
 		proxyApp := proxy.NewAppConns(clientCreator2, nil)
-		state, _ := stateAndStore(config, privVal.GetPubKey())
-		buildAppStateFromChain(proxyApp, state, chain, nBlocks, mode)
+		stateDB, state, _ := stateAndStore(config, privVal.GetPubKey())
+		buildAppStateFromChain(proxyApp, stateDB, state, chain, nBlocks, mode)
 	}
 
 	// now start the app using the handshake - it should sync
-	handshaker := NewHandshaker(state, store)
+	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	handshaker := NewHandshaker(stateDB, state, store, genDoc)
 	proxyApp := proxy.NewAppConns(clientCreator2, handshaker)
-	if _, err := proxyApp.Start(); err != nil {
+	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
+	defer proxyApp.Stop()
 
 	// get the latest app hash from the app
 	res, err := proxyApp.Query().InfoSync(abci.RequestInfo{""})
@@ -371,9 +388,9 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 
 	expectedBlocksToSync := NUM_BLOCKS - nBlocks
 	if nBlocks == NUM_BLOCKS && mode > 0 {
-		expectedBlocksToSync += 1
+		expectedBlocksToSync++
 	} else if nBlocks > 0 && mode == 1 {
-		expectedBlocksToSync += 1
+		expectedBlocksToSync++
 	}
 
 	if handshaker.NBlocks() != expectedBlocksToSync {
@@ -381,189 +398,208 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	}
 }
 
-func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) {
-	testPartSize := st.Params().BlockPartSizeBytes
-	err := st.ApplyBlock(nil, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool)
+func applyBlock(stateDB dbm.DB, st sm.State, blk *types.Block, proxyApp proxy.AppConns) sm.State {
+	testPartSize := st.ConsensusParams.BlockPartSizeBytes
+	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
+
+	blkID := types.BlockID{blk.Hash(), blk.MakePartSet(testPartSize).Header()}
+	newState, err := blockExec.ApplyBlock(st, blkID, blk)
 	if err != nil {
 		panic(err)
 	}
+	return newState
 }
 
-func buildAppStateFromChain(proxyApp proxy.AppConns,
-	state *sm.State, chain []*types.Block, nBlocks int, mode uint) {
+func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB,
+	state sm.State, chain []*types.Block, nBlocks int, mode uint) {
 	// start a new app without handshake, play nBlocks blocks
-	if _, err := proxyApp.Start(); err != nil {
+	if err := proxyApp.Start(); err != nil {
+		panic(err)
+	}
+	defer proxyApp.Stop()
+
+	validators := types.TM2PB.Validators(state.Validators)
+	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{
+		Validators: validators,
+	}); err != nil {
 		panic(err)
 	}
 
-	validators := types.TM2PB.Validators(state.Validators)
-	proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators})
-
-	defer proxyApp.Stop()
 	switch mode {
 	case 0:
 		for i := 0; i < nBlocks; i++ {
 			block := chain[i]
-			applyBlock(state, block, proxyApp)
+			state = applyBlock(stateDB, state, block, proxyApp)
 		}
 	case 1, 2:
 		for i := 0; i < nBlocks-1; i++ {
 			block := chain[i]
-			applyBlock(state, block, proxyApp)
+			state = applyBlock(stateDB, state, block, proxyApp)
 		}
 
 		if mode == 2 {
-			// update the dummy height and apphash
+			// update the kvstore height and apphash
 			// as if we ran commit but not
-			applyBlock(state, chain[nBlocks-1], proxyApp)
+			state = applyBlock(stateDB, state, chain[nBlocks-1], proxyApp)
 		}
 	}
 
 }
 
-func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.Block, mode uint) []byte {
+func buildTMStateFromChain(config *cfg.Config, stateDB dbm.DB, state sm.State, chain []*types.Block, mode uint) sm.State {
 	// run the whole chain against this client to build up the tendermint state
-	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "1")))
+	clientCreator := proxy.NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(path.Join(config.DBDir(), "1")))
 	proxyApp := proxy.NewAppConns(clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
-	if _, err := proxyApp.Start(); err != nil {
+	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
 	defer proxyApp.Stop()
 
 	validators := types.TM2PB.Validators(state.Validators)
-	proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators})
-
-	var latestAppHash []byte
+	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{
+		Validators: validators,
+	}); err != nil {
+		panic(err)
+	}
 
 	switch mode {
 	case 0:
 		// sync right up
 		for _, block := range chain {
-			applyBlock(state, block, proxyApp)
+			state = applyBlock(stateDB, state, block, proxyApp)
 		}
 
-		latestAppHash = state.AppHash
 	case 1, 2:
 		// sync up to the penultimate as if we stored the block.
 		// whether we commit or not depends on the appHash
 		for _, block := range chain[:len(chain)-1] {
-			applyBlock(state, block, proxyApp)
+			state = applyBlock(stateDB, state, block, proxyApp)
 		}
 
 		// apply the final block to a state copy so we can
 		// get the right next appHash but keep the state back
-		stateCopy := state.Copy()
-		applyBlock(stateCopy, chain[len(chain)-1], proxyApp)
-		latestAppHash = stateCopy.AppHash
+		applyBlock(stateDB, state, chain[len(chain)-1], proxyApp)
 	}
 
-	return latestAppHash
+	return state
 }
 
 //--------------------------
 // utils for making blocks
 
-func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
+func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 	// Search for height marker
-	gr, found, err := wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(0))
+	gr, found, err := wal.SearchForEndHeight(0, &WALSearchOptions{})
 	if err != nil {
 		return nil, nil, err
 	}
 	if !found {
 		return nil, nil, errors.New(cmn.Fmt("WAL does not contain height %d.", 1))
 	}
-	defer gr.Close()
+	defer gr.Close() // nolint: errcheck
 
 	// log.Notice("Build a blockchain by reading from the WAL")
 
-	var blockParts *types.PartSet
 	var blocks []*types.Block
 	var commits []*types.Commit
-	for {
-		line, err := gr.ReadLine()
-		if err != nil {
-			if err == io.EOF {
-				break
-			} else {
-				return nil, nil, err
-			}
-		}
 
-		piece, err := readPieceFromWAL([]byte(line))
-		if err != nil {
+	var thisBlockParts *types.PartSet
+	var thisBlockCommit *types.Commit
+	var height int64
+
+	dec := NewWALDecoder(gr)
+	for {
+		msg, err := dec.Decode()
+		if err == io.EOF {
+			break
+		} else if err != nil {
 			return nil, nil, err
 		}
+
+		piece := readPieceFromWAL(msg)
 		if piece == nil {
 			continue
 		}
 
 		switch p := piece.(type) {
-		case *types.PartSetHeader:
+		case EndHeightMessage:
 			// if its not the first one, we have a full block
-			if blockParts != nil {
-				var n int
-				block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), 0, &n, &err).(*types.Block)
+			if thisBlockParts != nil {
+				var block = new(types.Block)
+				_, err = cdc.UnmarshalBinaryReader(thisBlockParts.GetReader(), block, 0)
+				if err != nil {
+					panic(err)
+				}
+				if block.Height != height+1 {
+					panic(cmn.Fmt("read bad block from wal. got height %d, expected %d", block.Height, height+1))
+				}
+				commitHeight := thisBlockCommit.Precommits[0].Height
+				if commitHeight != height+1 {
+					panic(cmn.Fmt("commit doesnt match. got height %d, expected %d", commitHeight, height+1))
+				}
 				blocks = append(blocks, block)
+				commits = append(commits, thisBlockCommit)
+				height++
 			}
-			blockParts = types.NewPartSetFromHeader(*p)
+		case *types.PartSetHeader:
+			thisBlockParts = types.NewPartSetFromHeader(*p)
 		case *types.Part:
-			_, err := blockParts.AddPart(p, false)
+			_, err := thisBlockParts.AddPart(p)
 			if err != nil {
 				return nil, nil, err
 			}
 		case *types.Vote:
 			if p.Type == types.VoteTypePrecommit {
-				commit := &types.Commit{
+				thisBlockCommit = &types.Commit{
 					BlockID:    p.BlockID,
 					Precommits: []*types.Vote{p},
 				}
-				commits = append(commits, commit)
 			}
 		}
 	}
 	// grab the last block too
-	var n int
-	block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), 0, &n, &err).(*types.Block)
+	var block = new(types.Block)
+	_, err = cdc.UnmarshalBinaryReader(thisBlockParts.GetReader(), block, 0)
+	if err != nil {
+		panic(err)
+	}
+	if block.Height != height+1 {
+		panic(cmn.Fmt("read bad block from wal. got height %d, expected %d", block.Height, height+1))
+	}
+	commitHeight := thisBlockCommit.Precommits[0].Height
+	if commitHeight != height+1 {
+		panic(cmn.Fmt("commit doesnt match. got height %d, expected %d", commitHeight, height+1))
+	}
 	blocks = append(blocks, block)
+	commits = append(commits, thisBlockCommit)
 	return blocks, commits, nil
 }
 
-func readPieceFromWAL(msgBytes []byte) (interface{}, error) {
-	// Skip over empty and meta lines
-	if len(msgBytes) == 0 || msgBytes[0] == '#' {
-		return nil, nil
-	}
-	var err error
-	var msg TimedWALMessage
-	wire.ReadJSON(&msg, msgBytes, &err)
-	if err != nil {
-		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
-		return nil, fmt.Errorf("Error reading json data: %v", err)
-	}
-
+func readPieceFromWAL(msg *TimedWALMessage) interface{} {
 	// for logging
 	switch m := msg.Msg.(type) {
 	case msgInfo:
 		switch msg := m.Msg.(type) {
 		case *ProposalMessage:
-			return &msg.Proposal.BlockPartsHeader, nil
+			return &msg.Proposal.BlockPartsHeader
 		case *BlockPartMessage:
-			return msg.Part, nil
+			return msg.Part
 		case *VoteMessage:
-			return msg.Vote, nil
+			return msg.Vote
 		}
+	case EndHeightMessage:
+		return m
 	}
-	return nil, nil
+
+	return nil
 }
 
 // fresh state and mock store
-func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) {
+func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (dbm.DB, sm.State, *mockBlockStore) {
 	stateDB := dbm.NewMemDB()
-	state, _ := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
-	state.SetLogger(log.TestingLogger().With("module", "state"))
-
-	store := NewMockBlockStore(config, state.Params())
-	return state, store
+	state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile())
+	store := NewMockBlockStore(config, state.ConsensusParams)
+	return stateDB, state, store
 }
 
 //----------------------------------
@@ -581,21 +617,71 @@ func NewMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBl
 	return &mockBlockStore{config, params, nil, nil}
 }
 
-func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
-func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
-func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
+func (bs *mockBlockStore) Height() int64                       { return int64(len(bs.chain)) }
+func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain[height-1] }
+func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
 	block := bs.chain[height-1]
 	return &types.BlockMeta{
 		BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.params.BlockPartSizeBytes).Header()},
 		Header:  block.Header,
 	}
 }
-func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
+func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil }
 func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
 }
-func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
+func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit {
 	return bs.commits[height-1]
 }
-func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
+func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit {
 	return bs.commits[height-1]
 }
+
+//----------------------------------------
+
+func TestInitChainUpdateValidators(t *testing.T) {
+	val, _ := types.RandValidator(true, 10)
+	vals := types.NewValidatorSet([]*types.Validator{val})
+	app := &initChainApp{vals: types.TM2PB.Validators(vals)}
+	clientCreator := proxy.NewLocalClientCreator(app)
+
+	config := ResetConfig("proxy_test_")
+	privVal := privval.LoadFilePV(config.PrivValidatorFile())
+	stateDB, state, store := stateAndStore(config, privVal.GetPubKey())
+
+	oldValAddr := state.Validators.Validators[0].Address
+
+	// now start the app using the handshake - it should sync
+	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	handshaker := NewHandshaker(stateDB, state, store, genDoc)
+	proxyApp := proxy.NewAppConns(clientCreator, handshaker)
+	if err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+	defer proxyApp.Stop()
+
+	// reload the state, check the validator set was updated
+	state = sm.LoadState(stateDB)
+
+	newValAddr := state.Validators.Validators[0].Address
+	expectValAddr := val.Address
+	assert.NotEqual(t, oldValAddr, newValAddr)
+	assert.Equal(t, newValAddr, expectValAddr)
+}
+
+func newInitChainApp(vals []abci.Validator) *initChainApp {
+	return &initChainApp{
+		vals: vals,
+	}
+}
+
+// returns the vals on InitChain
+type initChainApp struct {
+	abci.BaseApplication
+	vals []abci.Validator
+}
+
+func (ica *initChainApp) InitChain(req abci.RequestInitChain) abci.ResponseInitChain {
+	return abci.ResponseInitChain{
+		Validators: ica.vals,
+	}
+}

consensus/state_test.go

@@ -2,13 +2,16 @@ package consensus
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"testing"
 	"time"
 
 	cstypes "github.com/tendermint/tendermint/consensus/types"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 )
 
 func init() {
@@ -52,12 +55,12 @@ x * TestHalt1 - if we see +2/3 precommits after timing out into new round, we sh
 //----------------------------------------------------------------------------------------------------
 // ProposeSuite
 
-func TestProposerSelection0(t *testing.T) {
+func TestStateProposerSelection0(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	height, round := cs1.Height, cs1.Round
 
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
 
 	startTestRound(cs1, height, round)
 
@@ -86,10 +89,10 @@ func TestProposerSelection0(t *testing.T) {
 }
 
 // Now let's do it all again, but starting from round 2 instead of 0
-func TestProposerSelection2(t *testing.T) {
+func TestStateProposerSelection2(t *testing.T) {
 	cs1, vss := randConsensusState(4) // test needs more work for more than 3 validators
 
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 
 	// this time we jump in at round 2
 	incrementRound(vss[1:]...)
@@ -115,13 +118,13 @@ func TestProposerSelection2(t *testing.T) {
 }
 
 // a non-validator should timeout into the prevote round
-func TestEnterProposeNoPrivValidator(t *testing.T) {
+func TestStateEnterProposeNoPrivValidator(t *testing.T) {
 	cs, _ := randConsensusState(1)
 	cs.SetPrivValidator(nil)
 	height, round := cs.Height, cs.Round
 
 	// Listen for propose timeout event
-	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
 
 	startTestRound(cs, height, round)
 
@@ -140,14 +143,14 @@ func TestEnterProposeNoPrivValidator(t *testing.T) {
 }
 
 // a validator should not timeout of the prevote round (TODO: unless the block is really big!)
-func TestEnterProposeYesPrivValidator(t *testing.T) {
+func TestStateEnterProposeYesPrivValidator(t *testing.T) {
 	cs, _ := randConsensusState(1)
 	height, round := cs.Height, cs.Round
 
 	// Listen for propose timeout event
 
-	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	proposalCh := subscribeToEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), 1)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
+	proposalCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal)
 
 	cs.enterNewRound(height, round)
 	cs.startRoutines(3)
@@ -176,15 +179,15 @@ func TestEnterProposeYesPrivValidator(t *testing.T) {
 	}
 }
 
-func TestBadProposal(t *testing.T) {
+func TestStateBadProposal(t *testing.T) {
 	cs1, vss := randConsensusState(2)
 	height, round := cs1.Height, cs1.Round
 	vs2 := vss[1]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
 
 	propBlock, _ := cs1.createProposalBlock() //changeProposer(t, cs1, vs2)
 
@@ -201,12 +204,14 @@ func TestBadProposal(t *testing.T) {
 	propBlock.AppHash = stateHash
 	propBlockParts := propBlock.MakePartSet(partSize)
 	proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{})
-	if err := vs2.SignProposal(config.ChainID, proposal); err != nil {
+	if err := vs2.SignProposal(config.ChainID(), proposal); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
 
 	// set the proposal block
-	cs1.SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	// start the machine
 	startTestRound(cs1, height, round)
@@ -234,13 +239,21 @@ func TestBadProposal(t *testing.T) {
 // FullRoundSuite
 
 // propose, prevote, and precommit a block
-func TestFullRound1(t *testing.T) {
+func TestStateFullRound1(t *testing.T) {
 	cs, vss := randConsensusState(1)
 	height, round := cs.Height, cs.Round
 
-	voteCh := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 0)
-	propCh := subscribeToEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	newRoundCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewRound(), 1)
+	// NOTE: buffer capacity of 0 ensures we can validate prevote and last commit
+	// before consensus can move to the next height (and cause a race condition)
+	cs.eventBus.Stop()
+	eventBus := types.NewEventBusWithBufferCapacity(0)
+	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+	cs.SetEventBus(eventBus)
+	eventBus.Start()
+
+	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
+	propCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal)
+	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
 
 	startTestRound(cs, height, round)
 
@@ -248,11 +261,9 @@ func TestFullRound1(t *testing.T) {
 
 	// grab proposal
 	re := <-propCh
-	propBlockHash := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState).ProposalBlock.Hash()
+	propBlockHash := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState).ProposalBlock.Hash()
 
 	<-voteCh // wait for prevote
-	// NOTE: voteChan cap of 0 ensures we can complete this
-	// before consensus can move to the next height (and cause a race condition)
 	validatePrevote(t, cs, round, vss[0], propBlockHash)
 
 	<-voteCh // wait for precommit
@@ -264,11 +275,11 @@ func TestFullRound1(t *testing.T) {
 }
 
 // nil is proposed, so prevote and precommit nil
-func TestFullRoundNil(t *testing.T) {
+func TestStateFullRoundNil(t *testing.T) {
 	cs, vss := randConsensusState(1)
 	height, round := cs.Height, cs.Round
 
-	voteCh := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1)
+	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
 
 	cs.enterPrevote(height, round)
 	cs.startRoutines(4)
@@ -282,13 +293,13 @@ func TestFullRoundNil(t *testing.T) {
 
 // run through propose, prevote, precommit commit with two validators
 // where the first validator has to wait for votes from the second
-func TestFullRound2(t *testing.T) {
+func TestStateFullRound2(t *testing.T) {
 	cs1, vss := randConsensusState(2)
 	vs2 := vss[1]
 	height, round := cs1.Height, cs1.Round
 
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
-	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, height, round)
@@ -323,18 +334,18 @@ func TestFullRound2(t *testing.T) {
 
 // two validators, 4 rounds.
 // two vals take turns proposing. val1 locks on first one, precommits nil on everything else
-func TestLockNoPOL(t *testing.T) {
+func TestStateLockNoPOL(t *testing.T) {
 	cs1, vss := randConsensusState(2)
 	vs2 := vss[1]
 	height := cs1.Height
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 
 	/*
 		Round1 (cs1, B) // B B // B B2
@@ -345,7 +356,7 @@ func TestLockNoPOL(t *testing.T) {
 	cs1.startRoutines(0)
 
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -385,7 +396,7 @@ func TestLockNoPOL(t *testing.T) {
 
 	// now we're on a new round and not the proposer, so wait for timeout
 	re = <-timeoutProposeCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	if rs.ProposalBlock != nil {
 		panic("Expected proposal block to be nil")
@@ -398,7 +409,7 @@ func TestLockNoPOL(t *testing.T) {
 	validatePrevote(t, cs1, 1, vss[0], rs.LockedBlock.Hash())
 
 	// add a conflicting prevote from the other validator
-	signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2)
+	signAddVotes(cs1, types.VoteTypePrevote, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2)
 	<-voteCh
 
 	// now we're going to enter prevote again, but with invalid args
@@ -413,7 +424,7 @@ func TestLockNoPOL(t *testing.T) {
 
 	// add conflicting precommit from vs2
 	// NOTE: in practice we should never get to a point where there are precommits for different blocks at the same round
-	signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2)
+	signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2)
 	<-voteCh
 
 	// (note we're entering precommit for a second time this round, but with invalid args
@@ -429,7 +440,7 @@ func TestLockNoPOL(t *testing.T) {
 	incrementRound(vs2)
 
 	re = <-proposalCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	// now we're on a new round and are the proposer
 	if !bytes.Equal(rs.ProposalBlock.Hash(), rs.LockedBlock.Hash()) {
@@ -469,7 +480,9 @@ func TestLockNoPOL(t *testing.T) {
 
 	// now we're on a new round and not the proposer
 	// so set the proposal block
-	cs1.SetProposalAndBlock(prop, propBlock, propBlock.MakePartSet(partSize), "")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlock.MakePartSet(partSize), ""); err != nil {
+		t.Fatal(err)
+	}
 
 	<-proposalCh
 	<-voteCh // prevote
@@ -490,18 +503,18 @@ func TestLockNoPOL(t *testing.T) {
 }
 
 // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka
-func TestLockPOLRelock(t *testing.T) {
+func TestStateLockPOLRelock(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlockHeader(), 1)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader)
 
 	// everything done from perspective of cs1
 
@@ -516,7 +529,7 @@ func TestLockPOLRelock(t *testing.T) {
 
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -546,7 +559,9 @@ func TestLockPOLRelock(t *testing.T) {
 	<-timeoutWaitCh
 
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
-	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	<-newRoundCh
 	t.Log("### ONTO ROUND 1")
@@ -590,9 +605,9 @@ func TestLockPOLRelock(t *testing.T) {
 	discardFromChan(voteCh, 2)
 
 	be := <-newBlockCh
-	b := be.(types.TMEventData).Unwrap().(types.EventDataNewBlockHeader)
+	b := be.(types.EventDataNewBlockHeader)
 	re = <-newRoundCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	if rs.Height != 2 {
 		panic("Expected height to increment")
 	}
@@ -603,17 +618,17 @@ func TestLockPOLRelock(t *testing.T) {
 }
 
 // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka
-func TestLockPOLUnlock(t *testing.T) {
+func TestStateLockPOLUnlock(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// everything done from perspective of cs1
@@ -628,7 +643,7 @@ func TestLockPOLUnlock(t *testing.T) {
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -654,11 +669,13 @@ func TestLockPOLUnlock(t *testing.T) {
 
 	// timeout to new round
 	re = <-timeoutWaitCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	lockedBlockHash := rs.LockedBlock.Hash()
 
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
-	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	<-newRoundCh
 	t.Log("#### ONTO ROUND 1")
@@ -698,23 +715,23 @@ func TestLockPOLUnlock(t *testing.T) {
 // a polka at round 1 but we miss it
 // then a polka at round 2 that we lock on
 // then we see the polka from round 1 but shouldn't unlock
-func TestLockPOLSafety1(t *testing.T) {
+func TestStateLockPOLSafety1(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	propBlock := rs.ProposalBlock
 
 	<-voteCh // prevote
@@ -745,7 +762,9 @@ func TestLockPOLSafety1(t *testing.T) {
 	incrementRound(vs2, vs3, vs4)
 
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
-	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	<-newRoundCh
 	t.Log("### ONTO ROUND 1")
@@ -762,7 +781,7 @@ func TestLockPOLSafety1(t *testing.T) {
 		re = <-proposalCh
 	}
 
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	if rs.LockedBlock != nil {
 		panic("we should not be locked!")
@@ -802,7 +821,7 @@ func TestLockPOLSafety1(t *testing.T) {
 	// we should prevote what we're locked on
 	validatePrevote(t, cs1, 2, vss[0], propBlockHash)
 
-	newStepCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRoundStep(), 1)
+	newStepCh := subscribe(cs1.eventBus, types.EventQueryNewRoundStep)
 
 	// add prevotes from the earlier round
 	addVotes(cs1, prevotes...)
@@ -819,17 +838,17 @@ func TestLockPOLSafety1(t *testing.T) {
 
 // What we want:
 // dont see P0, lock on P1 at R1, dont unlock using P0 at R2
-func TestLockPOLSafety2(t *testing.T) {
+func TestStateLockPOLSafety2(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// the block for R0: gets polkad but we miss it
@@ -857,7 +876,9 @@ func TestLockPOLSafety2(t *testing.T) {
 	startTestRound(cs1, height, 1)
 	<-newRoundCh
 
-	cs1.SetProposalAndBlock(prop1, propBlock1, propBlockParts1, "some peer")
+	if err := cs1.SetProposalAndBlock(prop1, propBlock1, propBlockParts1, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 	<-proposalCh
 
 	<-voteCh // prevote
@@ -879,10 +900,12 @@ func TestLockPOLSafety2(t *testing.T) {
 
 	// in round 2 we see the polkad block from round 0
 	newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0, propBlockID1)
-	if err := vs3.SignProposal(config.ChainID, newProp); err != nil {
+	if err := vs3.SignProposal(config.ChainID(), newProp); err != nil {
+		t.Fatal(err)
+	}
+	if err := cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer"); err != nil {
 		t.Fatal(err)
 	}
-	cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer")
 
 	// Add the pol votes
 	addVotes(cs1, prevotes...)
@@ -914,14 +937,14 @@ func TestLockPOLSafety2(t *testing.T) {
 // TODO: Slashing
 
 /*
-func TestSlashingPrevotes(t *testing.T) {
+func TestStateSlashingPrevotes(t *testing.T) {
 	cs1, vss := randConsensusState(2)
 	vs2 := vss[1]
 
 
-	proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1)
-	newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
@@ -949,14 +972,14 @@ func TestSlashingPrevotes(t *testing.T) {
 	// XXX: Check for existence of Dupeout info
 }
 
-func TestSlashingPrecommits(t *testing.T) {
+func TestStateSlashingPrecommits(t *testing.T) {
 	cs1, vss := randConsensusState(2)
 	vs2 := vss[1]
 
 
-	proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1)
-	newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
@@ -994,23 +1017,23 @@ func TestSlashingPrecommits(t *testing.T) {
 
 // 4 vals.
 // we receive a final precommit after going into next round, but others might have gone to commit already!
-func TestHalt1(t *testing.T) {
+func TestStateHalt1(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.ConsensusParams.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	propBlock := rs.ProposalBlock
 	propBlockParts := propBlock.MakePartSet(partSize)
 
@@ -1033,7 +1056,7 @@ func TestHalt1(t *testing.T) {
 	// timeout to new round
 	<-timeoutWaitCh
 	re = <-newRoundCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	t.Log("### ONTO ROUND 1")
 	/*Round2
@@ -1051,9 +1074,26 @@ func TestHalt1(t *testing.T) {
 	// receiving that precommit should take us straight to commit
 	<-newBlockCh
 	re = <-newRoundCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
+	rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	if rs.Height != 2 {
 		panic("expected height to increment")
 	}
 }
+
+// subscribe subscribes test client to the given query and returns a channel with cap = 1.
+func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan interface{} {
+	out := make(chan interface{}, 1)
+	err := eventBus.Subscribe(context.Background(), testSubscriber, q, out)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, q))
+	}
+	return out
+}
+
+// discardFromChan reads n values from the channel.
+func discardFromChan(ch <-chan interface{}, n int) {
+	for i := 0; i < n; i++ {
+		<-ch
+	}
+}

consensus/test_data/README.md

@@ -1,36 +0,0 @@
-# Generating test data
-
-To generate the data, run `build.sh`. See that script for more details.
-
-Make sure to adjust the stepChanges in the testCases if the number of messages changes.
-This sometimes happens for the `small_block2.cswal`, where the number of block parts changes between 4 and 5.
-
-If you need to change the signatures, you can use a script as follows:
-The privBytes comes from `config/tendermint_test/...`:
-
-```
-package main
-
-import (
-	"encoding/hex"
-	"fmt"
-
-	"github.com/tendermint/go-crypto"
-)
-
-func main() {
-	signBytes, err := hex.DecodeString("7B22636861696E5F6964223A2274656E6465726D696E745F74657374222C22766F7465223A7B22626C6F636B5F68617368223A2242453544373939433846353044354645383533364334333932464443384537423342313830373638222C22626C6F636B5F70617274735F686561646572223A506172745365747B543A31204236323237323535464632307D2C22686569676874223A312C22726F756E64223A302C2274797065223A327D7D")
-	if err != nil {
-		panic(err)
-	}
-	privBytes, err := hex.DecodeString("27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8")
-	if err != nil {
-		panic(err)
-	}
-	privKey := crypto.PrivKeyEd25519{}
-	copy(privKey[:], privBytes)
-	signature := privKey.Sign(signBytes)
-	fmt.Printf("Signature Bytes: %X\n", signature.Bytes())
-}
-```
-

consensus/test_data/build.sh

@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-
-# XXX: removes tendermint dir
-
-cd "$GOPATH/src/github.com/tendermint/tendermint" || exit 1
-
-# Make sure we have a tendermint command.
-if ! hash tendermint 2>/dev/null; then
-	make install
-fi
-
-# specify a dir to copy
-# TODO: eventually we should replace with `tendermint init --test`
-DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test
-
-TMHOME="$HOME/.tendermint"
-rm -rf "$TMHOME"
-cp -r "$DIR_TO_COPY" "$TMHOME"
-cp $TMHOME/config.toml $TMHOME/config.toml.bak
-
-function reset(){
-	tendermint unsafe_reset_all
-	cp $TMHOME/config.toml.bak $TMHOME/config.toml
-}
-
-reset
-
-# empty block
-function empty_block(){
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 5
-killall tendermint
-
-# /q would print up to and including the match, then quit.
-# /Q doesn't include the match.
-# http://unix.stackexchange.com/questions/11305/grep-show-all-the-file-up-to-the-match
-sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/empty_block.cswal
-
-reset
-}
-
-# many blocks
-function many_blocks(){
-bash scripts/txs/random.sh 1000 36657 &> /dev/null &
-PID=$!
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 7
-killall tendermint
-kill -9 $PID
-
-sed '/ENDHEIGHT: 6/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/many_blocks.cswal
-
-reset
-}
-
-
-# small block 1
-function small_block1(){
-bash scripts/txs/random.sh 1000 36657 &> /dev/null &
-PID=$!
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 10
-killall tendermint
-kill -9 $PID
-
-sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block1.cswal
-
-reset
-}
-
-
-# small block 2 (part size = 512)
-function small_block2(){
-echo "" >> ~/.tendermint/config.toml
-echo "block_part_size = 512" >> ~/.tendermint/config.toml
-bash scripts/txs/random.sh 1000 36657 &> /dev/null &
-PID=$!
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 5
-killall tendermint
-kill -9 $PID
-
-sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block2.cswal
-
-reset
-}
-
-
-
-case "$1" in
-	"small_block1")
-		small_block1
-		;;
-	"small_block2")
-		small_block2
-		;;
-	"empty_block")
-		empty_block
-		;;
-	"many_blocks")
-		many_blocks
-		;;
-	*)
-		small_block1
-		small_block2
-		empty_block
-		many_blocks
-esac
-
-

consensus/test_data/empty_block.cswal

@@ -1,10 +0,0 @@
-#ENDHEIGHT: 0
-{"time":"2017-04-27T22:24:01.346Z","msg":[3,{"duration":972946821,"height":1,"round":0,"step":1}]}
-{"time":"2017-04-27T22:24:01.349Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
-{"time":"2017-04-27T22:24:01.349Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"E785764AED6D92D7CC65C0A3A4ED9C8465198A05142C3E6C7F3EF601FDCD3A604900B77B7B87C046221EF99FD038A960398385BD5BBAA50EE4F86DE757B8F704"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.350Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114B96165CF4496C00000000000000114354594CBFC1A7BCA1AD0050ED6AA010023EADA390001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.351Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
-{"time":"2017-04-27T22:24:01.351Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"F3BBFBE7E4A5D619E2C498C3D1B912883786DD71","parts":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"}},"signature":[1,"35C937C78D061ECDC3770982A1330C9AA7F6FEF00835C43DEB50B8FCF69A3EEF221E675EE5E469114F64E4FBBABA414EB9170E1025FC47D3F0EADE46767D2E00"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.352Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
-{"time":"2017-04-27T22:24:01.352Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"F3BBFBE7E4A5D619E2C498C3D1B912883786DD71","parts":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"}},"signature":[1,"D1A7D27FCD5D352F3A3EDA8DE368520BC5B796662E32BCD8D91CDB8209A88DAF37CB7C4C93143D3C12B37C1435229268098CFFD0AD1400D88DA7606454692301"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.352Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}

consensus/test_data/small_block2.cswal

@@ -1,15 +0,0 @@
-#ENDHEIGHT: 0
-{"time":"2017-04-27T22:23:56.310Z","msg":[3,{"duration":969732098,"height":1,"round":0,"step":1}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"7624F6E943B7A207E16D1FA87EA099BD924E930F98E7DECBC01DB37735C619409588A67C2EABA9845FD6B80FDB65ECFCDA5F0DEFCEF74B8C34DB8E0540480203"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114B96164A30A118001620000000001141F6753D22BACA2180B1EADD722434EB28444D91D0114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010162011A3631363236333634333133303344363436333632363133313330011A3631363236333634333133313344363436333632363133313331011A3631363236333634333133323344363436333632363133313332011A3631363236333634333133333344363436333632363133313333011A3631363236333634333133343344363436333632363133313334011A3631363236333634333133353344363436333632363133313335011A3631363236333634333133363344363436333632363133313336011A3631363236333634333133373344363436333632363133313337011A3631363236333634333133383344363436333632363133313338011A3631363236333634333133393344363436333632363133313339011A3631363236333634333233303344363436333632363133323330011A3631363236333634333233313344363436333632363133323331011A3631363236333634333233323344363436333632363133323332011A3631363236333634333233333344363436333632363133323333011A3631363236333634333233343344363436333632363133323334011A36313632363336","proof":{"aunts":["49F4B71E3D7C457415069E2EA916DB12F67AA8D0","D35A72BEDAAAAC17045D7BFAAFA94C2EC0B0A4C2","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"34333233353344363436333632363133323335011A3631363236333634333233363344363436333632363133323336011A3631363236333634333233373344363436333632363133323337011A3631363236333634333233383344363436333632363133323338011A3631363236333634333233393344363436333632363133323339011A3631363236333634333333303344363436333632363133333330011A3631363236333634333333313344363436333632363133333331011A3631363236333634333333323344363436333632363133333332011A3631363236333634333333333344363436333632363133333333011A3631363236333634333333343344363436333632363133333334011A3631363236333634333333353344363436333632363133333335011A3631363236333634333333363344363436333632363133333336011A3631363236333634333333373344363436333632363133333337011A3631363236333634333333383344363436333632363133333338011A3631363236333634333333393344363436333632363133333339011A3631363236333634333433303344363436333632363133343330011A3631363236333634333433313344363436333632363133343331011A3631363236333634333433323344363436333632363133343332011A363136323633363433343333334436","proof":{"aunts":["5AD2A9A1A49A1FD6EF83F05FA4588F800B29DEF1","D35A72BEDAAAAC17045D7BFAAFA94C2EC0B0A4C2","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"3436333632363133343333011A3631363236333634333433343344363436333632363133343334011A3631363236333634333433353344363436333632363133343335011A3631363236333634333433363344363436333632363133343336011A3631363236333634333433373344363436333632363133343337011A3631363236333634333433383344363436333632363133343338011A3631363236333634333433393344363436333632363133343339011A3631363236333634333533303344363436333632363133353330011A3631363236333634333533313344363436333632363133353331011A3631363236333634333533323344363436333632363133353332011A3631363236333634333533333344363436333632363133353333011A3631363236333634333533343344363436333632363133353334011A3631363236333634333533353344363436333632363133353335011A3631363236333634333533363344363436333632363133353336011A3631363236333634333533373344363436333632363133353337011A3631363236333634333533383344363436333632363133353338011A3631363236333634333533393344363436333632363133353339011A3631363236333634333633303344363436333632363133363330011A3631363236333634333633313344363436333632363133","proof":{"aunts":["8B5786C3D871EE37B0F4B2DECAC39E157340DFBE","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"363331011A3631363236333634333633323344363436333632363133363332011A3631363236333634333633333344363436333632363133363333011A3631363236333634333633343344363436333632363133363334011A3631363236333634333633353344363436333632363133363335011A3631363236333634333633363344363436333632363133363336011A3631363236333634333633373344363436333632363133363337011A3631363236333634333633383344363436333632363133363338011A3631363236333634333633393344363436333632363133363339011A3631363236333634333733303344363436333632363133373330011A3631363236333634333733313344363436333632363133373331011A3631363236333634333733323344363436333632363133373332011A3631363236333634333733333344363436333632363133373333011A3631363236333634333733343344363436333632363133373334011A3631363236333634333733353344363436333632363133373335011A3631363236333634333733363344363436333632363133373336011A3631363236333634333733373344363436333632363133373337011A3631363236333634333733383344363436333632363133373338011A3631363236333634333733393344363436333632363133373339011A363136","proof":{"aunts":["56097661A1B2707588100586B3B1C2C8A51057D1","6DE889147DF528EEB5F7422E95DC45900CAFB619","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"3236333634333833303344363436333632363133383330011A3631363236333634333833313344363436333632363133383331011A3631363236333634333833323344363436333632363133383332011A3631363236333634333833333344363436333632363133383333011A3631363236333634333833343344363436333632363133383334011A3631363236333634333833353344363436333632363133383335011A3631363236333634333833363344363436333632363133383336011A3631363236333634333833373344363436333632363133383337011A3631363236333634333833383344363436333632363133383338011A3631363236333634333833393344363436333632363133383339011A3631363236333634333933303344363436333632363133393330011A3631363236333634333933313344363436333632363133393331011A3631363236333634333933323344363436333632363133393332011A3631363236333634333933333344363436333632363133393333011A3631363236333634333933343344363436333632363133393334011A3631363236333634333933353344363436333632363133393335011A3631363236333634333933363344363436333632363133393336011A3631363236333634333933373344363436333632363133393337011A3631363236333634333933","proof":{"aunts":["081D3DC5F11850851D5F0D760B98EE87BFA6B8B0","6DE889147DF528EEB5F7422E95DC45900CAFB619","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.313Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":5,"bytes":"383344363436333632363133393338011A3631363236333634333933393344363436333632363133393339011E363136323633363433313330333033443634363336323631333133303330011E363136323633363433313330333133443634363336323631333133303331011E363136323633363433313330333233443634363336323631333133303332011E363136323633363433313330333333443634363336323631333133303333011E363136323633363433313330333433443634363336323631333133303334011E363136323633363433313330333533443634363336323631333133303335011E363136323633363433313330333633443634363336323631333133303336011E3631363236333634333133303337334436343633363236313331333033370100000000","proof":{"aunts":["6AA912328C2B52EFA0ECE71F523E137E400EC484","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.314Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
-{"time":"2017-04-27T22:23:56.314Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"62371CF72F8662378691706DB256C833CF1AF81B","parts":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"}},"signature":[1,"255906FAAA50C84E85DABF7DE73468E4F95DB4E46F598848145926E2FAD77CA682BF07E09E2F3EC81FFBD9A036B67914A3C02F819B69248D777AEBA792725907"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.315Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
-{"time":"2017-04-27T22:23:56.315Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"62371CF72F8662378691706DB256C833CF1AF81B","parts":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"}},"signature":[1,"056CC15C748434D0A59B64B45CB56EDC1A437A426E68FA63DC7D61A7C17B0F768F207D81340D129A57C5A64195F8AFDD03B6BF28D7B2286290D61BCE88FCA304"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.316Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}

consensus/ticker.go

@@ -15,8 +15,8 @@ var (
 // conditional on the height/round/step in the timeoutInfo.
 // The timeoutInfo.Duration may be non-positive.
 type TimeoutTicker interface {
-	Start() (bool, error)
-	Stop() bool
+	Start() error
+	Stop() error
 	Chan() <-chan timeoutInfo       // on which to receive a timeout
 	ScheduleTimeout(ti timeoutInfo) // reset the timer
 
@@ -68,7 +68,7 @@ func (t *timeoutTicker) Chan() <-chan timeoutInfo {
 }
 
 // ScheduleTimeout schedules a new timeout by sending on the internal tickChan.
-// The timeoutRoutine is alwaya available to read from tickChan, so this won't block.
+// The timeoutRoutine is always available to read from tickChan, so this won't block.
 // The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
 func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) {
 	t.tickChan <- ti
@@ -127,7 +127,7 @@ func (t *timeoutTicker) timeoutRoutine() {
 			// We can eliminate it by merging the timeoutRoutine into receiveRoutine
 			//  and managing the timeouts ourselves with a millisecond ticker
 			go func(toi timeoutInfo) { t.tockChan <- toi }(ti)
-		case <-t.Quit:
+		case <-t.Quit():
 			return
 		}
 	}

consensus/types/height_vote_set.go

@@ -1,9 +1,12 @@
 package types
 
 import (
+	"errors"
+	"fmt"
 	"strings"
 	"sync"
 
+	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
 )
@@ -13,6 +16,10 @@ type RoundVoteSet struct {
 	Precommits *types.VoteSet
 }
 
+var (
+	GotVoteFromUnwantedRoundError = errors.New("Peer has sent a vote that does not match our round for more than one round")
+)
+
 /*
 Keeps track of all VoteSets from round 0 to round 'round'.
 
@@ -29,16 +36,16 @@ One for their LastCommit round, and another for the official commit round.
 */
 type HeightVoteSet struct {
 	chainID string
-	height  int
+	height  int64
 	valSet  *types.ValidatorSet
 
 	mtx               sync.Mutex
 	round             int                  // max tracked round
 	roundVoteSets     map[int]RoundVoteSet // keys: [0...round]
-	peerCatchupRounds map[string][]int     // keys: peer.Key; values: at most 2 rounds
+	peerCatchupRounds map[p2p.ID][]int     // keys: peer.ID; values: at most 2 rounds
 }
 
-func NewHeightVoteSet(chainID string, height int, valSet *types.ValidatorSet) *HeightVoteSet {
+func NewHeightVoteSet(chainID string, height int64, valSet *types.ValidatorSet) *HeightVoteSet {
 	hvs := &HeightVoteSet{
 		chainID: chainID,
 	}
@@ -46,20 +53,20 @@ func NewHeightVoteSet(chainID string, height int, valSet *types.ValidatorSet) *H
 	return hvs
 }
 
-func (hvs *HeightVoteSet) Reset(height int, valSet *types.ValidatorSet) {
+func (hvs *HeightVoteSet) Reset(height int64, valSet *types.ValidatorSet) {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 
 	hvs.height = height
 	hvs.valSet = valSet
 	hvs.roundVoteSets = make(map[int]RoundVoteSet)
-	hvs.peerCatchupRounds = make(map[string][]int)
+	hvs.peerCatchupRounds = make(map[p2p.ID][]int)
 
 	hvs.addRound(0)
 	hvs.round = 0
 }
 
-func (hvs *HeightVoteSet) Height() int {
+func (hvs *HeightVoteSet) Height() int64 {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 	return hvs.height
@@ -101,8 +108,8 @@ func (hvs *HeightVoteSet) addRound(round int) {
 }
 
 // Duplicate votes return added=false, err=nil.
-// By convention, peerKey is "" if origin is self.
-func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool, err error) {
+// By convention, peerID is "" if origin is self.
+func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerID p2p.ID) (added bool, err error) {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 	if !types.IsVoteTypeValid(vote.Type) {
@@ -110,15 +117,13 @@ func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool,
 	}
 	voteSet := hvs.getVoteSet(vote.Round, vote.Type)
 	if voteSet == nil {
-		if rndz := hvs.peerCatchupRounds[peerKey]; len(rndz) < 2 {
+		if rndz := hvs.peerCatchupRounds[peerID]; len(rndz) < 2 {
 			hvs.addRound(vote.Round)
 			voteSet = hvs.getVoteSet(vote.Round, vote.Type)
-			hvs.peerCatchupRounds[peerKey] = append(rndz, vote.Round)
+			hvs.peerCatchupRounds[peerID] = append(rndz, vote.Round)
 		} else {
-			// Peer has sent a vote that does not match our round,
-			// for more than one round.  Bad peer!
-			// TODO punish peer.
-			// log.Warn("Deal with peer giving votes from unwanted rounds")
+			// punish peer
+			err = GotVoteFromUnwantedRoundError
 			return
 		}
 	}
@@ -169,6 +174,26 @@ func (hvs *HeightVoteSet) getVoteSet(round int, type_ byte) *types.VoteSet {
 	}
 }
 
+// If a peer claims that it has 2/3 majority for given blockKey, call this.
+// NOTE: if there are too many peers, or too much peer churn,
+// this can cause memory issues.
+// TODO: implement ability to remove peers too
+func (hvs *HeightVoteSet) SetPeerMaj23(round int, type_ byte, peerID p2p.ID, blockID types.BlockID) error {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	if !types.IsVoteTypeValid(type_) {
+		return fmt.Errorf("SetPeerMaj23: Invalid vote type %v", type_)
+	}
+	voteSet := hvs.getVoteSet(round, type_)
+	if voteSet == nil {
+		return nil // something we don't know about yet
+	}
+	return voteSet.SetPeerMaj23(types.P2PID(peerID), blockID)
+}
+
+//---------------------------------------------------------
+// string and json
+
 func (hvs *HeightVoteSet) String() string {
 	return hvs.StringIndented("")
 }
@@ -202,19 +227,35 @@ func (hvs *HeightVoteSet) StringIndented(indent string) string {
 		indent)
 }
 
-// If a peer claims that it has 2/3 majority for given blockKey, call this.
-// NOTE: if there are too many peers, or too much peer churn,
-// this can cause memory issues.
-// TODO: implement ability to remove peers too
-func (hvs *HeightVoteSet) SetPeerMaj23(round int, type_ byte, peerID string, blockID types.BlockID) {
+func (hvs *HeightVoteSet) MarshalJSON() ([]byte, error) {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
-	if !types.IsVoteTypeValid(type_) {
-		return
-	}
-	voteSet := hvs.getVoteSet(round, type_)
-	if voteSet == nil {
-		return
-	}
-	voteSet.SetPeerMaj23(peerID, blockID)
+
+	allVotes := hvs.toAllRoundVotes()
+	return cdc.MarshalJSON(allVotes)
+}
+
+func (hvs *HeightVoteSet) toAllRoundVotes() []roundVotes {
+	totalRounds := hvs.round + 1
+	allVotes := make([]roundVotes, totalRounds)
+	// rounds 0 ~ hvs.round inclusive
+	for round := 0; round < totalRounds; round++ {
+		allVotes[round] = roundVotes{
+			Round:              round,
+			Prevotes:           hvs.roundVoteSets[round].Prevotes.VoteStrings(),
+			PrevotesBitArray:   hvs.roundVoteSets[round].Prevotes.BitArrayString(),
+			Precommits:         hvs.roundVoteSets[round].Precommits.VoteStrings(),
+			PrecommitsBitArray: hvs.roundVoteSets[round].Precommits.BitArrayString(),
+		}
+	}
+	// TODO: all other peer catchup rounds
+	return allVotes
+}
+
+type roundVotes struct {
+	Round              int      `json:"round"`
+	Prevotes           []string `json:"prevotes"`
+	PrevotesBitArray   string   `json:"prevotes_bit_array"`
+	Precommits         []string `json:"precommits"`
+	PrecommitsBitArray string   `json:"precommits_bit_array"`
 }

consensus/types/height_vote_set_test.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"testing"
+	"time"
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/types"
@@ -17,7 +18,7 @@ func init() {
 func TestPeerCatchupRounds(t *testing.T) {
 	valSet, privVals := types.RandValidatorSet(10, 1)
 
-	hvs := NewHeightVoteSet(config.ChainID, 1, valSet)
+	hvs := NewHeightVoteSet(config.ChainID(), 1, valSet)
 
 	vote999_0 := makeVoteHR(t, 1, 999, privVals, 0)
 	added, err := hvs.AddVote(vote999_0, "peer1")
@@ -33,8 +34,8 @@ func TestPeerCatchupRounds(t *testing.T) {
 
 	vote1001_0 := makeVoteHR(t, 1, 1001, privVals, 0)
 	added, err = hvs.AddVote(vote1001_0, "peer1")
-	if err != nil {
-		t.Error("AddVote error", err)
+	if err != GotVoteFromUnwantedRoundError {
+		t.Errorf("Expected GotVoteFromUnwantedRoundError, but got %v", err)
 	}
 	if added {
 		t.Error("Expected to *not* add vote from peer, too many catchup rounds.")
@@ -47,17 +48,18 @@ func TestPeerCatchupRounds(t *testing.T) {
 
 }
 
-func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidatorFS, valIndex int) *types.Vote {
+func makeVoteHR(t *testing.T, height int64, round int, privVals []types.PrivValidator, valIndex int) *types.Vote {
 	privVal := privVals[valIndex]
 	vote := &types.Vote{
 		ValidatorAddress: privVal.GetAddress(),
 		ValidatorIndex:   valIndex,
 		Height:           height,
 		Round:            round,
+		Timestamp:        time.Now().UTC(),
 		Type:             types.VoteTypePrecommit,
 		BlockID:          types.BlockID{[]byte("fakehash"), types.PartSetHeader{}},
 	}
-	chainID := config.ChainID
+	chainID := config.ChainID()
 	err := privVal.SignVote(chainID, vote)
 	if err != nil {
 		panic(cmn.Fmt("Error signing vote: %v", err))

consensus/types/peer_round_state.go

@@ -0,0 +1,57 @@
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+//-----------------------------------------------------------------------------
+
+// PeerRoundState contains the known state of a peer.
+// NOTE: Read-only when returned by PeerState.GetRoundState().
+type PeerRoundState struct {
+	Height                   int64               `json:"height"`                      // Height peer is at
+	Round                    int                 `json:"round"`                       // Round peer is at, -1 if unknown.
+	Step                     RoundStepType       `json:"step"`                        // Step peer is at
+	StartTime                time.Time           `json:"start_time"`                  // Estimated start of round 0 at this height
+	Proposal                 bool                `json:"proposal"`                    // True if peer has proposal for this round
+	ProposalBlockPartsHeader types.PartSetHeader `json:"proposal_block_parts_header"` //
+	ProposalBlockParts       *cmn.BitArray       `json:"proposal_block_parts"`        //
+	ProposalPOLRound         int                 `json:"proposal_pol_round"`          // Proposal's POL round. -1 if none.
+	ProposalPOL              *cmn.BitArray       `json:"proposal_pol"`                // nil until ProposalPOLMessage received.
+	Prevotes                 *cmn.BitArray       `json:"prevotes"`                    // All votes peer has for this round
+	Precommits               *cmn.BitArray       `json:"precommits"`                  // All precommits peer has for this round
+	LastCommitRound          int                 `json:"last_commit_round"`           // Round of commit for last height. -1 if none.
+	LastCommit               *cmn.BitArray       `json:"last_commit"`                 // All commit precommits of commit for last height.
+	CatchupCommitRound       int                 `json:"catchup_commit_round"`        // Round that we have commit for. Not necessarily unique. -1 if none.
+	CatchupCommit            *cmn.BitArray       `json:"catchup_commit"`              // All commit precommits peer has for this height & CatchupCommitRound
+}
+
+// String returns a string representation of the PeerRoundState
+func (prs PeerRoundState) String() string {
+	return prs.StringIndented("")
+}
+
+// StringIndented returns a string representation of the PeerRoundState
+func (prs PeerRoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`PeerRoundState{
+%s  %v/%v/%v @%v
+%s  Proposal %v -> %v
+%s  POL      %v (round %v)
+%s  Prevotes   %v
+%s  Precommits %v
+%s  LastCommit %v (round %v)
+%s  Catchup    %v (round %v)
+%s}`,
+		indent, prs.Height, prs.Round, prs.Step, prs.StartTime,
+		indent, prs.ProposalBlockPartsHeader, prs.ProposalBlockParts,
+		indent, prs.ProposalPOL, prs.ProposalPOLRound,
+		indent, prs.Prevotes,
+		indent, prs.Precommits,
+		indent, prs.LastCommit, prs.LastCommitRound,
+		indent, prs.CatchupCommit, prs.CatchupCommitRound,
+		indent)
+}

consensus/types/reactor.go

@@ -1,57 +0,0 @@
-package types
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/tendermint/tendermint/types"
-	cmn "github.com/tendermint/tmlibs/common"
-)
-
-//-----------------------------------------------------------------------------
-
-// PeerRoundState contains the known state of a peer.
-// NOTE: Read-only when returned by PeerState.GetRoundState().
-type PeerRoundState struct {
-	Height                   int                 // Height peer is at
-	Round                    int                 // Round peer is at, -1 if unknown.
-	Step                     RoundStepType       // Step peer is at
-	StartTime                time.Time           // Estimated start of round 0 at this height
-	Proposal                 bool                // True if peer has proposal for this round
-	ProposalBlockPartsHeader types.PartSetHeader //
-	ProposalBlockParts       *cmn.BitArray       //
-	ProposalPOLRound         int                 // Proposal's POL round. -1 if none.
-	ProposalPOL              *cmn.BitArray       // nil until ProposalPOLMessage received.
-	Prevotes                 *cmn.BitArray       // All votes peer has for this round
-	Precommits               *cmn.BitArray       // All precommits peer has for this round
-	LastCommitRound          int                 // Round of commit for last height. -1 if none.
-	LastCommit               *cmn.BitArray       // All commit precommits of commit for last height.
-	CatchupCommitRound       int                 // Round that we have commit for. Not necessarily unique. -1 if none.
-	CatchupCommit            *cmn.BitArray       // All commit precommits peer has for this height & CatchupCommitRound
-}
-
-// String returns a string representation of the PeerRoundState
-func (prs PeerRoundState) String() string {
-	return prs.StringIndented("")
-}
-
-// StringIndented returns a string representation of the PeerRoundState
-func (prs PeerRoundState) StringIndented(indent string) string {
-	return fmt.Sprintf(`PeerRoundState{
-%s  %v/%v/%v @%v
-%s  Proposal %v -> %v
-%s  POL      %v (round %v)
-%s  Prevotes   %v
-%s  Precommits %v
-%s  LastCommit %v (round %v)
-%s  Catchup    %v (round %v)
-%s}`,
-		indent, prs.Height, prs.Round, prs.Step, prs.StartTime,
-		indent, prs.ProposalBlockPartsHeader, prs.ProposalBlockParts,
-		indent, prs.ProposalPOL, prs.ProposalPOLRound,
-		indent, prs.Prevotes,
-		indent, prs.Precommits,
-		indent, prs.LastCommit, prs.LastCommitRound,
-		indent, prs.CatchupCommit, prs.CatchupCommitRound,
-		indent)
-}

consensus/types/round_state.go

@@ -0,0 +1,164 @@
+package types
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+//-----------------------------------------------------------------------------
+// RoundStepType enum type
+
+// RoundStepType enumerates the state of the consensus state machine
+type RoundStepType uint8 // These must be numeric, ordered.
+
+// RoundStepType
+const (
+	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
+	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
+	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
+	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
+	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
+	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
+	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
+	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
+	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
+)
+
+// String returns a string
+func (rs RoundStepType) String() string {
+	switch rs {
+	case RoundStepNewHeight:
+		return "RoundStepNewHeight"
+	case RoundStepNewRound:
+		return "RoundStepNewRound"
+	case RoundStepPropose:
+		return "RoundStepPropose"
+	case RoundStepPrevote:
+		return "RoundStepPrevote"
+	case RoundStepPrevoteWait:
+		return "RoundStepPrevoteWait"
+	case RoundStepPrecommit:
+		return "RoundStepPrecommit"
+	case RoundStepPrecommitWait:
+		return "RoundStepPrecommitWait"
+	case RoundStepCommit:
+		return "RoundStepCommit"
+	default:
+		return "RoundStepUnknown" // Cannot panic.
+	}
+}
+
+//-----------------------------------------------------------------------------
+
+// RoundState defines the internal consensus state.
+// NOTE: Not thread safe. Should only be manipulated by functions downstream
+// of the cs.receiveRoutine
+type RoundState struct {
+	Height             int64               `json:"height"` // Height we are working on
+	Round              int                 `json:"round"`
+	Step               RoundStepType       `json:"step"`
+	StartTime          time.Time           `json:"start_time"`
+	CommitTime         time.Time           `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found
+	Validators         *types.ValidatorSet `json:"validators"`
+	Proposal           *types.Proposal     `json:"proposal"`
+	ProposalBlock      *types.Block        `json:"proposal_block"`
+	ProposalBlockParts *types.PartSet      `json:"proposal_block_parts"`
+	LockedRound        int                 `json:"locked_round"`
+	LockedBlock        *types.Block        `json:"locked_block"`
+	LockedBlockParts   *types.PartSet      `json:"locked_block_parts"`
+	ValidRound         int                 `json:"valid_round"`       // Last known round with POL for non-nil valid block.
+	ValidBlock         *types.Block        `json:"valid_block"`       // Last known block of POL mentioned above.
+	ValidBlockParts    *types.PartSet      `json:"valid_block_parts"` // Last known block parts of POL metnioned above.
+	Votes              *HeightVoteSet      `json:"votes"`
+	CommitRound        int                 `json:"commit_round"` //
+	LastCommit         *types.VoteSet      `json:"last_commit"`  // Last precommits at Height-1
+	LastValidators     *types.ValidatorSet `json:"last_validators"`
+}
+
+// Compressed version of the RoundState for use in RPC
+type RoundStateSimple struct {
+	HeightRoundStep   string          `json:"height/round/step"`
+	StartTime         time.Time       `json:"start_time"`
+	ProposalBlockHash cmn.HexBytes    `json:"proposal_block_hash"`
+	LockedBlockHash   cmn.HexBytes    `json:"locked_block_hash"`
+	ValidBlockHash    cmn.HexBytes    `json:"valid_block_hash"`
+	Votes             json.RawMessage `json:"height_vote_set"`
+}
+
+// Compress the RoundState to RoundStateSimple
+func (rs *RoundState) RoundStateSimple() RoundStateSimple {
+	votesJSON, err := rs.Votes.MarshalJSON()
+	if err != nil {
+		panic(err)
+	}
+	return RoundStateSimple{
+		HeightRoundStep:   fmt.Sprintf("%d/%d/%d", rs.Height, rs.Round, rs.Step),
+		StartTime:         rs.StartTime,
+		ProposalBlockHash: rs.ProposalBlock.Hash(),
+		LockedBlockHash:   rs.LockedBlock.Hash(),
+		ValidBlockHash:    rs.ValidBlock.Hash(),
+		Votes:             votesJSON,
+	}
+}
+
+// RoundStateEvent returns the H/R/S of the RoundState as an event.
+func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
+	// XXX: copy the RoundState
+	// if we want to avoid this, we may need synchronous events after all
+	rsCopy := *rs
+	edrs := types.EventDataRoundState{
+		Height:     rs.Height,
+		Round:      rs.Round,
+		Step:       rs.Step.String(),
+		RoundState: &rsCopy,
+	}
+	return edrs
+}
+
+// String returns a string
+func (rs *RoundState) String() string {
+	return rs.StringIndented("")
+}
+
+// StringIndented returns a string
+func (rs *RoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`RoundState{
+%s  H:%v R:%v S:%v
+%s  StartTime:     %v
+%s  CommitTime:    %v
+%s  Validators:    %v
+%s  Proposal:      %v
+%s  ProposalBlock: %v %v
+%s  LockedRound:   %v
+%s  LockedBlock:   %v %v
+%s  ValidRound:   %v
+%s  ValidBlock:   %v %v
+%s  Votes:         %v
+%s  LastCommit:    %v
+%s  LastValidators:%v
+%s}`,
+		indent, rs.Height, rs.Round, rs.Step,
+		indent, rs.StartTime,
+		indent, rs.CommitTime,
+		indent, rs.Validators.StringIndented(indent+"  "),
+		indent, rs.Proposal,
+		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
+		indent, rs.LockedRound,
+		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
+		indent, rs.ValidRound,
+		indent, rs.ValidBlockParts.StringShort(), rs.ValidBlock.StringShort(),
+		indent, rs.Votes.StringIndented(indent+"  "),
+		indent, rs.LastCommit.StringShort(),
+		indent, rs.LastValidators.StringIndented(indent+"  "),
+		indent)
+}
+
+// StringShort returns a string
+func (rs *RoundState) StringShort() string {
+	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
+		rs.Height, rs.Round, rs.Step, rs.StartTime)
+}

consensus/types/round_state_test.go

@@ -0,0 +1,95 @@
+package types
+
+import (
+	"testing"
+	"time"
+
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/go-crypto"
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+func BenchmarkRoundStateDeepCopy(b *testing.B) {
+	b.StopTimer()
+
+	// Random validators
+	nval, ntxs := 100, 100
+	vset, _ := types.RandValidatorSet(nval, 1)
+	precommits := make([]*types.Vote, nval)
+	blockID := types.BlockID{
+		Hash: cmn.RandBytes(20),
+		PartsHeader: types.PartSetHeader{
+			Hash: cmn.RandBytes(20),
+		},
+	}
+	sig := crypto.SignatureEd25519{}
+	for i := 0; i < nval; i++ {
+		precommits[i] = &types.Vote{
+			ValidatorAddress: types.Address(cmn.RandBytes(20)),
+			Timestamp:        time.Now(),
+			BlockID:          blockID,
+			Signature:        sig,
+		}
+	}
+	txs := make([]types.Tx, ntxs)
+	for i := 0; i < ntxs; i++ {
+		txs[i] = cmn.RandBytes(100)
+	}
+	// Random block
+	block := &types.Block{
+		Header: &types.Header{
+			ChainID:         cmn.RandStr(12),
+			Time:            time.Now(),
+			LastBlockID:     blockID,
+			LastCommitHash:  cmn.RandBytes(20),
+			DataHash:        cmn.RandBytes(20),
+			ValidatorsHash:  cmn.RandBytes(20),
+			ConsensusHash:   cmn.RandBytes(20),
+			AppHash:         cmn.RandBytes(20),
+			LastResultsHash: cmn.RandBytes(20),
+			EvidenceHash:    cmn.RandBytes(20),
+		},
+		Data: &types.Data{
+			Txs: txs,
+		},
+		Evidence: types.EvidenceData{},
+		LastCommit: &types.Commit{
+			BlockID:    blockID,
+			Precommits: precommits,
+		},
+	}
+	parts := block.MakePartSet(4096)
+	// Random Proposal
+	proposal := &types.Proposal{
+		Timestamp: time.Now(),
+		BlockPartsHeader: types.PartSetHeader{
+			Hash: cmn.RandBytes(20),
+		},
+		POLBlockID: blockID,
+		Signature:  sig,
+	}
+	// Random HeightVoteSet
+	// TODO: hvs :=
+
+	rs := &RoundState{
+		StartTime:          time.Now(),
+		CommitTime:         time.Now(),
+		Validators:         vset,
+		Proposal:           proposal,
+		ProposalBlock:      block,
+		ProposalBlockParts: parts,
+		LockedBlock:        block,
+		LockedBlockParts:   parts,
+		ValidBlock:         block,
+		ValidBlockParts:    parts,
+		Votes:              nil, // TODO
+		LastCommit:         nil, // TODO
+		LastValidators:     vset,
+	}
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		amino.DeepCopy(rs)
+	}
+}

consensus/types/state.go

@@ -1,126 +0,0 @@
-package types
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/tendermint/tendermint/types"
-)
-
-//-----------------------------------------------------------------------------
-// RoundStepType enum type
-
-// RoundStepType enumerates the state of the consensus state machine
-type RoundStepType uint8 // These must be numeric, ordered.
-
-const (
-	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
-	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
-	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
-	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
-	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
-	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
-	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
-	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
-	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
-)
-
-// String returns a string
-func (rs RoundStepType) String() string {
-	switch rs {
-	case RoundStepNewHeight:
-		return "RoundStepNewHeight"
-	case RoundStepNewRound:
-		return "RoundStepNewRound"
-	case RoundStepPropose:
-		return "RoundStepPropose"
-	case RoundStepPrevote:
-		return "RoundStepPrevote"
-	case RoundStepPrevoteWait:
-		return "RoundStepPrevoteWait"
-	case RoundStepPrecommit:
-		return "RoundStepPrecommit"
-	case RoundStepPrecommitWait:
-		return "RoundStepPrecommitWait"
-	case RoundStepCommit:
-		return "RoundStepCommit"
-	default:
-		return "RoundStepUnknown" // Cannot panic.
-	}
-}
-
-//-----------------------------------------------------------------------------
-
-// RoundState defines the internal consensus state.
-// It is Immutable when returned from ConsensusState.GetRoundState()
-// TODO: Actually, only the top pointer is copied,
-// so access to field pointers is still racey
-type RoundState struct {
-	Height             int // Height we are working on
-	Round              int
-	Step               RoundStepType
-	StartTime          time.Time
-	CommitTime         time.Time // Subjective time when +2/3 precommits for Block at Round were found
-	Validators         *types.ValidatorSet
-	Proposal           *types.Proposal
-	ProposalBlock      *types.Block
-	ProposalBlockParts *types.PartSet
-	LockedRound        int
-	LockedBlock        *types.Block
-	LockedBlockParts   *types.PartSet
-	Votes              *HeightVoteSet
-	CommitRound        int            //
-	LastCommit         *types.VoteSet // Last precommits at Height-1
-	LastValidators     *types.ValidatorSet
-}
-
-// RoundStateEvent returns the H/R/S of the RoundState as an event.
-func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
-	edrs := types.EventDataRoundState{
-		Height:     rs.Height,
-		Round:      rs.Round,
-		Step:       rs.Step.String(),
-		RoundState: rs,
-	}
-	return edrs
-}
-
-// String returns a string
-func (rs *RoundState) String() string {
-	return rs.StringIndented("")
-}
-
-// StringIndented returns a string
-func (rs *RoundState) StringIndented(indent string) string {
-	return fmt.Sprintf(`RoundState{
-%s  H:%v R:%v S:%v
-%s  StartTime:     %v
-%s  CommitTime:    %v
-%s  Validators:    %v
-%s  Proposal:      %v
-%s  ProposalBlock: %v %v
-%s  LockedRound:   %v
-%s  LockedBlock:   %v %v
-%s  Votes:         %v
-%s  LastCommit: %v
-%s  LastValidators:    %v
-%s}`,
-		indent, rs.Height, rs.Round, rs.Step,
-		indent, rs.StartTime,
-		indent, rs.CommitTime,
-		indent, rs.Validators.StringIndented(indent+"    "),
-		indent, rs.Proposal,
-		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
-		indent, rs.LockedRound,
-		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
-		indent, rs.Votes.StringIndented(indent+"    "),
-		indent, rs.LastCommit.StringShort(),
-		indent, rs.LastValidators.StringIndented(indent+"    "),
-		indent)
-}
-
-// StringShort returns a string
-func (rs *RoundState) StringShort() string {
-	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
-		rs.Height, rs.Round, rs.Step, rs.StartTime)
-}

consensus/types/wire.go

@@ -0,0 +1,12 @@
+package types
+
+import (
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/go-crypto"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	crypto.RegisterAmino(cdc)
+}

consensus/wal.go

@@ -1,104 +1,323 @@
 package consensus
 
 import (
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"path/filepath"
 	"time"
 
-	wire "github.com/tendermint/go-wire"
+	"github.com/pkg/errors"
+
+	amino "github.com/tendermint/go-amino"
 	"github.com/tendermint/tendermint/types"
 	auto "github.com/tendermint/tmlibs/autofile"
 	cmn "github.com/tendermint/tmlibs/common"
 )
 
+const (
+	// must be greater than params.BlockGossip.BlockPartSizeBytes + a few bytes
+	maxMsgSizeBytes = 1024 * 1024 // 1MB
+)
+
 //--------------------------------------------------------
 // types and functions for savings consensus messages
 
 type TimedWALMessage struct {
-	Time time.Time  `json:"time"`
+	Time time.Time  `json:"time"` // for debugging purposes
 	Msg  WALMessage `json:"msg"`
 }
 
+// EndHeightMessage marks the end of the given height inside WAL.
+// @internal used by scripts/wal2json util.
+type EndHeightMessage struct {
+	Height int64 `json:"height"`
+}
+
 type WALMessage interface{}
 
-var _ = wire.RegisterInterface(
-	struct{ WALMessage }{},
-	wire.ConcreteType{types.EventDataRoundState{}, 0x01},
-	wire.ConcreteType{msgInfo{}, 0x02},
-	wire.ConcreteType{timeoutInfo{}, 0x03},
-)
+func RegisterWALMessages(cdc *amino.Codec) {
+	cdc.RegisterInterface((*WALMessage)(nil), nil)
+	cdc.RegisterConcrete(types.EventDataRoundState{}, "tendermint/wal/EventDataRoundState", nil)
+	cdc.RegisterConcrete(msgInfo{}, "tendermint/wal/MsgInfo", nil)
+	cdc.RegisterConcrete(timeoutInfo{}, "tendermint/wal/TimeoutInfo", nil)
+	cdc.RegisterConcrete(EndHeightMessage{}, "tendermint/wal/EndHeightMessage", nil)
+}
 
 //--------------------------------------------------------
 // Simple write-ahead logger
 
+// WAL is an interface for any write-ahead logger.
+type WAL interface {
+	Write(WALMessage)
+	WriteSync(WALMessage)
+	Group() *auto.Group
+	SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error)
+
+	Start() error
+	Stop() error
+	Wait()
+}
+
 // Write ahead logger writes msgs to disk before they are processed.
 // Can be used for crash-recovery and deterministic replay
 // TODO: currently the wal is overwritten during replay catchup
 //   give it a mode so it's either reading or appending - must read to end to start appending again
-type WAL struct {
+type baseWAL struct {
 	cmn.BaseService
 
 	group *auto.Group
-	light bool // ignore block parts
+
+	enc *WALEncoder
 }
 
-func NewWAL(walFile string, light bool) (*WAL, error) {
+func NewWAL(walFile string) (*baseWAL, error) {
+	err := cmn.EnsureDir(filepath.Dir(walFile), 0700)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to ensure WAL directory is in place")
+	}
+
 	group, err := auto.OpenGroup(walFile)
 	if err != nil {
 		return nil, err
 	}
-	wal := &WAL{
+	wal := &baseWAL{
 		group: group,
-		light: light,
+		enc:   NewWALEncoder(group),
 	}
-	wal.BaseService = *cmn.NewBaseService(nil, "WAL", wal)
+	wal.BaseService = *cmn.NewBaseService(nil, "baseWAL", wal)
 	return wal, nil
 }
 
-func (wal *WAL) OnStart() error {
+func (wal *baseWAL) Group() *auto.Group {
+	return wal.group
+}
+
+func (wal *baseWAL) OnStart() error {
 	size, err := wal.group.Head.Size()
 	if err != nil {
 		return err
 	} else if size == 0 {
-		wal.writeEndHeight(0)
+		wal.WriteSync(EndHeightMessage{0})
 	}
-	_, err = wal.group.Start()
+	err = wal.group.Start()
 	return err
 }
 
-func (wal *WAL) OnStop() {
-	wal.BaseService.OnStop()
+func (wal *baseWAL) OnStop() {
 	wal.group.Stop()
+	wal.group.Close()
 }
 
-// called in newStep and for each pass in receiveRoutine
-func (wal *WAL) Save(wmsg WALMessage) {
+// Write is called in newStep and for each receive on the
+// peerMsgQueue and the timeoutTicker.
+// NOTE: does not call fsync()
+func (wal *baseWAL) Write(msg WALMessage) {
 	if wal == nil {
 		return
 	}
-	if wal.light {
-		// in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts)
-		if mi, ok := wmsg.(msgInfo); ok {
-			if mi.PeerKey != "" {
-				return
+
+	// Write the wal message
+	if err := wal.enc.Encode(&TimedWALMessage{time.Now(), msg}); err != nil {
+		panic(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg))
+	}
+}
+
+// WriteSync is called when we receive a msg from ourselves
+// so that we write to disk before sending signed messages.
+// NOTE: calls fsync()
+func (wal *baseWAL) WriteSync(msg WALMessage) {
+	if wal == nil {
+		return
+	}
+
+	wal.Write(msg)
+	if err := wal.group.Flush(); err != nil {
+		panic(cmn.Fmt("Error flushing consensus wal buf to file. Error: %v \n", err))
+	}
+}
+
+// WALSearchOptions are optional arguments to SearchForEndHeight.
+type WALSearchOptions struct {
+	// IgnoreDataCorruptionErrors set to true will result in skipping data corruption errors.
+	IgnoreDataCorruptionErrors bool
+}
+
+// SearchForEndHeight searches for the EndHeightMessage with the given height
+// and returns an auto.GroupReader, whenever it was found or not and an error.
+// Group reader will be nil if found equals false.
+//
+// CONTRACT: caller must close group reader.
+func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+	var msg *TimedWALMessage
+	lastHeightFound := int64(-1)
+
+	// NOTE: starting from the last file in the group because we're usually
+	// searching for the last height. See replay.go
+	min, max := wal.group.MinIndex(), wal.group.MaxIndex()
+	wal.Logger.Debug("Searching for height", "height", height, "min", min, "max", max)
+	for index := max; index >= min; index-- {
+		gr, err = wal.group.NewReader(index)
+		if err != nil {
+			return nil, false, err
+		}
+
+		dec := NewWALDecoder(gr)
+		for {
+			msg, err = dec.Decode()
+			if err == io.EOF {
+				// OPTIMISATION: no need to look for height in older files if we've seen h < height
+				if lastHeightFound > 0 && lastHeightFound < height {
+					gr.Close()
+					return nil, false, nil
+				}
+				// check next file
+				break
+			}
+			if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) {
+				wal.Logger.Debug("Corrupted entry. Skipping...", "err", err)
+				// do nothing
+				continue
+			} else if err != nil {
+				gr.Close()
+				return nil, false, err
+			}
+
+			if m, ok := msg.Msg.(EndHeightMessage); ok {
+				lastHeightFound = m.Height
+				if m.Height == height { // found
+					wal.Logger.Debug("Found", "height", height, "index", index)
+					return gr, true, nil
+				}
 			}
 		}
+		gr.Close()
+	}
+
+	return nil, false, nil
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+// A WALEncoder writes custom-encoded WAL messages to an output stream.
+//
+// Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value (go-amino encoded)
+type WALEncoder struct {
+	wr io.Writer
+}
+
+// NewWALEncoder returns a new encoder that writes to wr.
+func NewWALEncoder(wr io.Writer) *WALEncoder {
+	return &WALEncoder{wr}
+}
+
+// Encode writes the custom encoding of v to the stream.
+func (enc *WALEncoder) Encode(v *TimedWALMessage) error {
+	data := cdc.MustMarshalBinaryBare(v)
+
+	crc := crc32.Checksum(data, crc32c)
+	length := uint32(len(data))
+	totalLength := 8 + int(length)
+
+	msg := make([]byte, totalLength)
+	binary.BigEndian.PutUint32(msg[0:4], crc)
+	binary.BigEndian.PutUint32(msg[4:8], length)
+	copy(msg[8:], data)
+
+	_, err := enc.wr.Write(msg)
+
+	return err
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+// IsDataCorruptionError returns true if data has been corrupted inside WAL.
+func IsDataCorruptionError(err error) bool {
+	_, ok := err.(DataCorruptionError)
+	return ok
+}
+
+// DataCorruptionError is an error that occures if data on disk was corrupted.
+type DataCorruptionError struct {
+	cause error
+}
+
+func (e DataCorruptionError) Error() string {
+	return fmt.Sprintf("DataCorruptionError[%v]", e.cause)
+}
+
+func (e DataCorruptionError) Cause() error {
+	return e.cause
+}
+
+// A WALDecoder reads and decodes custom-encoded WAL messages from an input
+// stream. See WALEncoder for the format used.
+//
+// It will also compare the checksums and make sure data size is equal to the
+// length from the header. If that is not the case, error will be returned.
+type WALDecoder struct {
+	rd io.Reader
+}
+
+// NewWALDecoder returns a new decoder that reads from rd.
+func NewWALDecoder(rd io.Reader) *WALDecoder {
+	return &WALDecoder{rd}
+}
+
+// Decode reads the next custom-encoded value from its reader and returns it.
+func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
+	b := make([]byte, 4)
+
+	_, err := dec.rd.Read(b)
+	if err == io.EOF {
+		return nil, err
 	}
-	// Write the wal message
-	var wmsgBytes = wire.JSONBytes(TimedWALMessage{time.Now(), wmsg})
-	err := wal.group.WriteLine(string(wmsgBytes))
 	if err != nil {
-		cmn.PanicQ(cmn.Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, wmsg))
+		return nil, fmt.Errorf("failed to read checksum: %v", err)
 	}
-	// TODO: only flush when necessary
-	if err := wal.group.Flush(); err != nil {
-		cmn.PanicQ(cmn.Fmt("Error flushing consensus wal buf to file. Error: %v \n", err))
+	crc := binary.BigEndian.Uint32(b)
+
+	b = make([]byte, 4)
+	_, err = dec.rd.Read(b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read length: %v", err)
 	}
+	length := binary.BigEndian.Uint32(b)
+
+	if length > maxMsgSizeBytes {
+		return nil, fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes)
+	}
+
+	data := make([]byte, length)
+	_, err = dec.rd.Read(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read data: %v", err)
+	}
+
+	// check checksum before decoding data
+	actualCRC := crc32.Checksum(data, crc32c)
+	if actualCRC != crc {
+		return nil, DataCorruptionError{fmt.Errorf("checksums do not match: (read: %v, actual: %v)", crc, actualCRC)}
+	}
+
+	var res = new(TimedWALMessage) // nolint: gosimple
+	err = cdc.UnmarshalBinaryBare(data, res)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to decode data: %v", err)}
+	}
+
+	return res, err
 }
 
-func (wal *WAL) writeEndHeight(height int) {
-	wal.group.WriteLine(cmn.Fmt("#ENDHEIGHT: %v", height))
+type nilWAL struct{}
 
-	// TODO: only flush when necessary
-	if err := wal.group.Flush(); err != nil {
-		cmn.PanicQ(cmn.Fmt("Error flushing consensus wal buf to file. Error: %v \n", err))
-	}
+func (nilWAL) Write(m WALMessage)     {}
+func (nilWAL) WriteSync(m WALMessage) {}
+func (nilWAL) Group() *auto.Group     { return nil }
+func (nilWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+	return nil, false, nil
 }
+func (nilWAL) Start() error { return nil }
+func (nilWAL) Stop() error  { return nil }
+func (nilWAL) Wait()        {}

consensus/wal_fuzz.go

@@ -0,0 +1,31 @@
+// +build gofuzz
+
+package consensus
+
+import (
+	"bytes"
+	"io"
+)
+
+func Fuzz(data []byte) int {
+	dec := NewWALDecoder(bytes.NewReader(data))
+	for {
+		msg, err := dec.Decode()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			if msg != nil {
+				panic("msg != nil on error")
+			}
+			return 0
+		}
+		var w bytes.Buffer
+		enc := NewWALEncoder(&w)
+		err = enc.Encode(msg)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return 1
+}

consensus/wal_generator.go

@@ -0,0 +1,205 @@
+package consensus
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/tendermint/abci/example/kvstore"
+	bc "github.com/tendermint/tendermint/blockchain"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/privval"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+	auto "github.com/tendermint/tmlibs/autofile"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+)
+
+// WALWithNBlocks generates a consensus WAL. It does this by spining up a
+// stripped down version of node (proxy app, event bus, consensus state) with a
+// persistent kvstore application and special consensus wal instance
+// (byteBufferWAL) and waits until numBlocks are created. Then it returns a WAL
+// content.
+func WALWithNBlocks(numBlocks int) (data []byte, err error) {
+	config := getConfig()
+
+	app := kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), "wal_generator"))
+
+	logger := log.TestingLogger().With("wal_generator", "wal_generator")
+	logger.Info("generating WAL (last height msg excluded)", "numBlocks", numBlocks)
+
+	/////////////////////////////////////////////////////////////////////////////
+	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
+	// NOTE: we can't import node package because of circular dependency
+	privValidatorFile := config.PrivValidatorFile()
+	privValidator := privval.LoadOrGenFilePV(privValidatorFile)
+	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read genesis file")
+	}
+	stateDB := db.NewMemDB()
+	blockStoreDB := db.NewMemDB()
+	state, err := sm.MakeGenesisState(genDoc)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make genesis state")
+	}
+	blockStore := bc.NewBlockStore(blockStoreDB)
+	handshaker := NewHandshaker(stateDB, state, blockStore, genDoc)
+	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), handshaker)
+	proxyApp.SetLogger(logger.With("module", "proxy"))
+	if err := proxyApp.Start(); err != nil {
+		return nil, errors.Wrap(err, "failed to start proxy app connections")
+	}
+	defer proxyApp.Stop()
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(logger.With("module", "events"))
+	if err := eventBus.Start(); err != nil {
+		return nil, errors.Wrap(err, "failed to start event bus")
+	}
+	defer eventBus.Stop()
+	mempool := sm.MockMempool{}
+	evpool := sm.MockEvidencePool{}
+	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
+	consensusState := NewConsensusState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
+	consensusState.SetLogger(logger)
+	consensusState.SetEventBus(eventBus)
+	if privValidator != nil {
+		consensusState.SetPrivValidator(privValidator)
+	}
+	// END OF COPY PASTE
+	/////////////////////////////////////////////////////////////////////////////
+
+	// set consensus wal to buffered WAL, which will write all incoming msgs to buffer
+	var b bytes.Buffer
+	wr := bufio.NewWriter(&b)
+	numBlocksWritten := make(chan struct{})
+	wal := newByteBufferWAL(logger, NewWALEncoder(wr), int64(numBlocks), numBlocksWritten)
+	// see wal.go#103
+	wal.Write(EndHeightMessage{0})
+	consensusState.wal = wal
+
+	if err := consensusState.Start(); err != nil {
+		return nil, errors.Wrap(err, "failed to start consensus state")
+	}
+	defer consensusState.Stop()
+
+	select {
+	case <-numBlocksWritten:
+		wr.Flush()
+		return b.Bytes(), nil
+	case <-time.After(1 * time.Minute):
+		wr.Flush()
+		return b.Bytes(), fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks)
+	}
+}
+
+// f**ing long, but unique for each test
+func makePathname() string {
+	// get path
+	p, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	// fmt.Println(p)
+	sep := string(filepath.Separator)
+	return strings.Replace(p, sep, "_", -1)
+}
+
+func randPort() int {
+	// returns between base and base + spread
+	base, spread := 20000, 20000
+	return base + cmn.RandIntn(spread)
+}
+
+func makeAddrs() (string, string, string) {
+	start := randPort()
+	return fmt.Sprintf("tcp://0.0.0.0:%d", start),
+		fmt.Sprintf("tcp://0.0.0.0:%d", start+1),
+		fmt.Sprintf("tcp://0.0.0.0:%d", start+2)
+}
+
+// getConfig returns a config for test cases
+func getConfig() *cfg.Config {
+	pathname := makePathname()
+	c := cfg.ResetTestRoot(fmt.Sprintf("%s_%d", pathname, cmn.RandInt()))
+
+	// and we use random ports to run in parallel
+	tm, rpc, grpc := makeAddrs()
+	c.P2P.ListenAddress = tm
+	c.RPC.ListenAddress = rpc
+	c.RPC.GRPCListenAddress = grpc
+	return c
+}
+
+// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops
+// when the heightToStop is reached. Client will be notified via
+// signalWhenStopsTo channel.
+type byteBufferWAL struct {
+	enc               *WALEncoder
+	stopped           bool
+	heightToStop      int64
+	signalWhenStopsTo chan<- struct{}
+
+	logger log.Logger
+}
+
+// needed for determinism
+var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")
+
+func newByteBufferWAL(logger log.Logger, enc *WALEncoder, nBlocks int64, signalStop chan<- struct{}) *byteBufferWAL {
+	return &byteBufferWAL{
+		enc:               enc,
+		heightToStop:      nBlocks,
+		signalWhenStopsTo: signalStop,
+		logger:            logger,
+	}
+}
+
+// Save writes message to the internal buffer except when heightToStop is
+// reached, in which case it will signal the caller via signalWhenStopsTo and
+// skip writing.
+func (w *byteBufferWAL) Write(m WALMessage) {
+	if w.stopped {
+		w.logger.Debug("WAL already stopped. Not writing message", "msg", m)
+		return
+	}
+
+	if endMsg, ok := m.(EndHeightMessage); ok {
+		w.logger.Debug("WAL write end height message", "height", endMsg.Height, "stopHeight", w.heightToStop)
+		if endMsg.Height == w.heightToStop {
+			w.logger.Debug("Stopping WAL at height", "height", endMsg.Height)
+			w.signalWhenStopsTo <- struct{}{}
+			w.stopped = true
+			return
+		}
+	}
+
+	w.logger.Debug("WAL Write Message", "msg", m)
+	err := w.enc.Encode(&TimedWALMessage{fixedTime, m})
+	if err != nil {
+		panic(fmt.Sprintf("failed to encode the msg %v", m))
+	}
+}
+
+func (w *byteBufferWAL) WriteSync(m WALMessage) {
+	w.Write(m)
+}
+
+func (w *byteBufferWAL) Group() *auto.Group {
+	panic("not implemented")
+}
+func (w *byteBufferWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+	return nil, false, nil
+}
+
+func (w *byteBufferWAL) Start() error { return nil }
+func (w *byteBufferWAL) Stop() error  { return nil }
+func (w *byteBufferWAL) Wait()        {}

consensus/wal_test.go

@@ -0,0 +1,133 @@
+package consensus
+
+import (
+	"bytes"
+	"crypto/rand"
+	// "sync"
+	"testing"
+	"time"
+
+	"github.com/tendermint/tendermint/consensus/types"
+	tmtypes "github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWALEncoderDecoder(t *testing.T) {
+	now := time.Now()
+	msgs := []TimedWALMessage{
+		TimedWALMessage{Time: now, Msg: EndHeightMessage{0}},
+		TimedWALMessage{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
+	}
+
+	b := new(bytes.Buffer)
+
+	for _, msg := range msgs {
+		b.Reset()
+
+		enc := NewWALEncoder(b)
+		err := enc.Encode(&msg)
+		require.NoError(t, err)
+
+		dec := NewWALDecoder(b)
+		decoded, err := dec.Decode()
+		require.NoError(t, err)
+
+		assert.Equal(t, msg.Time.UTC(), decoded.Time)
+		assert.Equal(t, msg.Msg, decoded.Msg)
+	}
+}
+
+func TestWALSearchForEndHeight(t *testing.T) {
+	walBody, err := WALWithNBlocks(6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	walFile := tempWALWithData(walBody)
+
+	wal, err := NewWAL(walFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	h := int64(3)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, cmn.Fmt("expected not to err on height %d", h))
+	assert.True(t, found, cmn.Fmt("expected to find end height for %d", h))
+	assert.NotNil(t, gr, "expected group not to be nil")
+	defer gr.Close()
+
+	dec := NewWALDecoder(gr)
+	msg, err := dec.Decode()
+	assert.NoError(t, err, "expected to decode a message")
+	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
+	assert.True(t, ok, "expected message of type EventDataRoundState")
+	assert.Equal(t, rs.Height, h+1, cmn.Fmt("wrong height"))
+}
+
+/*
+var initOnce sync.Once
+
+func registerInterfacesOnce() {
+	initOnce.Do(func() {
+		var _ = wire.RegisterInterface(
+			struct{ WALMessage }{},
+			wire.ConcreteType{[]byte{}, 0x10},
+		)
+	})
+}
+*/
+
+func nBytes(n int) []byte {
+	buf := make([]byte, n)
+	n, _ = rand.Read(buf)
+	return buf[:n]
+}
+
+func benchmarkWalDecode(b *testing.B, n int) {
+	// registerInterfacesOnce()
+
+	buf := new(bytes.Buffer)
+	enc := NewWALEncoder(buf)
+
+	data := nBytes(n)
+	enc.Encode(&TimedWALMessage{Msg: data, Time: time.Now().Round(time.Second)})
+
+	encoded := buf.Bytes()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		buf.Write(encoded)
+		dec := NewWALDecoder(buf)
+		if _, err := dec.Decode(); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.ReportAllocs()
+}
+
+func BenchmarkWalDecode512B(b *testing.B) {
+	benchmarkWalDecode(b, 512)
+}
+
+func BenchmarkWalDecode10KB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024)
+}
+func BenchmarkWalDecode100KB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024)
+}
+func BenchmarkWalDecode1MB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024)
+}
+func BenchmarkWalDecode10MB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024*1024)
+}
+func BenchmarkWalDecode100MB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024*1024)
+}
+func BenchmarkWalDecode1GB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024*1024)
+}

consensus/wire.go

@@ -0,0 +1,14 @@
+package consensus
+
+import (
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/go-crypto"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	RegisterConsensusMessages(cdc)
+	RegisterWALMessages(cdc)
+	crypto.RegisterAmino(cdc)
+}

docker-compose.yml

@@ -0,0 +1,68 @@
+version: '3'
+
+services:
+  node0:
+    container_name: node0
+    image: "tendermint/localnode"
+    ports:
+      - "26656-26657:26656-26657"
+    environment:
+      - ID=0
+      - LOG=$${LOG:-tendermint.log}
+    volumes:
+      - ./build:/tendermint:Z
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.2
+
+  node1:
+    container_name: node1
+    image: "tendermint/localnode"
+    ports:
+      - "26659-26660:26656-26657"
+    environment:
+      - ID=1
+      - LOG=$${LOG:-tendermint.log}
+    volumes:
+      - ./build:/tendermint:Z
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.3
+
+  node2:
+    container_name: node2
+    image: "tendermint/localnode"
+    environment:
+      - ID=2
+      - LOG=$${LOG:-tendermint.log}
+    ports:
+      - "26661-26662:26656-26657"
+    volumes:
+      - ./build:/tendermint:Z
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.4
+
+  node3:
+    container_name: node3
+    image: "tendermint/localnode"
+    environment:
+      - ID=3
+      - LOG=$${LOG:-tendermint.log}
+    ports:
+      - "26663-26664:26656-26657"
+    volumes:
+      - ./build:/tendermint:Z
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.5
+
+networks:
+  localnet:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+      -
+        subnet: 192.167.10.0/16
+

docs/.python-version

@@ -0,0 +1 @@
+2.7.14

docs/Makefile

@@ -12,6 +12,9 @@ BUILDDIR      = _build
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
+install:
+	@pip install -r requirements.txt
+
 .PHONY: help Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new

docs/_static/custom_collapsible_code.css

@@ -0,0 +1,17 @@
+.toggle {
+    padding-bottom: 1em ;
+}
+
+.toggle .header {
+    display: block;
+    clear: both;
+    cursor: pointer;
+}
+
+.toggle .header:after {
+    content: " ▼";
+}
+
+.toggle .header.open:after {
+    content: " ▲";
+}

docs/_static/custom_collapsible_code.js

@@ -0,0 +1,10 @@
+let makeCodeBlocksCollapsible = function() {
+    $(".toggle > *").hide();
+    $(".toggle .header").show();
+    $(".toggle .header").click(function() {
+        $(this).parent().children().not(".header").toggle({"duration": 400});
+        $(this).parent().children(".header").toggleClass("open");
+    });
+};
+// we could use the }(); way if we would have access to jQuery in HEAD, i.e. we would need to force the theme
+// to load jQuery before our custom scripts

docs/_templates/layout.html

@@ -0,0 +1,20 @@
+{% extends "!layout.html" %}
+
+{% set css_files = css_files + ["_static/custom_collapsible_code.css"] %}
+
+# sadly, I didn't find a css style way to add custom JS to a list that is automagically added to head like CSS (above) #}
+{% block extrahead %}
+<script type="text/javascript" src="_static/custom_collapsible_code.js"></script>
+{% endblock %}
+
+{% block footer %}
+<script type="text/javascript">
+    $(document).ready(function() {
+        // using this approach as we don't have access to the jQuery selectors
+        // when executing the function on load in HEAD
+        makeCodeBlocksCollapsible();
+    });
+</script>
+{% endblock %}
+
+

docs/abci-cli.md

@@ -0,0 +1,329 @@
+# Using ABCI-CLI
+
+To facilitate testing and debugging of ABCI servers and simple apps, we
+built a CLI, the `abci-cli`, for sending ABCI messages from the command
+line.
+
+## Install
+
+Make sure you [have Go installed](https://golang.org/doc/install).
+
+Next, install the `abci-cli` tool and example applications:
+
+    go get -u github.com/tendermint/abci/cmd/abci-cli
+
+If this fails, you may need to use [dep](https://github.com/golang/dep)
+to get vendored dependencies:
+
+    cd $GOPATH/src/github.com/tendermint/abci
+    make get_tools
+    make get_vendor_deps
+    make install
+
+Now run `abci-cli` to see the list of commands:
+
+    Usage:
+      abci-cli [command]
+
+    Available Commands:
+      batch       Run a batch of abci commands against an application
+      check_tx    Validate a tx
+      commit      Commit the application state and return the Merkle root hash
+      console     Start an interactive abci console for multiple commands
+      counter     ABCI demo example
+      deliver_tx  Deliver a new tx to the application
+      kvstore       ABCI demo example
+      echo        Have the application echo a message
+      help        Help about any command
+      info        Get some info about the application
+      query       Query the application state
+      set_option  Set an options on the application
+
+    Flags:
+          --abci string      socket or grpc (default "socket")
+          --address string   address of application socket (default "tcp://127.0.0.1:26658")
+      -h, --help             help for abci-cli
+      -v, --verbose          print the command and results as if it were a console session
+
+    Use "abci-cli [command] --help" for more information about a command.
+
+## KVStore - First Example
+
+The `abci-cli` tool lets us send ABCI messages to our application, to
+help build and debug them.
+
+The most important messages are `deliver_tx`, `check_tx`, and `commit`,
+but there are others for convenience, configuration, and information
+purposes.
+
+We'll start a kvstore application, which was installed at the same time
+as `abci-cli` above. The kvstore just stores transactions in a merkle
+tree.
+
+Its code can be found
+[here](https://github.com/tendermint/abci/blob/master/cmd/abci-cli/abci-cli.go)
+and looks like:
+
+    func cmdKVStore(cmd *cobra.Command, args []string) error {
+        logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+    
+        // Create the application - in memory or persisted to disk
+        var app types.Application
+        if flagPersist == "" {
+            app = kvstore.NewKVStoreApplication()
+        } else {
+            app = kvstore.NewPersistentKVStoreApplication(flagPersist)
+            app.(*kvstore.PersistentKVStoreApplication).SetLogger(logger.With("module", "kvstore"))
+        }
+    
+        // Start the listener
+        srv, err := server.NewServer(flagAddrD, flagAbci, app)
+        if err != nil {
+            return err
+        }
+        srv.SetLogger(logger.With("module", "abci-server"))
+        if err := srv.Start(); err != nil {
+            return err
+        }
+    
+        // Wait forever
+        cmn.TrapSignal(func() {
+            // Cleanup
+            srv.Stop()
+        })
+        return nil
+    }
+
+Start by running:
+
+    abci-cli kvstore
+
+And in another terminal, run
+
+    abci-cli echo hello
+    abci-cli info
+
+You'll see something like:
+
+    -> data: hello
+    -> data.hex: 68656C6C6F
+
+and:
+
+    -> data: {"size":0}
+    -> data.hex: 7B2273697A65223A307D
+
+An ABCI application must provide two things:
+
+-   a socket server
+-   a handler for ABCI messages
+
+When we run the `abci-cli` tool we open a new connection to the
+application's socket server, send the given ABCI message, and wait for a
+response.
+
+The server may be generic for a particular language, and we provide a
+[reference implementation in
+Golang](https://github.com/tendermint/abci/tree/master/server). See the
+[list of other ABCI implementations](./ecosystem.html) for servers in
+other languages.
+
+The handler is specific to the application, and may be arbitrary, so
+long as it is deterministic and conforms to the ABCI interface
+specification.
+
+So when we run `abci-cli info`, we open a new connection to the ABCI
+server, which calls the `Info()` method on the application, which tells
+us the number of transactions in our Merkle tree.
+
+Now, since every command opens a new connection, we provide the
+`abci-cli console` and `abci-cli batch` commands, to allow multiple ABCI
+messages to be sent over a single connection.
+
+Running `abci-cli console` should drop you in an interactive console for
+speaking ABCI messages to your application.
+
+Try running these commands:
+
+    > echo hello
+    -> code: OK
+    -> data: hello
+    -> data.hex: 0x68656C6C6F
+
+    > info
+    -> code: OK
+    -> data: {"size":0}
+    -> data.hex: 0x7B2273697A65223A307D
+
+    > commit
+    -> code: OK
+    -> data.hex: 0x0000000000000000
+
+    > deliver_tx "abc"
+    -> code: OK
+
+    > info
+    -> code: OK
+    -> data: {"size":1}
+    -> data.hex: 0x7B2273697A65223A317D
+
+    > commit
+    -> code: OK
+    -> data.hex: 0x0200000000000000
+
+    > query "abc"
+    -> code: OK
+    -> log: exists
+    -> height: 0
+    -> value: abc
+    -> value.hex: 616263
+
+    > deliver_tx "def=xyz"
+    -> code: OK
+
+    > commit
+    -> code: OK
+    -> data.hex: 0x0400000000000000
+
+    > query "def"
+    -> code: OK
+    -> log: exists
+    -> height: 0
+    -> value: xyz
+    -> value.hex: 78797A
+
+Note that if we do `deliver_tx "abc"` it will store `(abc, abc)`, but if
+we do `deliver_tx "abc=efg"` it will store `(abc, efg)`.
+
+Similarly, you could put the commands in a file and run
+`abci-cli --verbose batch < myfile`.
+
+## Counter - Another Example
+
+Now that we've got the hang of it, let's try another application, the
+"counter" app.
+
+Like the kvstore app, its code can be found
+[here](https://github.com/tendermint/abci/blob/master/cmd/abci-cli/abci-cli.go)
+and looks like:
+
+    func cmdCounter(cmd *cobra.Command, args []string) error {
+    
+        app := counter.NewCounterApplication(flagSerial)
+    
+        logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+    
+        // Start the listener
+        srv, err := server.NewServer(flagAddrC, flagAbci, app)
+        if err != nil {
+            return err
+        }
+        srv.SetLogger(logger.With("module", "abci-server"))
+        if err := srv.Start(); err != nil {
+            return err
+        }
+    
+        // Wait forever
+        cmn.TrapSignal(func() {
+            // Cleanup
+            srv.Stop()
+        })
+        return nil
+    }
+
+The counter app doesn't use a Merkle tree, it just counts how many times
+we've sent a transaction, asked for a hash, or committed the state. The
+result of `commit` is just the number of transactions sent.
+
+This application has two modes: `serial=off` and `serial=on`.
+
+When `serial=on`, transactions must be a big-endian encoded incrementing
+integer, starting at 0.
+
+If `serial=off`, there are no restrictions on transactions.
+
+We can toggle the value of `serial` using the `set_option` ABCI message.
+
+When `serial=on`, some transactions are invalid. In a live blockchain,
+transactions collect in memory before they are committed into blocks. To
+avoid wasting resources on invalid transactions, ABCI provides the
+`check_tx` message, which application developers can use to accept or
+reject transactions, before they are stored in memory or gossipped to
+other peers.
+
+In this instance of the counter app, `check_tx` only allows transactions
+whose integer is greater than the last committed one.
+
+Let's kill the console and the kvstore application, and start the
+counter app:
+
+    abci-cli counter
+
+In another window, start the `abci-cli console`:
+
+    > set_option serial on
+    -> code: OK
+    -> log: OK (SetOption doesn't return anything.)
+
+    > check_tx 0x00
+    -> code: OK
+
+    > check_tx 0xff
+    -> code: OK
+
+    > deliver_tx 0x00
+    -> code: OK
+
+    > check_tx 0x00
+    -> code: BadNonce
+    -> log: Invalid nonce. Expected >= 1, got 0
+
+    > deliver_tx 0x01
+    -> code: OK
+
+    > deliver_tx 0x04
+    -> code: BadNonce
+    -> log: Invalid nonce. Expected 2, got 4
+
+    > info
+    -> code: OK
+    -> data: {"hashes":0,"txs":2}
+    -> data.hex: 0x7B22686173686573223A302C22747873223A327D
+
+This is a very simple application, but between `counter` and `kvstore`,
+its easy to see how you can build out arbitrary application states on
+top of the ABCI. [Hyperledger's
+Burrow](https://github.com/hyperledger/burrow) also runs atop ABCI,
+bringing with it Ethereum-like accounts, the Ethereum virtual-machine,
+Monax's permissioning scheme, and native contracts extensions.
+
+But the ultimate flexibility comes from being able to write the
+application easily in any language.
+
+We have implemented the counter in a number of languages [see the
+example directory](https://github.com/tendermint/abci/tree/master/example).
+
+To run the Node JS version, `cd` to `example/js` and run
+
+    node app.js
+
+(you'll have to kill the other counter application process). In another
+window, run the console and those previous ABCI commands. You should get
+the same results as for the Go version.
+
+## Bounties
+
+Want to write the counter app in your favorite language?! We'd be happy
+to add you to our [ecosystem](https://tendermint.com/ecosystem)! We're
+also offering [bounties](https://hackerone.com/tendermint/) for
+implementations in new languages!
+
+The `abci-cli` is designed strictly for testing and debugging. In a real
+deployment, the role of sending messages is taken by Tendermint, which
+connects to the app using three separate connections, each with its own
+pattern of messages.
+
+For more information, see the [application developers
+guide](./app-development.html). For examples of running an ABCI app with
+Tendermint, see the [getting started guide](./getting-started.html).
+Next is the ABCI specification.

docs/abci-cli.rst

@@ -1,255 +0,0 @@
-Using ABCI-CLI
-==============
-
-To facilitate testing and debugging of ABCI servers and simple apps, we
-built a CLI, the ``abci-cli``, for sending ABCI messages from the
-command line.
-
-Install
--------
-
-Make sure you `have Go installed <https://golang.org/doc/install>`__.
-
-Next, install the ``abci-cli`` tool and example applications:
-
-::
-
-    go get -u github.com/tendermint/abci/cmd/...
-
-If this fails, you may need to use ``glide`` to get vendored
-dependencies:
-
-::
-
-    go get github.com/Masterminds/glide
-    cd $GOPATH/src/github.com/tendermint/abci
-    glide install
-    go install ./cmd/...
-
-Now run ``abci-cli --help`` to see the list of commands:
-
-::
-
-    COMMANDS:
-       batch        Run a batch of ABCI commands against an application
-       console      Start an interactive console for multiple commands
-       echo         Have the application echo a message
-       info         Get some info about the application
-       set_option   Set an option on the application
-       deliver_tx    Append a new tx to application
-       check_tx     Validate a tx
-       commit       Get application Merkle root hash
-       help, h      Shows a list of commands or help for one command
-
-    GLOBAL OPTIONS:
-       --address "tcp://127.0.0.1:46658"    address of application socket
-       --help, -h                           show help
-       --version, -v                        print the version
-
-Dummy - First Example
----------------------
-
-The ``abci-cli`` tool lets us send ABCI messages to our application, to
-help build and debug them.
-
-The most important messages are ``deliver_tx``, ``check_tx``, and
-``commit``, but there are others for convenience, configuration, and
-information purposes.
-
-Let's start a dummy application, which was installed at the same time as
-``abci-cli`` above. The dummy just stores transactions in a merkle tree:
-
-::
-
-    dummy
-
-In another terminal, run
-
-::
-
-    abci-cli echo hello
-    abci-cli info
-
-The application should echo ``hello`` and give you some information
-about itself.
-
-An ABCI application must provide two things:
-
--  a socket server
--  a handler for ABCI messages
-
-When we run the ``abci-cli`` tool we open a new connection to the
-application's socket server, send the given ABCI message, and wait for a
-response.
-
-The server may be generic for a particular language, and we provide a
-`reference implementation in
-Golang <https://github.com/tendermint/abci/tree/master/server>`__. See
-the `list of other ABCI
-implementations <https://tendermint.com/ecosystem>`__ for servers in
-other languages.
-
-The handler is specific to the application, and may be arbitrary, so
-long as it is deterministic and conforms to the ABCI interface
-specification.
-
-So when we run ``abci-cli info``, we open a new connection to the ABCI
-server, which calls the ``Info()`` method on the application, which
-tells us the number of transactions in our Merkle tree.
-
-Now, since every command opens a new connection, we provide the
-``abci-cli console`` and ``abci-cli batch`` commands, to allow multiple
-ABCI messages to be sent over a single connection.
-
-Running ``abci-cli console`` should drop you in an interactive console
-for speaking ABCI messages to your application.
-
-Try running these commands:
-
-::
-
-    > echo hello
-    -> data: hello
-
-    > info
-    -> data: {"size":0}
-
-    > commit
-    -> data: 0x
-
-    > deliver_tx "abc"
-    -> code: OK
-
-    > info
-    -> data: {"size":1}
-
-    > commit
-    -> data: 0x750502FC7E84BBD788ED589624F06CFA871845D1
-
-    > query "abc"
-    -> code: OK
-    -> data: {"index":0,"value":"abc","exists":true}
-
-    > deliver_tx "def=xyz"
-    -> code: OK
-
-    > commit
-    -> data: 0x76393B8A182E450286B0694C629ECB51B286EFD5
-
-    > query "def"
-    -> code: OK
-    -> data: {"index":1,"value":"xyz","exists":true}
-
-Note that if we do ``deliver_tx "abc"`` it will store ``(abc, abc)``,
-but if we do ``deliver_tx "abc=efg"`` it will store ``(abc, efg)``.
-
-Similarly, you could put the commands in a file and run
-``abci-cli --verbose batch < myfile``.
-
-Counter - Another Example
--------------------------
-
-Now that we've got the hang of it, let's try another application, the
-"counter" app.
-
-The counter app doesn't use a Merkle tree, it just counts how many times
-we've sent a transaction, asked for a hash, or committed the state. The
-result of ``commit`` is just the number of transactions sent.
-
-This application has two modes: ``serial=off`` and ``serial=on``.
-
-When ``serial=on``, transactions must be a big-endian encoded
-incrementing integer, starting at 0.
-
-If ``serial=off``, there are no restrictions on transactions.
-
-We can toggle the value of ``serial`` using the ``set_option`` ABCI
-message.
-
-When ``serial=on``, some transactions are invalid. In a live blockchain,
-transactions collect in memory before they are committed into blocks. To
-avoid wasting resources on invalid transactions, ABCI provides the
-``check_tx`` message, which application developers can use to accept or
-reject transactions, before they are stored in memory or gossipped to
-other peers.
-
-In this instance of the counter app, ``check_tx`` only allows
-transactions whose integer is greater than the last committed one.
-
-Let's kill the console and the dummy application, and start the counter
-app:
-
-::
-
-    counter
-
-In another window, start the ``abci-cli console``:
-
-::
-
-    > set_option serial on
-    -> data: serial=on
-
-    > check_tx 0x00
-    -> code: OK
-
-    > check_tx 0xff
-    -> code: OK
-
-    > deliver_tx 0x00
-    -> code: OK
-
-    > check_tx 0x00
-    -> code: BadNonce
-    -> log: Invalid nonce. Expected >= 1, got 0
-
-    > deliver_tx 0x01
-    -> code: OK
-
-    > deliver_tx 0x04
-    -> code: BadNonce
-    -> log: Invalid nonce. Expected 2, got 4
-
-    > info
-    -> data: {"hashes":0,"txs":2}
-
-This is a very simple application, but between ``counter`` and
-``dummy``, its easy to see how you can build out arbitrary application
-states on top of the ABCI. `Hyperledger's
-Burrow <https://github.com/hyperledger/burrow>`__ also runs atop ABCI,
-bringing with it Ethereum-like accounts, the Ethereum virtual-machine,
-Monax's permissioning scheme, and native contracts extensions.
-
-But the ultimate flexibility comes from being able to write the
-application easily in any language.
-
-We have implemented the counter in a number of languages (see the
-example directory).
-
-To run the Node JS version, ``cd`` to ``example/js`` and run
-
-::
-
-    node app.js
-
-(you'll have to kill the other counter application process). In another
-window, run the console and those previous ABCI commands. You should get
-the same results as for the Go version.
-
-Bounties
---------
-
-Want to write the counter app in your favorite language?! We'd be happy
-to add you to our `ecosystem <https://tendermint.com/ecosystem>`__!
-We're also offering `bounties <https://tendermint.com/bounties>`__ for
-implementations in new languages!
-
-The ``abci-cli`` is designed strictly for testing and debugging. In a
-real deployment, the role of sending messages is taken by Tendermint,
-which connects to the app using three separate connections, each with
-its own pattern of messages.
-
-For more information, see the `application developers
-guide <./app-development.html>`__. For examples of running an ABCI
-app with Tendermint, see the `getting started
-guide <./getting-started.html>`__.

docs/app-architecture.md

@@ -0,0 +1,50 @@
+# Application Architecture Guide
+
+Here we provide a brief guide on the recommended architecture of a
+Tendermint blockchain application.
+
+The following diagram provides a superb example:
+
+<https://drive.google.com/open?id=1yR2XpRi9YCY9H9uMfcw8-RMJpvDyvjz9>
+
+The end-user application here is the Cosmos Voyager, at the bottom left.
+Voyager communicates with a REST API exposed by a local Light-Client
+Daemon. The Light-Client Daemon is an application specific program that
+communicates with Tendermint nodes and verifies Tendermint light-client
+proofs through the Tendermint Core RPC. The Tendermint Core process
+communicates with a local ABCI application, where the user query or
+transaction is actually processed.
+
+The ABCI application must be a deterministic result of the Tendermint
+consensus - any external influence on the application state that didn't
+come through Tendermint could cause a consensus failure. Thus *nothing*
+should communicate with the application except Tendermint via ABCI.
+
+If the application is written in Go, it can be compiled into the
+Tendermint binary. Otherwise, it should use a unix socket to communicate
+with Tendermint. If it's necessary to use TCP, extra care must be taken
+to encrypt and authenticate the connection.
+
+All reads from the app happen through the Tendermint `/abci_query`
+endpoint. All writes to the app happen through the Tendermint
+`/broadcast_tx_*` endpoints.
+
+The Light-Client Daemon is what provides light clients (end users) with
+nearly all the security of a full node. It formats and broadcasts
+transactions, and verifies proofs of queries and transaction results.
+Note that it need not be a daemon - the Light-Client logic could instead
+be implemented in the same process as the end-user application.
+
+Note for those ABCI applications with weaker security requirements, the
+functionality of the Light-Client Daemon can be moved into the ABCI
+application process itself. That said, exposing the application process
+to anything besides Tendermint over ABCI requires extreme caution, as
+all transactions, and possibly all queries, should still pass through
+Tendermint.
+
+See the following for more extensive documentation:
+- [Interchain Standard for the Light-Client REST API](https://github.com/cosmos/cosmos-sdk/pull/1028)
+- [Tendermint RPC Docs](https://tendermint.github.io/slate/)
+- [Tendermint in Production](https://github.com/tendermint/tendermint/pull/1618)
+- [Tendermint Basics](https://tendermint.readthedocs.io/en/master/using-tendermint.html)
+- [ABCI spec](https://github.com/tendermint/abci/blob/develop/specification.md)

docs/app-architecture.rst

@@ -1,125 +0,0 @@
-Application Architecture Guide
-==============================
-
-Overview
---------
-
-A blockchain application is more than the consensus engine and the
-transaction logic (eg. smart contracts, business logic) as implemented
-in the ABCI app. There are also (mobile, web, desktop) clients that will
-need to connect and make use of the app. We will assume for now that you
-have a well designed transactions and database model, but maybe this
-will be the topic of another article. This article is more interested in
-various ways of setting up the "plumbing" and connecting these pieces,
-and demonstrating some evolving best practices.
-
-Security
---------
-
-A very important aspect when constructing a blockchain is security. The
-consensus model can be DoSed (no consensus possible) by corrupting 1/3
-of the validators and exploited (writing arbitrary blocks) by corrupting
-2/3 of the validators. So, while the security is not that of the
-"weakest link", you should take care that the "average link" is
-sufficiently hardened.
-
-One big attack surface on the validators is the communication between
-the ABCI app and the tendermint core. This should be highly protected.
-Ideally, the app and the core are running on the same machine, so no
-external agent can target the communication channel. You can use unix
-sockets (with permissions preventing access from other users), or even
-compile the two apps into one binary if the ABCI app is also writen in
-go. If you are unable to do that due to language support, then the ABCI
-app should bind a TCP connection to localhost (127.0.0.1), which is less
-efficient and secure, but still not reachable from outside. If you must
-run the ABCI app and tendermint core on separate machines, make sure you
-have a secure communication channel (ssh tunnel?)
-
-Now assuming, you have linked together your app and the core securely,
-you must also make sure no one can get on the machine it is hosted on.
-At this point it is basic network security. Run on a secure operating
-system (SELinux?). Limit who has access to the machine (user accounts,
-but also where the physical machine is hosted). Turn off all services
-except for ssh, which should only be accessible by some well-guarded
-public/private key pairs (no password). And maybe even firewall off
-access to the ports used by the validators, so only known validators can
-connect.
-
-There was also a suggestion on slack from @jhon about compiling
-everything together with a unikernel for more security, such as
-`Mirage <https://mirage.io>`__ or
-`UNIK <https://github.com/emc-advanced-dev/unik>`__.
-
-Connecting your client to the blockchain
-----------------------------------------
-
-Tendermint Core RPC
-~~~~~~~~~~~~~~~~~~~
-
-The concept is that the ABCI app is completely hidden from the outside
-world and only communicated through a tested and secured `interface
-exposed by the tendermint core <./specification/rpc.html>`__. This interface
-exposes a lot of data on the block header and consensus process, which
-is quite useful for externally verifying the system. It also includes
-3(!) methods to broadcast a transaction (propose it for the blockchain,
-and possibly await a response). And one method to query app-specific
-data from the ABCI application.
-
-Pros:
-* Server code already written
-* Access to block headers to validate merkle proofs (nice for light clients)
-* Basic read/write functionality is supported
-
-Cons:
-* Limited interface to app. All queries must be serialized into
-[]byte (less expressive than JSON over HTTP) and there is no way to push
-data from ABCI app to the client (eg. notify me if account X receives a
-transaction)
-
-Custom ABCI server
-~~~~~~~~~~~~~~~~~~
-
-This was proposed by @wolfposd on slack and demonstrated by
-`TMChat <https://github.com/wolfposd/TMChat>`__, a sample app. The
-concept is to write a custom server for your app (with typical REST
-API/websockets/etc for easy use by a mobile app). This custom server is
-in the same binary as the ABCI app and data store, so can easily react
-to complex events there that involve understanding the data format (send
-a message if my balance drops below 500). All "writes" sent to this
-server are proxied via websocket/JSON-RPC to tendermint core. When they
-come back as deliver\_tx over ABCI, they will be written to the data
-store. For "reads", we can do any queries we wish that are supported by
-our architecture, using any web technology that is useful. The general
-architecture is shown in the following diagram:
-
-Pros: \* Separates application logic from blockchain logic \* Allows
-much richer, more flexible client-facing API \* Allows pub-sub, watching
-certain fields, etc.
-
-Cons: \* Access to ABCI app can be dangerous (be VERY careful not to
-write unless it comes from the validator node) \* No direct access to
-the blockchain headers to verify tx \* You must write your own API (but
-maybe that's a pro...)
-
-Hybrid solutions
-~~~~~~~~~~~~~~~~
-
-Likely the least secure but most versatile. The client can access both
-the tendermint node for all blockchain info, as well as a custom app
-server, for complex queries and pub-sub on the abci app.
-
-Pros: All from both above solutions
-
-Cons: Even more complexity; even more attack vectors (less
-security)
-
-Scalability
------------
-
-Read replica using non-validating nodes? They could forward transactions
-to the validators (fewer connections, more security), and locally allow
-all queries in any of the above configurations. Thus, while
-transaction-processing speed is limited by the speed of the abci app and
-the number of validators, one should be able to scale our read
-performance to quite an extent (until the replication process drains too
-many resources from the validator nodes).