Merge pull request #960 from tendermint/release-v0.14.0

Release v0.14.0

.codecov.yml

@@ -19,3 +19,8 @@ coverage:
 comment:
   layout: "header, diff"
   behavior: default  # update if exists else create new
+
+ignore:
+  - "docs"
+  - "*.md"
+  - "*.rst"

.editorconfig

@@ -8,10 +8,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[Makefile]
-indent_style = tab
-
-[*.sh]
+[*.{sh,Makefile}]
 indent_style = tab
 
 [*.proto]

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS: https://help.github.com/articles/about-codeowners/
+
+# Everything goes through Bucky and Anton. For now.
+*       @ebuchman @melekes

.gitignore

@@ -15,3 +15,8 @@ test/p2p/data/
 test/logs
 .glide
 coverage.txt
+docs/_build
+docs/tools
+
+scripts/wal2json/wal2json
+scripts/cutWALUntil/cutWALUntil

CHANGELOG.md

@@ -13,7 +13,6 @@ FEATURES:
 - Use the chain as its own CA for nodes and validators
 - Tooling to run multiple blockchains/apps, possibly in a single process
 - State syncing (without transaction replay)
-- Improved support for querying history and state
 - Add authentication and rate-limitting to the RPC
 
 IMPROVEMENTS:
@@ -28,17 +27,113 @@ BUG FIXES:
 - Graceful handling/recovery for apps that have non-determinism or fail to halt
 - Graceful handling/recovery for violations of safety, or liveness
 
+## 0.14.0 (December 11, 2017)
+
+BREAKING CHANGES:
+- consensus/wal: removed separator
+- rpc/client: changed Subscribe/Unsubscribe/UnsubscribeAll funcs signatures to be identical to event bus.
+
+FEATURES:
+- new `tendermint lite` command (and `lite/proxy` pkg) for running a light-client RPC proxy.
+    NOTE it is currently insecure and its APIs are not yet covered by semver
+
+IMPROVEMENTS:
+- rpc/client: can act as event bus subscriber (See https://github.com/tendermint/tendermint/issues/945).
+- p2p: use exponential backoff from seconds to hours when attempting to reconnect to persistent peer
+- config: moniker defaults to the machine's hostname instead of "anonymous"
+
+BUG FIXES:
+- p2p: no longer exit if one of the seed addresses is incorrect
+
+## 0.13.0 (December 6, 2017)
+
+BREAKING CHANGES:
+- abci: update to v0.8 using gogo/protobuf; includes tx tags, vote info in RequestBeginBlock, data.Bytes everywhere, use int64, etc.
+- types: block heights are now `int64` everywhere
+- types & node: EventSwitch and EventCache have been replaced by EventBus and EventBuffer; event types have been overhauled
+- node: EventSwitch methods now refer to EventBus
+- rpc/lib/types: RPCResponse is no longer a pointer; WSRPCConnection interface has been modified
+- rpc/client: WaitForOneEvent takes an EventsClient instead of types.EventSwitch
+- rpc/client: Add/RemoveListenerForEvent are now Subscribe/Unsubscribe
+- rpc/core/types: ResultABCIQuery wraps an abci.ResponseQuery
+- rpc: `/subscribe` and `/unsubscribe` take `query` arg instead of `event`
+- rpc: `/status` returns the LatestBlockTime in human readable form instead of in nanoseconds
+- mempool: cached transactions return an error instead of an ABCI response with BadNonce
+
+FEATURES:
+- rpc: new `/unsubscribe_all` WebSocket RPC endpoint
+- rpc: new `/tx_search` endpoint for filtering transactions by more complex queries
+- p2p/trust: new trust metric for tracking peers. See ADR-006
+- config: TxIndexConfig allows to set what DeliverTx tags to index
+
+IMPROVEMENTS:
+- New asynchronous events system using `tmlibs/pubsub`
+- logging: Various small improvements
+- consensus: Graceful shutdown when app crashes
+- tests: Fix various non-deterministic errors
+- p2p: more defensive programming
+
+BUG FIXES:
+- consensus: fix panic where prs.ProposalBlockParts is not initialized
+- p2p: fix panic on bad channel
+
+## 0.12.1 (November 27, 2017)
+
+BUG FIXES:
+- upgrade tmlibs dependency to enable Windows builds for Tendermint
+
+## 0.12.0 (October 27, 2017)
+
+BREAKING CHANGES:
+ - rpc/client: websocket ResultsCh and ErrorsCh unified in ResponsesCh.
+ - rpc/client: ABCIQuery no longer takes `prove`
+ - state: remove GenesisDoc from state.
+ - consensus: new binary WAL format provides efficiency and uses checksums to detect corruption
+    - use scripts/wal2json to convert to json for debugging
+
+FEATURES:
+ - new `certifiers` pkg contains the tendermint light-client library (name subject to change)!
+ - rpc: `/genesis` includes the `app_options` .
+ - rpc: `/abci_query` takes an additional `height` parameter to support historical queries.
+ - rpc/client: new ABCIQueryWithOptions supports options like `trusted` (set false to get a proof) and `height` to query a historical height.
+
+IMPROVEMENTS:
+ - rpc: `/genesis` result includes `app_options`
+ - rpc/lib/client: add jitter to reconnects.
+ - rpc/lib/types: `RPCError` satisfies the `error` interface.
+
+BUG FIXES:
+ - rpc/client: fix ws deadlock after stopping
+ - blockchain: fix panic on AddBlock when peer is nil
+ - mempool: fix sending on TxsAvailable when a tx has been invalidated
+ - consensus: dont run WAL catchup if we fast synced
+
+## 0.11.1 (October 10, 2017)
+
+IMPROVEMENTS:
+ - blockchain/reactor: respondWithNoResponseMessage for missing height
+
+BUG FIXES:
+ - rpc: fixed client WebSocket timeout
+ - rpc: client now resubscribes on reconnection
+ - rpc: fix panics on missing params
+ - rpc: fix `/dump_consensus_state` to have normal json output (NOTE: technically breaking, but worth a bug fix label)
+ - types: fixed out of range error in VoteSet.addVote
+ - consensus: fix wal autofile via https://github.com/tendermint/tmlibs/blob/master/CHANGELOG.md#032-october-2-2017
+
 ## 0.11.0 (September 22, 2017)
 
 BREAKING:
- - state: every validator set change is persisted to disk, which required some changes to the `State` structure.
- - cmd: if there is no genesis, exit immediately instead of waiting around for one to show.
- - p2p: new `p2p.Peer` interface used for all reactor methods (instead of `*p2p.Peer` struct).
- - types: `Signer.Sign` returns an error.
+ - genesis file: validator `amount` is now `power`
+ - abci: Info, BeginBlock, InitChain all take structs
  - rpc: various changes to match JSONRPC spec (http://www.jsonrpc.org/specification), including breaking ones:
     - requests that previously returned HTTP code 4XX now return 200 with an error code in the JSONRPC.
     - `rpctypes.RPCResponse` uses new `RPCError` type instead of `string`.
- - abci: Info, BeginBlock, InitChain all take structs
+
+ - cmd: if there is no genesis, exit immediately instead of waiting around for one to show.
+ - types: `Signer.Sign` returns an error.
+ - state: every validator set change is persisted to disk, which required some changes to the `State` structure.
+ - p2p: new `p2p.Peer` interface used for all reactor methods (instead of `*p2p.Peer` struct).
 
 FEATURES:
  - rpc: `/validators?height=X` allows querying of validators at previous heights.

CONTRIBUTING.md

@@ -8,9 +8,9 @@ Please make sure to use `gofmt` before every commit - the easiest way to do this
 
 ## Forking
 
-Please note that Go requires code to live under absolute paths, which complicates forking. 
-While my fork lives at `https://github.com/ebuchman/tendermint`, 
-the code should never exist at  `$GOPATH/src/github.com/ebuchman/tendermint`. 
+Please note that Go requires code to live under absolute paths, which complicates forking.
+While my fork lives at `https://github.com/ebuchman/tendermint`,
+the code should never exist at  `$GOPATH/src/github.com/ebuchman/tendermint`.
 Instead, we use `git remote` to add the fork as a new remote for the original repo,
 `$GOPATH/src/github.com/tendermint/tendermint `, and do all the work there.
 
@@ -38,11 +38,22 @@ We use [glide](https://github.com/masterminds/glide) to manage dependencies.
 That said, the master branch of every Tendermint repository should just build with `go get`, which means they should be kept up-to-date with their dependencies so we can get away with telling people they can just `go get` our software.
 Since some dependencies are not under our control, a third party may break our build, in which case we can fall back on `glide install`. Even for dependencies under our control, glide helps us keeps multiple repos in sync as they evolve. Anything with an executable, such as apps, tools, and the core, should use glide.
 
-Run `bash scripts/glide/status.sh` to get a list of vendored dependencies that may not be up-to-date. 
+Run `bash scripts/glide/status.sh` to get a list of vendored dependencies that may not be up-to-date.
+
+## Vagrant
+
+If you are a [Vagrant](https://www.vagrantup.com/) user, all you have to do to get started hacking Tendermint is:
+
+```
+vagrant up
+vagrant ssh
+cd ~/go/src/github.com/tendermint/tendermint
+make test
+```
 
 ## Testing
 
-All repos should be hooked up to circle. 
+All repos should be hooked up to circle.
 If they have `.go` files in the root directory, they will be automatically tested by circle using `go test -v -race ./...`. If not, they will need a `circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and includes its continuous integration status using a badge in the `README.md`.
 
 ## Branching Model and Release
@@ -75,3 +86,15 @@ especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint
 - push to release-vX.X.X to run the extended integration tests on the CI
 - merge to master
 - merge master back to develop
+
+### Hotfix Procedure:
+- start on `master`
+- checkout a new branch named hotfix-vX.X.X
+- make the required changes
+  - these changes should be small and an absolute necessity
+  - add a note to CHANGELOG.md
+- bumb versions
+- push to hotfix-vX.X.X to run the extended integration tests on the CI
+- merge hotfix-vX.X.X to master
+- merge hotfix-vX.X.X to develop
+- delete the hotfix-vX.X.X branch

DOCKER/Dockerfile

@@ -1,8 +1,8 @@
 FROM alpine:3.6
 
 # This is the release of tendermint to pull in.
-ENV TM_VERSION 0.10.0
-ENV TM_SHA256SUM a29852b8d51c00db93c87c3d148fa419a047abd38f32b2507a905805131acc19
+ENV TM_VERSION 0.13.0
+ENV TM_SHA256SUM 36d773d4c2890addc61cc87a72c1e9c21c89516921b0defb0edfebde719b4b85
 
 # Tendermint will be looking for genesis file in /tendermint (unless you change
 # `genesis_file` in config.toml). You can put your config.toml and private

DOCKER/README.md

@@ -1,6 +1,10 @@
 # Supported tags and respective `Dockerfile` links
 
-- `0.10.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/e5342f4054ab784b2cd6150e14f01053d7c8deb2/DOCKER/Dockerfile)
+- `0.13.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/a28b3fff49dce2fb31f90abb2fc693834e0029c2/DOCKER/Dockerfile)
+- `0.12.1` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/457c688346b565e90735431619ca3ca597ef9007/DOCKER/Dockerfile)
+- `0.12.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/70d8afa6e952e24c573ece345560a5971bf2cc0e/DOCKER/Dockerfile)
+- `0.11.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/9177cc1f64ca88a4a0243c5d1773d10fba67e201/DOCKER/Dockerfile)
+- `0.10.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/e5342f4054ab784b2cd6150e14f01053d7c8deb2/DOCKER/Dockerfile)
 - `0.9.1`, `0.9`, [(Dockerfile)](https://github.com/tendermint/tendermint/blob/809e0e8c5933604ba8b2d096803ada7c5ec4dfd3/DOCKER/Dockerfile)
 - `0.9.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/d474baeeea6c22b289e7402449572f7c89ee21da/DOCKER/Dockerfile)
 - `0.8.0`, `0.8` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/bf64dd21fdb193e54d8addaaaa2ecf7ac371de8c/DOCKER/Dockerfile)
@@ -8,13 +12,24 @@
 
 `develop` tag points to the [develop](https://github.com/tendermint/tendermint/tree/develop) branch.
 
+# Quick reference
+
+* **Where to get help:**
+  https://tendermint.com/community
+
+* **Where to file issues:**
+  https://github.com/tendermint/tendermint/issues
+
+* **Supported Docker versions:**
+  [the latest release](https://github.com/moby/moby/releases) (down to 1.6 on a best-effort basis)
+
 # Tendermint
 
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines.
 
-For more background, see the [introduction](https://tendermint.com/intro).
+For more background, see the [introduction](https://tendermint.readthedocs.io/en/master/introduction.html).
 
-To get started developing applications, see the [application developers guide](https://tendermint.com/docs/guides/app-development).
+To get started developing applications, see the [application developers guide](https://tendermint.readthedocs.io/en/master/getting-started.html).
 
 # How to use this image
 
@@ -31,26 +46,12 @@ docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint node --proxy_app
 
 If you want to see many containers talking to each other, consider using [mintnet-kubernetes](https://github.com/tendermint/tools/tree/master/mintnet-kubernetes), which is a tool for running Tendermint-based applications on a Kubernetes cluster.
 
-# Supported Docker versions
-
-This image is officially supported on Docker version 1.13.1.
-
-Support for older versions (down to 1.6) is provided on a best-effort basis.
-
-Please see [the Docker installation documentation](https://docs.docker.com/installation/) for details on how to upgrade your Docker daemon.
-
 # License
 
 View [license information](https://raw.githubusercontent.com/tendermint/tendermint/master/LICENSE) for the software contained in this image.
 
 # User Feedback
 
-## Issues
-
-If you have any problems with or questions about this image, please contact us through a [GitHub](https://github.com/tendermint/tendermint/issues) issue. If the issue is related to a CVE, please check for [a `cve-tracker` issue on the `official-images` repository](https://github.com/docker-library/official-images/issues?q=label%3Acve-tracker) first.
-
-You can also reach the image maintainers via [Slack](http://forum.tendermint.com:3000/).
-
 ## Contributing
 
 You are invited to contribute new features, fixes, or updates, large or small; we are always thrilled to receive pull requests, and do our best to process them as fast as we can.

INSTALL.md

@@ -1 +0,0 @@
-The installation guide has moved to the [docs directory](docs/guides/install-from-source.md) in order to easily be rendered by the website. Please update your links accordingly.

Makefile

@@ -1,30 +1,32 @@
 GOTOOLS = \
 					github.com/mitchellh/gox \
-					github.com/Masterminds/glide \
-					honnef.co/go/tools/cmd/megacheck
+					github.com/tcnksm/ghr \
+					gopkg.in/alecthomas/gometalinter.v2
 
 PACKAGES=$(shell go list ./... | grep -v '/vendor/')
 BUILD_TAGS?=tendermint
 TMHOME = $${TMHOME:-$$HOME/.tendermint}
 
-all: install test
+BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short HEAD`"
 
-install: get_vendor_deps
-	@go install --ldflags '-extldflags "-static"' \
-		--ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" ./cmd/tendermint
+all: get_vendor_deps install test
+
+install:
+	go install $(BUILD_FLAGS) ./cmd/tendermint
 
 build:
-	go build \
-		--ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`"  -o build/tendermint ./cmd/tendermint/
+	go build $(BUILD_FLAGS) -o build/tendermint ./cmd/tendermint/
 
 build_race:
-	go build -race -o build/tendermint ./cmd/tendermint
+	go build -race $(BUILD_FLAGS) -o build/tendermint ./cmd/tendermint
 
 # dist builds binaries for all platforms and packages them for distribution
 dist:
 	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
 
 test:
+	@echo "--> Running linter"
+	@make metalinter_test
 	@echo "--> Running go test"
 	@go test $(PACKAGES)
 
@@ -35,49 +37,73 @@ test_race:
 test_integrations:
 	@bash ./test/test.sh
 
+test_release:
+	@go test -tags release $(PACKAGES)
+
 test100:
 	@for i in {1..100}; do make test; done
 
+vagrant_test:
+	vagrant up
+	vagrant ssh -c 'make install'
+	vagrant ssh -c 'make test_race'
+	vagrant ssh -c 'make test_integrations'
+
 draw_deps:
 	# requires brew install graphviz or apt-get install graphviz
 	go get github.com/RobotsAndPencils/goviz
 	@goviz -i github.com/tendermint/tendermint/cmd/tendermint -d 3 | dot -Tpng -o dependency-graph.png
 
-list_deps:
-	@go list -f '{{join .Deps "\n"}}' ./... | \
-		grep -v /vendor/ | sort | uniq | \
-		xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
-
-get_deps:
-	@echo "--> Running go get"
-	@go get -v -d $(PACKAGES)
-	@go list -f '{{join .TestImports "\n"}}' ./... | \
-		grep -v /vendor/ | sort | uniq | \
-		xargs go get -v -d
-
-get_vendor_deps: ensure_tools
+get_vendor_deps:
+	@hash glide 2>/dev/null || go get github.com/Masterminds/glide
 	@rm -rf vendor/
 	@echo "--> Running glide install"
-	@glide install
+	@$(GOPATH)/bin/glide install
 
-update_deps: tools
-	@echo "--> Updating dependencies"
-	@go get -d -u ./...
+update_vendor_deps:
+	@$(GOPATH)/bin/glide update
 
-revision:
-	-echo `git rev-parse --verify HEAD` > $(TMHOME)/revision
-	-echo `git rev-parse --verify HEAD` >> $(TMHOME)/revision_history
+update_tools:
+	@echo "--> Updating tools"
+	@go get -u $(GOTOOLS)
 
 tools:
-	go get -u -v $(GOTOOLS)
-
-ensure_tools:
-	go get $(GOTOOLS)
+	@echo "--> Installing tools"
+	@go get $(GOTOOLS)
+	$(GOPATH)/bin/gometalinter.v2 --install
 
 ### Formatting, linting, and vetting
 
-megacheck:
-	@for pkg in ${PACKAGES}; do megacheck "$$pkg"; done
+metalinter:
+	$(GOPATH)/bin/gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./...
 
+metalinter_test:
+	$(GOPATH)/bin/gometalinter.v2 --vendor --deadline=600s --disable-all  \
+		--enable=deadcode \
+		--enable=gosimple \
+	 	--enable=misspell \
+		--enable=safesql \
+		./...
 
-.PHONY: install build build_race dist test test_race test_integrations test100 draw_deps list_deps get_deps get_vendor_deps update_deps revision tools
+		#--enable=gas \
+		#--enable=maligned \
+		#--enable=dupl \
+		#--enable=errcheck \
+		#--enable=goconst \
+		#--enable=gocyclo \
+		#--enable=goimports \
+		#--enable=golint \ <== comments on anything exported
+		#--enable=gotype \
+	 	#--enable=ineffassign \
+	   	#--enable=interfacer \
+	   	#--enable=megacheck \
+	   	#--enable=staticcheck \
+	   	#--enable=structcheck \
+	   	#--enable=unconvert \
+	   	#--enable=unparam \
+		#--enable=unused \
+	   	#--enable=varcheck \
+		#--enable=vet \
+		#--enable=vetshadow \
+
+.PHONY: install build build_race dist test test_race test_integrations test100 draw_deps get_vendor_deps update_vendor_deps update_tools tools test_release

README.md

@@ -8,12 +8,13 @@ Or [Blockchain](https://en.wikipedia.org/wiki/Blockchain_(database)) for short.
 [![API Reference](
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/tendermint/tendermint)
+[![Go version](https://img.shields.io/badge/go-1.9.2-blue.svg)](https://github.com/moovweb/gvm)
 [![Rocket.Chat](https://demo.rocket.chat/images/join-chat.svg)](https://cosmos.rocket.chat/)
 [![license](https://img.shields.io/github/license/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/blob/master/LICENSE)
 [![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint)
 
 
-Branch    | Tests | Coverage 
+Branch    | Tests | Coverage
 ----------|-------|----------
 master    | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint)
 develop   | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/develop/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint)
@@ -23,7 +24,7 @@ _NOTE: This is alpha software. Please contact us if you intend to run it in prod
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language -
 and securely replicates it on many machines.
 
-For more information, from introduction to install to application development, [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master).
+For more information, from introduction to install to application development, [Read The Docs](https://tendermint.readthedocs.io/en/master/).
 
 ## Install
 
@@ -33,13 +34,13 @@ To install from source, you should be able to:
 
 `go get -u github.com/tendermint/tendermint/cmd/tendermint`
 
-For more details (or if it fails), [read the docs](http://tendermint.readthedocs.io/projects/tools/en/master/install.html).
+For more details (or if it fails), [read the docs](https://tendermint.readthedocs.io/en/master/install.html).
 
 ## Resources
 
 ### Tendermint Core
 
-All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs.
+All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](https://tendermint.readthedocs.io/en/master/). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs.
 
 ### Sub-projects
 
@@ -56,6 +57,7 @@ All resources involving the use of, building application on, or developing for,
 
 * [Ethermint](http://github.com/tendermint/ethermint); Ethereum on Tendermint
 * [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework
+* [Many more](https://tendermint.readthedocs.io/en/master/ecosystem.html#abci-applications)
 
 ### More
 

Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
 
   config.vm.provider "virtualbox" do |v|
     v.memory = 4096
@@ -10,29 +10,40 @@ Vagrant.configure("2") do |config|
   end
 
   config.vm.provision "shell", inline: <<-SHELL
+    # add docker repo
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable"
+
+    # and golang 1.9 support
+    # official repo doesn't have race detection runtime...
+    # add-apt-repository ppa:gophers/archive
+    add-apt-repository ppa:longsleep/golang-backports
+
+    # install base requirements
     apt-get update
-    apt-get install -y --no-install-recommends wget curl jq shellcheck bsdmainutils psmisc
+    apt-get install -y --no-install-recommends wget curl jq \
+        make shellcheck bsdmainutils psmisc
+    apt-get install -y docker-ce golang-1.9-go
 
-    wget -qO- https://get.docker.com/ | sh
-    usermod -a -G docker vagrant
-    apt-get autoremove -y
+    # needed for docker
+    usermod -a -G docker ubuntu
 
-    curl -O https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
-    tar -xvf go1.8.linux-amd64.tar.gz
-    rm -rf /usr/local/go
-    mv go /usr/local
-    rm -f go1.8.linux-amd64.tar.gz
-    mkdir -p /home/vagrant/go/bin
-    echo 'export PATH=$PATH:/usr/local/go/bin:/home/vagrant/go/bin' >> /home/vagrant/.bash_profile
-    echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.bash_profile
+    # use "EOF" not EOF to avoid variable substitution of $PATH
+    cat << "EOF" >> /home/ubuntu/.bash_profile
+export PATH=$PATH:/usr/lib/go-1.9/bin:/home/ubuntu/go/bin
+export GOPATH=/home/ubuntu/go
+export LC_ALL=en_US.UTF-8
+cd go/src/github.com/tendermint/tendermint
+EOF
 
-    echo 'export LC_ALL=en_US.UTF-8' >> /home/vagrant/.bash_profile
+    mkdir -p /home/ubuntu/go/bin
+    mkdir -p /home/ubuntu/go/src/github.com/tendermint
+    ln -s /vagrant /home/ubuntu/go/src/github.com/tendermint/tendermint
 
-    mkdir -p /home/vagrant/go/src/github.com/tendermint
-    ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint
+    chown -R ubuntu:ubuntu /home/ubuntu/go
+    chown ubuntu:ubuntu /home/ubuntu/.bash_profile
 
-    chown -R vagrant:vagrant /home/vagrant/go
-
-    su - vagrant -c 'cd /home/vagrant/go/src/github.com/tendermint/tendermint && make get_vendor_deps'
+    # get all deps and tools, ready to install/test
+    su - ubuntu -c 'cd /home/ubuntu/go/src/github.com/tendermint/tendermint && make get_vendor_deps && make tools'
   SHELL
 end

appveyor.yml

@@ -0,0 +1,13 @@
+version: 1.0.{build}
+configuration: Release
+platform:
+- x64
+- x86
+clone_folder: c:\go\path\src\github.com\tendermint\tendermint
+before_build:
+- cmd: set GOPATH=%GOROOT%\path
+- cmd: set PATH=%GOPATH%\bin;%PATH%
+- cmd: make get_vendor_deps
+build_script:
+- cmd: make test
+test: off

benchmarks/blockchain/.gitignore

@@ -0,0 +1,2 @@
+data
+

benchmarks/blockchain/localsync.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+DATA=$GOPATH/src/github.com/tendermint/tendermint/benchmarks/blockchain/data
+if [ ! -d $DATA ]; then
+  echo "no data found, generating a chain... (this only has to happen once)"
+
+  tendermint init --home $DATA
+  cp $DATA/config.toml $DATA/config2.toml
+  echo "
+  [consensus]
+  timeout_commit = 0
+  " >> $DATA/config.toml
+
+  echo "starting node"
+  tendermint node \
+    --home $DATA \
+    --proxy_app dummy \
+    --p2p.laddr tcp://127.0.0.1:56656 \
+    --rpc.laddr tcp://127.0.0.1:56657 \
+    --log_level error &
+
+  echo "making blocks for 60s"
+  sleep 60
+
+  mv $DATA/config2.toml $DATA/config.toml
+
+  kill %1
+
+  echo "done generating chain."
+fi
+
+# validator node
+HOME1=$TMPDIR$RANDOM$RANDOM
+cp -R $DATA $HOME1
+echo "starting validator node"
+tendermint node \
+  --home $HOME1 \
+  --proxy_app dummy \
+  --p2p.laddr tcp://127.0.0.1:56656 \
+  --rpc.laddr tcp://127.0.0.1:56657 \
+  --log_level error &
+sleep 1
+
+# downloader node
+HOME2=$TMPDIR$RANDOM$RANDOM
+tendermint init --home $HOME2
+cp $HOME1/genesis.json $HOME2
+printf "starting downloader node"
+tendermint node \
+  --home $HOME2 \
+  --proxy_app dummy \
+  --p2p.laddr tcp://127.0.0.1:56666 \
+  --rpc.laddr tcp://127.0.0.1:56667 \
+  --p2p.seeds 127.0.0.1:56656 \
+  --log_level error &
+
+# wait for node to start up so we only count time where we are actually syncing
+sleep 0.5
+while curl localhost:56667/status 2> /dev/null | grep "\"latest_block_height\": 0," > /dev/null
+do
+  printf '.'
+  sleep 0.2
+done
+echo
+
+echo "syncing blockchain for 10s"
+for i in {1..10}
+do
+  sleep 1
+  HEIGHT="$(curl localhost:56667/status 2> /dev/null \
+    | grep 'latest_block_height' \
+    | grep -o ' [0-9]*' \
+    | xargs)"
+  let 'RATE = HEIGHT / i'
+  echo "height: $HEIGHT, blocks/sec: $RATE"
+done
+
+kill %1
+kill %2
+rm -rf $HOME1 $HOME2

benchmarks/codec_test.go

@@ -2,11 +2,13 @@ package benchmarks
 
 import (
 	"testing"
+	"time"
 
 	"github.com/tendermint/go-crypto"
-	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/go-wire"
+
 	proto "github.com/tendermint/tendermint/benchmarks/proto"
+	"github.com/tendermint/tendermint/p2p"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 )
 
@@ -26,7 +28,7 @@ func BenchmarkEncodeStatusWire(b *testing.B) {
 		PubKey:            pubKey,
 		LatestBlockHash:   []byte("SOMEBYTES"),
 		LatestBlockHeight: 123,
-		LatestBlockTime:   1234,
+		LatestBlockTime:   time.Unix(0, 1234),
 	}
 	b.StartTimer()
 

benchmarks/map_test.go

@@ -1,8 +1,9 @@
 package benchmarks
 
 import (
-	. "github.com/tendermint/tmlibs/common"
 	"testing"
+
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 func BenchmarkSomething(b *testing.B) {
@@ -11,11 +12,11 @@ func BenchmarkSomething(b *testing.B) {
 	numChecks := 100000
 	keys := make([]string, numItems)
 	for i := 0; i < numItems; i++ {
-		keys[i] = RandStr(100)
+		keys[i] = cmn.RandStr(100)
 	}
 	txs := make([]string, numChecks)
 	for i := 0; i < numChecks; i++ {
-		txs[i] = RandStr(100)
+		txs[i] = cmn.RandStr(100)
 	}
 	b.StartTimer()
 

benchmarks/os_test.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"testing"
 
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 func BenchmarkFileWrite(b *testing.B) {
@@ -14,16 +14,20 @@ func BenchmarkFileWrite(b *testing.B) {
 	if err != nil {
 		b.Error(err)
 	}
-	testString := RandStr(200) + "\n"
+	testString := cmn.RandStr(200) + "\n"
 	b.StartTimer()
 
 	for i := 0; i < b.N; i++ {
-		file.Write([]byte(testString))
+		_, err := file.Write([]byte(testString))
+		if err != nil {
+			b.Error(err)
+		}
 	}
 
-	file.Close()
-	err = os.Remove("benchmark_file_write.out")
-	if err != nil {
+	if err := file.Close(); err != nil {
+		b.Error(err)
+	}
+	if err := os.Remove("benchmark_file_write.out"); err != nil {
 		b.Error(err)
 	}
 }

benchmarks/proto/test.pb.go

@@ -24,9 +24,6 @@ import bytes "bytes"
 
 import strings "strings"
 import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
-import sort "sort"
-import strconv "strconv"
-import reflect "reflect"
 
 import io "io"
 
@@ -392,31 +389,6 @@ func (this *PubKeyEd25519) GoString() string {
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
-func valueToGoStringTest(v interface{}, typ string) string {
-	rv := reflect.ValueOf(v)
-	if rv.IsNil() {
-		return "nil"
-	}
-	pv := reflect.Indirect(rv).Interface()
-	return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
-}
-func extensionToGoStringTest(e map[int32]github_com_gogo_protobuf_proto.Extension) string {
-	if e == nil {
-		return "nil"
-	}
-	s := "map[int32]proto.Extension{"
-	keys := make([]int, 0, len(e))
-	for k := range e {
-		keys = append(keys, int(k))
-	}
-	sort.Ints(keys)
-	ss := []string{}
-	for _, k := range keys {
-		ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString())
-	}
-	s += strings.Join(ss, ",") + "}"
-	return s
-}
 func (m *ResultStatus) Marshal() (data []byte, err error) {
 	size := m.Size()
 	data = make([]byte, size)
@@ -586,24 +558,6 @@ func (m *PubKeyEd25519) MarshalTo(data []byte) (int, error) {
 	return i, nil
 }
 
-func encodeFixed64Test(data []byte, offset int, v uint64) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	data[offset+4] = uint8(v >> 32)
-	data[offset+5] = uint8(v >> 40)
-	data[offset+6] = uint8(v >> 48)
-	data[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Test(data []byte, offset int, v uint32) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintTest(data []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		data[offset] = uint8(v&0x7f | 0x80)
@@ -689,9 +643,6 @@ func sovTest(x uint64) (n int) {
 	}
 	return n
 }
-func sozTest(x uint64) (n int) {
-	return sovTest(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
 func (this *ResultStatus) String() string {
 	if this == nil {
 		return "nil"
@@ -742,14 +693,6 @@ func (this *PubKeyEd25519) String() string {
 	}, "")
 	return s
 }
-func valueToStringTest(v interface{}) string {
-	rv := reflect.ValueOf(v)
-	if rv.IsNil() {
-		return "nil"
-	}
-	pv := reflect.Indirect(rv).Interface()
-	return fmt.Sprintf("*%v", pv)
-}
 func (m *ResultStatus) Unmarshal(data []byte) error {
 	var hasFields [1]uint64
 	l := len(data)

benchmarks/simu/counter.go

@@ -3,9 +3,8 @@ package main
 import (
 	"context"
 	"encoding/binary"
-	"time"
-	//"encoding/hex"
 	"fmt"
+	"time"
 
 	rpcclient "github.com/tendermint/tendermint/rpc/lib/client"
 	cmn "github.com/tendermint/tmlibs/common"
@@ -13,7 +12,7 @@ import (
 
 func main() {
 	wsc := rpcclient.NewWSClient("127.0.0.1:46657", "/websocket")
-	_, err := wsc.Start()
+	err := wsc.Start()
 	if err != nil {
 		cmn.Exit(err.Error())
 	}
@@ -22,7 +21,7 @@ func main() {
 	// Read a bunch of responses
 	go func() {
 		for {
-			_, ok := <-wsc.ResultsCh
+			_, ok := <-wsc.ResponsesCh
 			if !ok {
 				break
 			}

blockchain/pool.go

@@ -6,16 +6,30 @@ import (
 	"time"
 
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 	flow "github.com/tendermint/tmlibs/flowrate"
 	"github.com/tendermint/tmlibs/log"
 )
 
+/*
+
+eg, L = latency = 0.1s
+	P = num peers = 10
+	FN = num full nodes
+	BS = 1kB block size
+	CB = 1 Mbit/s = 128 kB/s
+	CB/P = 12.8 kB
+	B/S = CB/P/BS = 12.8 blocks/s
+
+	12.8 * 0.1 = 1.28 blocks on conn
+
+*/
+
 const (
-	requestIntervalMS         = 250
-	maxTotalRequesters        = 300
+	requestIntervalMS         = 100
+	maxTotalRequesters        = 1000
 	maxPendingRequests        = maxTotalRequesters
-	maxPendingRequestsPerPeer = 75
+	maxPendingRequestsPerPeer = 50
 	minRecvRate               = 10240 // 10Kb/s
 )
 
@@ -33,33 +47,34 @@ var peerTimeoutSeconds = time.Duration(15) // not const so we can override with
 */
 
 type BlockPool struct {
-	BaseService
+	cmn.BaseService
 	startTime time.Time
 
 	mtx sync.Mutex
 	// block requests
-	requesters map[int]*bpRequester
-	height     int   // the lowest key in requesters.
+	requesters map[int64]*bpRequester
+	height     int64 // the lowest key in requesters.
 	numPending int32 // number of requests pending assignment or block response
 	// peers
-	peers map[string]*bpPeer
+	peers         map[string]*bpPeer
+	maxPeerHeight int64
 
 	requestsCh chan<- BlockRequest
 	timeoutsCh chan<- string
 }
 
-func NewBlockPool(start int, requestsCh chan<- BlockRequest, timeoutsCh chan<- string) *BlockPool {
+func NewBlockPool(start int64, requestsCh chan<- BlockRequest, timeoutsCh chan<- string) *BlockPool {
 	bp := &BlockPool{
 		peers: make(map[string]*bpPeer),
 
-		requesters: make(map[int]*bpRequester),
+		requesters: make(map[int64]*bpRequester),
 		height:     start,
 		numPending: 0,
 
 		requestsCh: requestsCh,
 		timeoutsCh: timeoutsCh,
 	}
-	bp.BaseService = *NewBaseService(nil, "BlockPool", bp)
+	bp.BaseService = *cmn.NewBaseService(nil, "BlockPool", bp)
 	return bp
 }
 
@@ -69,16 +84,16 @@ func (pool *BlockPool) OnStart() error {
 	return nil
 }
 
-func (pool *BlockPool) OnStop() {
-	pool.BaseService.OnStop()
-}
+func (pool *BlockPool) OnStop() {}
 
 // Run spawns requesters as needed.
 func (pool *BlockPool) makeRequestersRoutine() {
+
 	for {
 		if !pool.IsRunning() {
 			break
 		}
+
 		_, numPending, lenRequesters := pool.GetStatus()
 		if numPending >= maxPendingRequests {
 			// sleep for a bit.
@@ -117,7 +132,7 @@ func (pool *BlockPool) removeTimedoutPeers() {
 	}
 }
 
-func (pool *BlockPool) GetStatus() (height int, numPending int32, lenRequesters int) {
+func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequesters int) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
@@ -135,16 +150,10 @@ func (pool *BlockPool) IsCaughtUp() bool {
 		return false
 	}
 
-	maxPeerHeight := 0
-	for _, peer := range pool.peers {
-		maxPeerHeight = MaxInt(maxPeerHeight, peer.height)
-	}
-
 	// some conditions to determine if we're caught up
 	receivedBlockOrTimedOut := (pool.height > 0 || time.Since(pool.startTime) > 5*time.Second)
-	ourChainIsLongestAmongPeers := maxPeerHeight == 0 || pool.height >= maxPeerHeight
+	ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= pool.maxPeerHeight
 	isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers
-	pool.Logger.Info(Fmt("IsCaughtUp: %v", isCaughtUp), "height", pool.height, "maxPeerHeight", maxPeerHeight)
 	return isCaughtUp
 }
 
@@ -180,23 +189,24 @@ func (pool *BlockPool) PopRequest() {
 		delete(pool.requesters, pool.height)
 		pool.height++
 	} else {
-		PanicSanity(Fmt("Expected requester to pop, got nothing at height %v", pool.height))
+		cmn.PanicSanity(cmn.Fmt("Expected requester to pop, got nothing at height %v", pool.height))
 	}
 }
 
 // Invalidates the block at pool.height,
 // Remove the peer and redo request from others.
-func (pool *BlockPool) RedoRequest(height int) {
+func (pool *BlockPool) RedoRequest(height int64) {
 	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
 	request := pool.requesters[height]
-	pool.mtx.Unlock()
 
 	if request.block == nil {
-		PanicSanity("Expected block to be non-nil")
+		cmn.PanicSanity("Expected block to be non-nil")
 	}
 	// RemovePeer will redo all requesters associated with this peer.
 	// TODO: record this malfeasance
-	pool.RemovePeer(request.peerID)
+	pool.removePeer(request.peerID)
 }
 
 // TODO: ensure that blocks come in order for each peer.
@@ -206,20 +216,31 @@ func (pool *BlockPool) AddBlock(peerID string, block *types.Block, blockSize int
 
 	requester := pool.requesters[block.Height]
 	if requester == nil {
+		// a block we didn't expect.
+		// TODO:if height is too far ahead, punish peer
 		return
 	}
 
 	if requester.setBlock(block, peerID) {
 		pool.numPending--
 		peer := pool.peers[peerID]
-		peer.decrPending(blockSize)
+		if peer != nil {
+			peer.decrPending(blockSize)
+		}
 	} else {
 		// Bad peer?
 	}
 }
 
+// MaxPeerHeight returns the highest height reported by a peer.
+func (pool *BlockPool) MaxPeerHeight() int64 {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+	return pool.maxPeerHeight
+}
+
 // Sets the peer's alleged blockchain height.
-func (pool *BlockPool) SetPeerHeight(peerID string, height int) {
+func (pool *BlockPool) SetPeerHeight(peerID string, height int64) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
@@ -231,6 +252,10 @@ func (pool *BlockPool) SetPeerHeight(peerID string, height int) {
 		peer.setLogger(pool.Logger.With("peer", peerID))
 		pool.peers[peerID] = peer
 	}
+
+	if height > pool.maxPeerHeight {
+		pool.maxPeerHeight = height
+	}
 }
 
 func (pool *BlockPool) RemovePeer(peerID string) {
@@ -254,7 +279,7 @@ func (pool *BlockPool) removePeer(peerID string) {
 
 // Pick an available peer with at least the given minHeight.
 // If no peers are available, returns nil.
-func (pool *BlockPool) pickIncrAvailablePeer(minHeight int) *bpPeer {
+func (pool *BlockPool) pickIncrAvailablePeer(minHeight int64) *bpPeer {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
@@ -279,17 +304,24 @@ func (pool *BlockPool) makeNextRequester() {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
-	nextHeight := pool.height + len(pool.requesters)
+	nextHeight := pool.height + pool.requestersLen()
 	request := newBPRequester(pool, nextHeight)
-	request.SetLogger(pool.Logger.With("height", nextHeight))
+	// request.SetLogger(pool.Logger.With("height", nextHeight))
 
 	pool.requesters[nextHeight] = request
 	pool.numPending++
 
-	request.Start()
+	err := request.Start()
+	if err != nil {
+		request.Logger.Error("Error starting request", "err", err)
+	}
 }
 
-func (pool *BlockPool) sendRequest(height int, peerID string) {
+func (pool *BlockPool) requestersLen() int64 {
+	return int64(len(pool.requesters))
+}
+
+func (pool *BlockPool) sendRequest(height int64, peerID string) {
 	if !pool.IsRunning() {
 		return
 	}
@@ -309,12 +341,13 @@ func (pool *BlockPool) debug() string {
 	defer pool.mtx.Unlock()
 
 	str := ""
-	for h := pool.height; h < pool.height+len(pool.requesters); h++ {
+	nextHeight := pool.height + pool.requestersLen()
+	for h := pool.height; h < nextHeight; h++ {
 		if pool.requesters[h] == nil {
-			str += Fmt("H(%v):X ", h)
+			str += cmn.Fmt("H(%v):X ", h)
 		} else {
-			str += Fmt("H(%v):", h)
-			str += Fmt("B?(%v) ", pool.requesters[h].block != nil)
+			str += cmn.Fmt("H(%v):", h)
+			str += cmn.Fmt("B?(%v) ", pool.requesters[h].block != nil)
 		}
 	}
 	return str
@@ -327,7 +360,7 @@ type bpPeer struct {
 	id          string
 	recvMonitor *flow.Monitor
 
-	height     int
+	height     int64
 	numPending int32
 	timeout    *time.Timer
 	didTimeout bool
@@ -335,7 +368,7 @@ type bpPeer struct {
 	logger log.Logger
 }
 
-func newBPPeer(pool *BlockPool, peerID string, height int) *bpPeer {
+func newBPPeer(pool *BlockPool, peerID string, height int64) *bpPeer {
 	peer := &bpPeer{
 		pool:       pool,
 		id:         peerID,
@@ -394,9 +427,9 @@ func (peer *bpPeer) onTimeout() {
 //-------------------------------------
 
 type bpRequester struct {
-	BaseService
+	cmn.BaseService
 	pool       *BlockPool
-	height     int
+	height     int64
 	gotBlockCh chan struct{}
 	redoCh     chan struct{}
 
@@ -405,7 +438,7 @@ type bpRequester struct {
 	block  *types.Block
 }
 
-func newBPRequester(pool *BlockPool, height int) *bpRequester {
+func newBPRequester(pool *BlockPool, height int64) *bpRequester {
 	bpr := &bpRequester{
 		pool:       pool,
 		height:     height,
@@ -415,7 +448,7 @@ func newBPRequester(pool *BlockPool, height int) *bpRequester {
 		peerID: "",
 		block:  nil,
 	}
-	bpr.BaseService = *NewBaseService(nil, "bpRequester", bpr)
+	bpr.BaseService = *cmn.NewBaseService(nil, "bpRequester", bpr)
 	return bpr
 }
 
@@ -517,6 +550,6 @@ OUTER_LOOP:
 //-------------------------------------
 
 type BlockRequest struct {
-	Height int
+	Height int64
 	PeerID string
 }

blockchain/pool_test.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
 )
 
@@ -16,27 +16,32 @@ func init() {
 
 type testPeer struct {
 	id     string
-	height int
+	height int64
 }
 
-func makePeers(numPeers int, minHeight, maxHeight int) map[string]testPeer {
+func makePeers(numPeers int, minHeight, maxHeight int64) map[string]testPeer {
 	peers := make(map[string]testPeer, numPeers)
 	for i := 0; i < numPeers; i++ {
-		peerID := RandStr(12)
-		height := minHeight + rand.Intn(maxHeight-minHeight)
+		peerID := cmn.RandStr(12)
+		height := minHeight + rand.Int63n(maxHeight-minHeight)
 		peers[peerID] = testPeer{peerID, height}
 	}
 	return peers
 }
 
 func TestBasic(t *testing.T) {
-	start := 42
+	start := int64(42)
 	peers := makePeers(10, start+1, 1000)
 	timeoutsCh := make(chan string, 100)
 	requestsCh := make(chan BlockRequest, 100)
 	pool := NewBlockPool(start, requestsCh, timeoutsCh)
 	pool.SetLogger(log.TestingLogger())
-	pool.Start()
+
+	err := pool.Start()
+	if err != nil {
+		t.Error(err)
+	}
+
 	defer pool.Stop()
 
 	// Introduce each peer.
@@ -82,13 +87,16 @@ func TestBasic(t *testing.T) {
 }
 
 func TestTimeout(t *testing.T) {
-	start := 42
+	start := int64(42)
 	peers := makePeers(10, start+1, 1000)
 	timeoutsCh := make(chan string, 100)
 	requestsCh := make(chan BlockRequest, 100)
 	pool := NewBlockPool(start, requestsCh, timeoutsCh)
 	pool.SetLogger(log.TestingLogger())
-	pool.Start()
+	err := pool.Start()
+	if err != nil {
+		t.Error(err)
+	}
 	defer pool.Stop()
 
 	for _, peer := range peers {

blockchain/reactor.go

@@ -12,14 +12,15 @@ import (
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 )
 
 const (
 	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
 	BlockchainChannel = byte(0x40)
 
-	defaultChannelCapacity = 100
-	trySyncIntervalMS      = 100
+	defaultChannelCapacity = 1000
+	trySyncIntervalMS      = 50
 	// stop syncing when last block's time is
 	// within this much of the system time.
 	// stopSyncingDurationMinutes = 10
@@ -33,7 +34,7 @@ const (
 type consensusReactor interface {
 	// for when we switch from blockchain reactor and fast sync to
 	// the consensus machine
-	SwitchToConsensus(*sm.State)
+	SwitchToConsensus(*sm.State, int)
 }
 
 // BlockchainReactor handles long-term catchup syncing.
@@ -48,14 +49,11 @@ type BlockchainReactor struct {
 	requestsCh   chan BlockRequest
 	timeoutsCh   chan string
 
-	evsw types.EventSwitch
+	eventBus *types.EventBus
 }
 
 // NewBlockchainReactor returns new reactor instance.
 func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor {
-	if state.LastBlockHeight == store.Height()-1 {
-		store.height-- // XXX HACK, make this better
-	}
 	if state.LastBlockHeight != store.Height() {
 		cmn.PanicSanity(cmn.Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
 	}
@@ -79,11 +77,19 @@ func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus,
 	return bcR
 }
 
-// OnStart implements BaseService
+// SetLogger implements cmn.Service by setting the logger on reactor and pool.
+func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
+	bcR.BaseService.Logger = l
+	bcR.pool.Logger = l
+}
+
+// OnStart implements cmn.Service.
 func (bcR *BlockchainReactor) OnStart() error {
-	bcR.BaseReactor.OnStart()
+	if err := bcR.BaseReactor.OnStart(); err != nil {
+		return err
+	}
 	if bcR.fastSync {
-		_, err := bcR.pool.Start()
+		err := bcR.pool.Start()
 		if err != nil {
 			return err
 		}
@@ -92,7 +98,7 @@ func (bcR *BlockchainReactor) OnStart() error {
 	return nil
 }
 
-// OnStop implements BaseService
+// OnStop implements cmn.Service.
 func (bcR *BlockchainReactor) OnStop() {
 	bcR.BaseReactor.OnStop()
 	bcR.pool.Stop()
@@ -101,10 +107,10 @@ func (bcR *BlockchainReactor) OnStop() {
 // GetChannels implements Reactor
 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	return []*p2p.ChannelDescriptor{
-		&p2p.ChannelDescriptor{
+		{
 			ID:                BlockchainChannel,
-			Priority:          5,
-			SendQueueCapacity: 100,
+			Priority:          10,
+			SendQueueCapacity: 1000,
 		},
 	}
 }
@@ -114,6 +120,8 @@ func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) {
 	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
 		// doing nothing, will try later in `poolRoutine`
 	}
+	// peer is added to the pool once we receive the first
+	// bcStatusResponseMessage from the peer and call pool.SetPeerHeight
 }
 
 // RemovePeer implements Reactor by removing peer from the pool.
@@ -121,6 +129,24 @@ func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
 	bcR.pool.RemovePeer(peer.Key())
 }
 
+// respondToPeer loads a block and sends it to the requesting peer,
+// if we have it. Otherwise, we'll respond saying we don't have it.
+// According to the Tendermint spec, if all nodes are honest,
+// no node should be requesting for a block that's non-existent.
+func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, src p2p.Peer) (queued bool) {
+	block := bcR.store.LoadBlock(msg.Height)
+	if block != nil {
+		msg := &bcBlockResponseMessage{Block: block}
+		return src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
+	}
+
+	bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height)
+
+	return src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{
+		&bcNoBlockResponseMessage{Height: msg.Height},
+	})
+}
+
 // Receive implements Reactor by handling 4 types of messages (look below).
 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
 	_, msg, err := DecodeMessage(msgBytes, bcR.maxMsgSize())
@@ -134,16 +160,8 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 	// TODO: improve logic to satisfy megacheck
 	switch msg := msg.(type) {
 	case *bcBlockRequestMessage:
-		// Got a request for a block. Respond with block if we have it.
-		block := bcR.store.LoadBlock(msg.Height)
-		if block != nil {
-			msg := &bcBlockResponseMessage{Block: block}
-			queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
-			if !queued {
-				// queue is full, just ignore.
-			}
-		} else {
-			// TODO peer is asking for things we don't have.
+		if queued := bcR.respondToPeer(msg, src); !queued {
+			// Unfortunately not queued since the queue is full.
 		}
 	case *bcBlockResponseMessage:
 		// Got a block.
@@ -165,7 +183,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 // maxMsgSize returns the maximum allowable size of a
 // message on the blockchain reactor.
 func (bcR *BlockchainReactor) maxMsgSize() int {
-	return bcR.state.Params().BlockSizeParams.MaxBytes + 2
+	return bcR.state.Params.BlockSizeParams.MaxBytes + 2
 }
 
 // Handle messages from the poolReactor telling the reactor what to do.
@@ -177,6 +195,13 @@ func (bcR *BlockchainReactor) poolRoutine() {
 	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
 	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
 
+	blocksSynced := 0
+
+	chainID := bcR.state.ChainID
+
+	lastHundred := time.Now()
+	lastRate := 0.0
+
 FOR_LOOP:
 	for {
 		select {
@@ -200,18 +225,18 @@ FOR_LOOP:
 			}
 		case <-statusUpdateTicker.C:
 			// ask for status updates
-			go bcR.BroadcastStatusRequest()
+			go bcR.BroadcastStatusRequest() // nolint: errcheck
 		case <-switchToConsensusTicker.C:
-			height, numPending, _ := bcR.pool.GetStatus()
+			height, numPending, lenRequesters := bcR.pool.GetStatus()
 			outbound, inbound, _ := bcR.Switch.NumPeers()
-			bcR.Logger.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
+			bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters,
 				"outbound", outbound, "inbound", inbound)
 			if bcR.pool.IsCaughtUp() {
 				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
 				bcR.pool.Stop()
 
 				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
-				conR.SwitchToConsensus(bcR.state)
+				conR.SwitchToConsensus(bcR.state, blocksSynced)
 
 				break FOR_LOOP
 			}
@@ -226,16 +251,16 @@ FOR_LOOP:
 					// We need both to sync the first block.
 					break SYNC_LOOP
 				}
-				firstParts := first.MakePartSet(bcR.state.Params().BlockPartSizeBytes)
+				firstParts := first.MakePartSet(bcR.state.Params.BlockPartSizeBytes)
 				firstPartsHeader := firstParts.Header()
 				// Finally, verify the first block using the second's commit
 				// NOTE: we can probably make this more efficient, but note that calling
 				// first.Hash() doesn't verify the tx contents, so MakePartSet() is
 				// currently necessary.
 				err := bcR.state.Validators.VerifyCommit(
-					bcR.state.ChainID, types.BlockID{first.Hash(), firstPartsHeader}, first.Height, second.LastCommit)
+					chainID, types.BlockID{first.Hash(), firstPartsHeader}, first.Height, second.LastCommit)
 				if err != nil {
-					bcR.Logger.Info("error in validation", "err", err)
+					bcR.Logger.Error("Error in validation", "err", err)
 					bcR.pool.RedoRequest(first.Height)
 					break SYNC_LOOP
 				} else {
@@ -247,11 +272,19 @@ FOR_LOOP:
 					// NOTE: we could improve performance if we
 					// didn't make the app commit to disk every block
 					// ... but we would need a way to get the hash without it persisting
-					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{})
+					err := bcR.state.ApplyBlock(bcR.eventBus, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{})
 					if err != nil {
 						// TODO This is bad, are we zombie?
 						cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
 					}
+					blocksSynced += 1
+
+					if blocksSynced%100 == 0 {
+						lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+						bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
+							"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
+						lastHundred = time.Now()
+					}
 				}
 			}
 			continue FOR_LOOP
@@ -267,19 +300,20 @@ func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
 	return nil
 }
 
-// SetEventSwitch implements events.Eventable
-func (bcR *BlockchainReactor) SetEventSwitch(evsw types.EventSwitch) {
-	bcR.evsw = evsw
+// SetEventBus sets event bus.
+func (bcR *BlockchainReactor) SetEventBus(b *types.EventBus) {
+	bcR.eventBus = b
 }
 
 //-----------------------------------------------------------------------------
 // Messages
 
 const (
-	msgTypeBlockRequest   = byte(0x10)
-	msgTypeBlockResponse  = byte(0x11)
-	msgTypeStatusResponse = byte(0x20)
-	msgTypeStatusRequest  = byte(0x21)
+	msgTypeBlockRequest    = byte(0x10)
+	msgTypeBlockResponse   = byte(0x11)
+	msgTypeNoBlockResponse = byte(0x12)
+	msgTypeStatusResponse  = byte(0x20)
+	msgTypeStatusRequest   = byte(0x21)
 )
 
 // BlockchainMessage is a generic message for this reactor.
@@ -289,6 +323,7 @@ var _ = wire.RegisterInterface(
 	struct{ BlockchainMessage }{},
 	wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest},
 	wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse},
+	wire.ConcreteType{&bcNoBlockResponseMessage{}, msgTypeNoBlockResponse},
 	wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse},
 	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
 )
@@ -309,13 +344,21 @@ func DecodeMessage(bz []byte, maxSize int) (msgType byte, msg BlockchainMessage,
 //-------------------------------------
 
 type bcBlockRequestMessage struct {
-	Height int
+	Height int64
 }
 
 func (m *bcBlockRequestMessage) String() string {
 	return cmn.Fmt("[bcBlockRequestMessage %v]", m.Height)
 }
 
+type bcNoBlockResponseMessage struct {
+	Height int64
+}
+
+func (brm *bcNoBlockResponseMessage) String() string {
+	return cmn.Fmt("[bcNoBlockResponseMessage %d]", brm.Height)
+}
+
 //-------------------------------------
 
 // NOTE: keep up-to-date with maxBlockchainResponseSize
@@ -330,7 +373,7 @@ func (m *bcBlockResponseMessage) String() string {
 //-------------------------------------
 
 type bcStatusRequestMessage struct {
-	Height int
+	Height int64
 }
 
 func (m *bcStatusRequestMessage) String() string {
@@ -340,7 +383,7 @@ func (m *bcStatusRequestMessage) String() string {
 //-------------------------------------
 
 type bcStatusResponseMessage struct {
-	Height int
+	Height int64
 }
 
 func (m *bcStatusResponseMessage) String() string {

blockchain/reactor_test.go

@@ -0,0 +1,151 @@
+package blockchain
+
+import (
+	"testing"
+
+	wire "github.com/tendermint/go-wire"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/p2p"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+func newBlockchainReactor(maxBlockHeight int64) *BlockchainReactor {
+	logger := log.TestingLogger()
+	config := cfg.ResetTestRoot("blockchain_reactor_test")
+
+	blockStore := NewBlockStore(dbm.NewMemDB())
+
+	// Get State
+	state, _ := sm.GetState(dbm.NewMemDB(), config.GenesisFile())
+	state.SetLogger(logger.With("module", "state"))
+	state.Save()
+
+	// Make the blockchainReactor itself
+	fastSync := true
+	bcReactor := NewBlockchainReactor(state.Copy(), nil, blockStore, fastSync)
+	bcReactor.SetLogger(logger.With("module", "blockchain"))
+
+	// Next: we need to set a switch in order for peers to be added in
+	bcReactor.Switch = p2p.NewSwitch(cfg.DefaultP2PConfig())
+
+	// Lastly: let's add some blocks in
+	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
+		firstBlock := makeBlock(blockHeight, state)
+		secondBlock := makeBlock(blockHeight+1, state)
+		firstParts := firstBlock.MakePartSet(state.Params.BlockGossipParams.BlockPartSizeBytes)
+		blockStore.SaveBlock(firstBlock, firstParts, secondBlock.LastCommit)
+	}
+
+	return bcReactor
+}
+
+func TestNoBlockMessageResponse(t *testing.T) {
+	maxBlockHeight := int64(20)
+
+	bcr := newBlockchainReactor(maxBlockHeight)
+	bcr.Start()
+	defer bcr.Stop()
+
+	// Add some peers in
+	peer := newbcrTestPeer(cmn.RandStr(12))
+	bcr.AddPeer(peer)
+
+	chID := byte(0x01)
+
+	tests := []struct {
+		height   int64
+		existent bool
+	}{
+		{maxBlockHeight + 2, false},
+		{10, true},
+		{1, true},
+		{100, false},
+	}
+
+	for _, tt := range tests {
+		reqBlockMsg := &bcBlockRequestMessage{tt.height}
+		reqBlockBytes := wire.BinaryBytes(struct{ BlockchainMessage }{reqBlockMsg})
+		bcr.Receive(chID, peer, reqBlockBytes)
+		value := peer.lastValue()
+		msg := value.(struct{ BlockchainMessage }).BlockchainMessage
+
+		if tt.existent {
+			if blockMsg, ok := msg.(*bcBlockResponseMessage); !ok {
+				t.Fatalf("Expected to receive a block response for height %d", tt.height)
+			} else if blockMsg.Block.Height != tt.height {
+				t.Fatalf("Expected response to be for height %d, got %d", tt.height, blockMsg.Block.Height)
+			}
+		} else {
+			if noBlockMsg, ok := msg.(*bcNoBlockResponseMessage); !ok {
+				t.Fatalf("Expected to receive a no block response for height %d", tt.height)
+			} else if noBlockMsg.Height != tt.height {
+				t.Fatalf("Expected response to be for height %d, got %d", tt.height, noBlockMsg.Height)
+			}
+		}
+	}
+}
+
+//----------------------------------------------
+// utility funcs
+
+func makeTxs(height int64) (txs []types.Tx) {
+	for i := 0; i < 10; i++ {
+		txs = append(txs, types.Tx([]byte{byte(height), byte(i)}))
+	}
+	return txs
+}
+
+func makeBlock(height int64, state *sm.State) *types.Block {
+	prevHash := state.LastBlockID.Hash
+	prevParts := types.PartSetHeader{}
+	valHash := state.Validators.Hash()
+	prevBlockID := types.BlockID{prevHash, prevParts}
+	block, _ := types.MakeBlock(height, "test_chain", makeTxs(height),
+		new(types.Commit), prevBlockID, valHash, state.AppHash, state.Params.BlockGossipParams.BlockPartSizeBytes)
+	return block
+}
+
+// The Test peer
+type bcrTestPeer struct {
+	cmn.Service
+	key string
+	ch  chan interface{}
+}
+
+var _ p2p.Peer = (*bcrTestPeer)(nil)
+
+func newbcrTestPeer(key string) *bcrTestPeer {
+	return &bcrTestPeer{
+		Service: cmn.NewBaseService(nil, "bcrTestPeer", nil),
+		key:     key,
+		ch:      make(chan interface{}, 2),
+	}
+}
+
+func (tp *bcrTestPeer) lastValue() interface{} { return <-tp.ch }
+
+func (tp *bcrTestPeer) TrySend(chID byte, value interface{}) bool {
+	if _, ok := value.(struct{ BlockchainMessage }).BlockchainMessage.(*bcStatusResponseMessage); ok {
+		// Discard status response messages since they skew our results
+		// We only want to deal with:
+		// + bcBlockResponseMessage
+		// + bcNoBlockResponseMessage
+	} else {
+		tp.ch <- value
+	}
+	return true
+}
+
+func (tp *bcrTestPeer) Send(chID byte, data interface{}) bool { return tp.TrySend(chID, data) }
+func (tp *bcrTestPeer) NodeInfo() *p2p.NodeInfo               { return nil }
+func (tp *bcrTestPeer) Status() p2p.ConnectionStatus          { return p2p.ConnectionStatus{} }
+func (tp *bcrTestPeer) Key() string                           { return tp.key }
+func (tp *bcrTestPeer) IsOutbound() bool                      { return false }
+func (tp *bcrTestPeer) IsPersistent() bool                    { return true }
+func (tp *bcrTestPeer) Get(s string) interface{}              { return s }
+func (tp *bcrTestPeer) Set(string, interface{})               {}

blockchain/store.go

@@ -7,10 +7,10 @@ import (
 	"io"
 	"sync"
 
-	. "github.com/tendermint/tmlibs/common"
-	dbm "github.com/tendermint/tmlibs/db"
-	"github.com/tendermint/go-wire"
+	wire "github.com/tendermint/go-wire"
 	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
 )
 
 /*
@@ -25,13 +25,14 @@ Currently the precommit signatures are duplicated in the Block parts as
 well as the Commit.  In the future this may change, perhaps by moving
 the Commit data outside the Block.
 
-Panics indicate probable corruption in the data
+// NOTE: BlockStore methods will panic if they encounter errors
+// deserializing loaded data, indicating probable corruption on disk.
 */
 type BlockStore struct {
 	db dbm.DB
 
 	mtx    sync.RWMutex
-	height int
+	height int64
 }
 
 func NewBlockStore(db dbm.DB) *BlockStore {
@@ -43,7 +44,7 @@ func NewBlockStore(db dbm.DB) *BlockStore {
 }
 
 // Height() returns the last known contiguous block height.
-func (bs *BlockStore) Height() int {
+func (bs *BlockStore) Height() int64 {
 	bs.mtx.RLock()
 	defer bs.mtx.RUnlock()
 	return bs.height
@@ -57,7 +58,7 @@ func (bs *BlockStore) GetReader(key []byte) io.Reader {
 	return bytes.NewReader(bytez)
 }
 
-func (bs *BlockStore) LoadBlock(height int) *types.Block {
+func (bs *BlockStore) LoadBlock(height int64) *types.Block {
 	var n int
 	var err error
 	r := bs.GetReader(calcBlockMetaKey(height))
@@ -66,7 +67,7 @@ func (bs *BlockStore) LoadBlock(height int) *types.Block {
 	}
 	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block meta: %v", err))
+		cmn.PanicCrisis(cmn.Fmt("Error reading block meta: %v", err))
 	}
 	bytez := []byte{}
 	for i := 0; i < blockMeta.BlockID.PartsHeader.Total; i++ {
@@ -75,12 +76,12 @@ func (bs *BlockStore) LoadBlock(height int) *types.Block {
 	}
 	block := wire.ReadBinary(&types.Block{}, bytes.NewReader(bytez), 0, &n, &err).(*types.Block)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block: %v", err))
+		cmn.PanicCrisis(cmn.Fmt("Error reading block: %v", err))
 	}
 	return block
 }
 
-func (bs *BlockStore) LoadBlockPart(height int, index int) *types.Part {
+func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
 	var n int
 	var err error
 	r := bs.GetReader(calcBlockPartKey(height, index))
@@ -89,12 +90,12 @@ func (bs *BlockStore) LoadBlockPart(height int, index int) *types.Part {
 	}
 	part := wire.ReadBinary(&types.Part{}, r, 0, &n, &err).(*types.Part)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block part: %v", err))
+		cmn.PanicCrisis(cmn.Fmt("Error reading block part: %v", err))
 	}
 	return part
 }
 
-func (bs *BlockStore) LoadBlockMeta(height int) *types.BlockMeta {
+func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
 	var n int
 	var err error
 	r := bs.GetReader(calcBlockMetaKey(height))
@@ -103,14 +104,14 @@ func (bs *BlockStore) LoadBlockMeta(height int) *types.BlockMeta {
 	}
 	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading block meta: %v", err))
+		cmn.PanicCrisis(cmn.Fmt("Error reading block meta: %v", err))
 	}
 	return blockMeta
 }
 
 // The +2/3 and other Precommit-votes for block at `height`.
 // This Commit comes from block.LastCommit for `height+1`.
-func (bs *BlockStore) LoadBlockCommit(height int) *types.Commit {
+func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
 	var n int
 	var err error
 	r := bs.GetReader(calcBlockCommitKey(height))
@@ -119,13 +120,13 @@ func (bs *BlockStore) LoadBlockCommit(height int) *types.Commit {
 	}
 	commit := wire.ReadBinary(&types.Commit{}, r, 0, &n, &err).(*types.Commit)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading commit: %v", err))
+		cmn.PanicCrisis(cmn.Fmt("Error reading commit: %v", err))
 	}
 	return commit
 }
 
 // NOTE: the Precommit-vote heights are for the block at `height`
-func (bs *BlockStore) LoadSeenCommit(height int) *types.Commit {
+func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
 	var n int
 	var err error
 	r := bs.GetReader(calcSeenCommitKey(height))
@@ -134,7 +135,7 @@ func (bs *BlockStore) LoadSeenCommit(height int) *types.Commit {
 	}
 	commit := wire.ReadBinary(&types.Commit{}, r, 0, &n, &err).(*types.Commit)
 	if err != nil {
-		PanicCrisis(Fmt("Error reading commit: %v", err))
+		cmn.PanicCrisis(cmn.Fmt("Error reading commit: %v", err))
 	}
 	return commit
 }
@@ -147,10 +148,10 @@ func (bs *BlockStore) LoadSeenCommit(height int) *types.Commit {
 func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
 	height := block.Height
 	if height != bs.Height()+1 {
-		PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
+		cmn.PanicSanity(cmn.Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
 	}
 	if !blockParts.IsComplete() {
-		PanicSanity(Fmt("BlockStore can only save complete block part sets"))
+		cmn.PanicSanity(cmn.Fmt("BlockStore can only save complete block part sets"))
 	}
 
 	// Save block meta
@@ -184,9 +185,9 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
 	bs.db.SetSync(nil, nil)
 }
 
-func (bs *BlockStore) saveBlockPart(height int, index int, part *types.Part) {
+func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) {
 	if height != bs.Height()+1 {
-		PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
+		cmn.PanicSanity(cmn.Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
 	}
 	partBytes := wire.BinaryBytes(part)
 	bs.db.Set(calcBlockPartKey(height, index), partBytes)
@@ -194,19 +195,19 @@ func (bs *BlockStore) saveBlockPart(height int, index int, part *types.Part) {
 
 //-----------------------------------------------------------------------------
 
-func calcBlockMetaKey(height int) []byte {
+func calcBlockMetaKey(height int64) []byte {
 	return []byte(fmt.Sprintf("H:%v", height))
 }
 
-func calcBlockPartKey(height int, partIndex int) []byte {
+func calcBlockPartKey(height int64, partIndex int) []byte {
 	return []byte(fmt.Sprintf("P:%v:%v", height, partIndex))
 }
 
-func calcBlockCommitKey(height int) []byte {
+func calcBlockCommitKey(height int64) []byte {
 	return []byte(fmt.Sprintf("C:%v", height))
 }
 
-func calcSeenCommitKey(height int) []byte {
+func calcSeenCommitKey(height int64) []byte {
 	return []byte(fmt.Sprintf("SC:%v", height))
 }
 
@@ -215,13 +216,13 @@ func calcSeenCommitKey(height int) []byte {
 var blockStoreKey = []byte("blockStore")
 
 type BlockStoreStateJSON struct {
-	Height int
+	Height int64
 }
 
 func (bsj BlockStoreStateJSON) Save(db dbm.DB) {
 	bytes, err := json.Marshal(bsj)
 	if err != nil {
-		PanicSanity(Fmt("Could not marshal state bytes: %v", err))
+		cmn.PanicSanity(cmn.Fmt("Could not marshal state bytes: %v", err))
 	}
 	db.SetSync(blockStoreKey, bytes)
 }
@@ -236,7 +237,7 @@ func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON {
 	bsj := BlockStoreStateJSON{}
 	err := json.Unmarshal(bytes, &bsj)
 	if err != nil {
-		PanicCrisis(Fmt("Could not unmarshal bytes: %X", bytes))
+		cmn.PanicCrisis(cmn.Fmt("Could not unmarshal bytes: %X", bytes))
 	}
 	return bsj
 }

circle.yml

@@ -7,6 +7,7 @@ machine:
     GOPATH: "$HOME/.go_project"
     PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
     PROJECT_PATH: "$PROJECT_PARENT_PATH/$CIRCLE_PROJECT_REPONAME"
+    PATH: "$HOME/.go_project/bin:${PATH}"
   hosts:
     localhost: 127.0.0.1
 
@@ -24,6 +25,7 @@ dependencies:
 
 test:
   override:
+    - cd "$PROJECT_PATH" && make tools && make get_vendor_deps && make metalinter_test
     - cd "$PROJECT_PATH" && set -o pipefail && make test_integrations 2>&1 | tee test_integrations.log:
         timeout: 1800
   post:

cmd/tendermint/commands/gen_validator.go

@@ -19,7 +19,10 @@ var GenValidatorCmd = &cobra.Command{
 
 func genValidator(cmd *cobra.Command, args []string) {
 	privValidator := types.GenPrivValidatorFS("")
-	privValidatorJSONBytes, _ := json.MarshalIndent(privValidator, "", "\t")
+	privValidatorJSONBytes, err := json.MarshalIndent(privValidator, "", "\t")
+	if err != nil {
+		panic(err)
+	}
 	fmt.Printf(`%v
 `, string(privValidatorJSONBytes))
 }

cmd/tendermint/commands/init.go

@@ -28,12 +28,14 @@ func initFiles(cmd *cobra.Command, args []string) {
 			genDoc := types.GenesisDoc{
 				ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)),
 			}
-			genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{
+			genDoc.Validators = []types.GenesisValidator{{
 				PubKey: privValidator.GetPubKey(),
 				Power:  10,
 			}}
 
-			genDoc.SaveAs(genFile)
+			if err := genDoc.SaveAs(genFile); err != nil {
+				panic(err)
+			}
 		}
 
 		logger.Info("Initialized tendermint", "genesis", config.GenesisFile(), "priv_validator", config.PrivValidatorFile())

cmd/tendermint/commands/lite.go

@@ -0,0 +1,60 @@
+package commands
+
+import (
+	"github.com/spf13/cobra"
+
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/lite/proxy"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+)
+
+// LiteCmd represents the base command when called without any subcommands
+var LiteCmd = &cobra.Command{
+	Use:   "lite",
+	Short: "Run lite-client proxy server, verifying tendermint rpc",
+	Long: `This node will run a secure proxy to a tendermint rpc server.
+
+All calls that can be tracked back to a block header by a proof
+will be verified before passing them back to the caller. Other that
+that it will present the same interface as a full tendermint node,
+just with added trust and running locally.`,
+	RunE:         runProxy,
+	SilenceUsage: true,
+}
+
+var (
+	listenAddr string
+	nodeAddr   string
+	chainID    string
+	home       string
+)
+
+func init() {
+	LiteCmd.Flags().StringVar(&listenAddr, "laddr", ":8888", "Serve the proxy on the given port")
+	LiteCmd.Flags().StringVar(&nodeAddr, "node", "localhost:46657", "Connect to a Tendermint node at this address")
+	LiteCmd.Flags().StringVar(&chainID, "chain-id", "tendermint", "Specify the Tendermint chain ID")
+	LiteCmd.Flags().StringVar(&home, "home-dir", ".tendermint-lite", "Specify the home directory")
+}
+
+func runProxy(cmd *cobra.Command, args []string) error {
+	// First, connect a client
+	node := rpcclient.NewHTTP(nodeAddr, "/websocket")
+
+	cert, err := proxy.GetCertifier(chainID, home, nodeAddr)
+	if err != nil {
+		return err
+	}
+	sc := proxy.SecureClient(node, cert)
+
+	err = proxy.StartProxy(sc, listenAddr, logger)
+	if err != nil {
+		return err
+	}
+
+	cmn.TrapSignal(func() {
+		// TODO: close up shop
+	})
+
+	return nil
+}

cmd/tendermint/commands/reset_priv_validator.go

@@ -25,10 +25,13 @@ var ResetPrivValidatorCmd = &cobra.Command{
 }
 
 // ResetAll removes the privValidator files.
-// Exported so other CLI tools can use  it
+// Exported so other CLI tools can use it.
 func ResetAll(dbDir, privValFile string, logger log.Logger) {
 	resetPrivValidatorFS(privValFile, logger)
-	os.RemoveAll(dbDir)
+	if err := os.RemoveAll(dbDir); err != nil {
+		logger.Error("Error removing directory", "err", err)
+		return
+	}
 	logger.Info("Removed all data", "dir", dbDir)
 }
 

cmd/tendermint/commands/root_test.go

@@ -3,15 +3,15 @@ package commands
 import (
 	"os"
 	"strconv"
+	"testing"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tmlibs/cli"
-
-	"testing"
 )
 
 var (
@@ -26,10 +26,15 @@ const (
 // modify in the test cases.
 // NOTE: it unsets all TM* env variables.
 func isolate(cmds ...*cobra.Command) cli.Executable {
-	os.Unsetenv("TMHOME")
-	os.Unsetenv("TM_HOME")
-	os.Unsetenv("TMROOT")
-	os.Unsetenv("TM_ROOT")
+	if err := os.Unsetenv("TMHOME"); err != nil {
+		panic(err)
+	}
+	if err := os.Unsetenv("TM_HOME"); err != nil {
+		panic(err)
+	}
+	if err := os.RemoveAll(defaultRoot); err != nil {
+		panic(err)
+	}
 
 	viper.Reset()
 	config = cfg.DefaultConfig()
@@ -70,7 +75,7 @@ func TestRootConfig(t *testing.T) {
 		{nil, nil, defaultRoot, defaults.Moniker, defaults.FastSync, dmax},
 		// try multiple ways of setting root (two flags, cli vs. env)
 		{[]string{"--home", conf}, nil, conf, cvals["moniker"], cfast, dmax},
-		{nil, map[string]string{"TMROOT": conf}, conf, cvals["moniker"], cfast, dmax},
+		{nil, map[string]string{"TMHOME": conf}, conf, cvals["moniker"], cfast, dmax},
 		// check setting p2p subflags two different ways
 		{[]string{"--p2p.max_num_peers", "420"}, nil, defaultRoot, defaults.Moniker, defaults.FastSync, 420},
 		{nil, map[string]string{"TM_P2P_MAX_NUM_PEERS": "17"}, defaultRoot, defaults.Moniker, defaults.FastSync, 17},

cmd/tendermint/commands/run_node.go

@@ -49,7 +49,7 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command {
 				return fmt.Errorf("Failed to create node: %v", err)
 			}
 
-			if _, err := n.Start(); err != nil {
+			if err := n.Start(); err != nil {
 				return fmt.Errorf("Failed to start node: %v", err)
 			} else {
 				logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())

cmd/tendermint/commands/testnet.go

@@ -63,7 +63,9 @@ func testnetFiles(cmd *cobra.Command, args []string) {
 	// Write genesis file.
 	for i := 0; i < nValidators; i++ {
 		mach := cmn.Fmt("mach%d", i)
-		genDoc.SaveAs(path.Join(dataDir, mach, "genesis.json"))
+		if err := genDoc.SaveAs(path.Join(dataDir, mach, "genesis.json")); err != nil {
+			panic(err)
+		}
 	}
 
 	fmt.Println(cmn.Fmt("Successfully initialized %v node directories", nValidators))

cmd/tendermint/main.go

@@ -15,6 +15,7 @@ func main() {
 		cmd.GenValidatorCmd,
 		cmd.InitFilesCmd,
 		cmd.ProbeUpnpCmd,
+		cmd.LiteCmd,
 		cmd.ReplayCmd,
 		cmd.ReplayConsoleCmd,
 		cmd.ResetAllCmd,
@@ -37,5 +38,7 @@ func main() {
 	rootCmd.AddCommand(cmd.NewRunNodeCmd(nodeFunc))
 
 	cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint"))
-	cmd.Execute()
+	if err := cmd.Execute(); err != nil {
+		panic(err)
+	}
 }

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 )
@@ -16,6 +17,7 @@ type Config struct {
 	P2P       *P2PConfig       `mapstructure:"p2p"`
 	Mempool   *MempoolConfig   `mapstructure:"mempool"`
 	Consensus *ConsensusConfig `mapstructure:"consensus"`
+	TxIndex   *TxIndexConfig   `mapstructure:"tx_index"`
 }
 
 // DefaultConfig returns a default configuration for a Tendermint node
@@ -26,6 +28,7 @@ func DefaultConfig() *Config {
 		P2P:        DefaultP2PConfig(),
 		Mempool:    DefaultMempoolConfig(),
 		Consensus:  DefaultConsensusConfig(),
+		TxIndex:    DefaultTxIndexConfig(),
 	}
 }
 
@@ -37,6 +40,7 @@ func TestConfig() *Config {
 		P2P:        TestP2PConfig(),
 		Mempool:    DefaultMempoolConfig(),
 		Consensus:  TestConsensusConfig(),
+		TxIndex:    DefaultTxIndexConfig(),
 	}
 }
 
@@ -93,9 +97,6 @@ type BaseConfig struct {
 	// so the app can decide if we should keep the connection or not
 	FilterPeers bool `mapstructure:"filter_peers"` // false
 
-	// What indexer to use for transactions
-	TxIndex string `mapstructure:"tx_index"`
-
 	// Database backend: leveldb | memdb
 	DBBackend string `mapstructure:"db_backend"`
 
@@ -108,14 +109,13 @@ func DefaultBaseConfig() BaseConfig {
 	return BaseConfig{
 		Genesis:           "genesis.json",
 		PrivValidator:     "priv_validator.json",
-		Moniker:           "anonymous",
+		Moniker:           defaultMoniker,
 		ProxyApp:          "tcp://127.0.0.1:46658",
 		ABCI:              "socket",
 		LogLevel:          DefaultPackageLogLevels(),
 		ProfListenAddress: "",
 		FastSync:          true,
 		FilterPeers:       false,
-		TxIndex:           "kv",
 		DBBackend:         "leveldb",
 		DBPath:            "data",
 	}
@@ -255,7 +255,7 @@ func TestP2PConfig() *P2PConfig {
 	return conf
 }
 
-// AddrBookFile returns the full path to the address bool
+// AddrBookFile returns the full path to the address book
 func (p *P2PConfig) AddrBookFile() string {
 	return rootify(p.AddrBook, p.RootDir)
 }
@@ -412,6 +412,41 @@ func (c *ConsensusConfig) SetWalFile(walFile string) {
 	c.walFile = walFile
 }
 
+//-----------------------------------------------------------------------------
+// TxIndexConfig
+
+// TxIndexConfig defines the confuguration for the transaction
+// indexer, including tags to index.
+type TxIndexConfig struct {
+	// What indexer to use for transactions
+	//
+	// Options:
+	//   1) "null" (default)
+	//   2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
+	Indexer string `mapstructure:"indexer"`
+
+	// Comma-separated list of tags to index (by default the only tag is tx hash)
+	//
+	// It's recommended to index only a subset of tags due to possible memory
+	// bloat. This is, of course, depends on the indexer's DB and the volume of
+	// transactions.
+	IndexTags string `mapstructure:"index_tags"`
+
+	// When set to true, tells indexer to index all tags. Note this may be not
+	// desirable (see the comment above). IndexTags has a precedence over
+	// IndexAllTags (i.e. when given both, IndexTags will be indexed).
+	IndexAllTags bool `mapstructure:"index_all_tags"`
+}
+
+// DefaultTxIndexConfig returns a default configuration for the transaction indexer.
+func DefaultTxIndexConfig() *TxIndexConfig {
+	return &TxIndexConfig{
+		Indexer:      "kv",
+		IndexTags:    "",
+		IndexAllTags: false,
+	}
+}
+
 //-----------------------------------------------------------------------------
 // Utils
 
@@ -422,3 +457,18 @@ func rootify(path, root string) string {
 	}
 	return filepath.Join(root, path)
 }
+
+//-----------------------------------------------------------------------------
+// Moniker
+
+var defaultMoniker = getDefaultMoniker()
+
+// getDefaultMoniker returns a default moniker, which is the host name. If runtime
+// fails to get the host name, "anonymous" will be returned.
+func getDefaultMoniker() string {
+	moniker, err := os.Hostname()
+	if err != nil {
+		moniker = "anonymous"
+	}
+	return moniker
+}

config/toml.go

@@ -12,16 +12,18 @@ import (
 /****** these are for production settings ***********/
 
 func EnsureRoot(rootDir string) {
-	cmn.EnsureDir(rootDir, 0700)
-	cmn.EnsureDir(rootDir+"/data", 0700)
+	if err := cmn.EnsureDir(rootDir, 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+	if err := cmn.EnsureDir(rootDir+"/data", 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
 
 	configFilePath := path.Join(rootDir, "config.toml")
 
 	// Write default config file if missing.
 	if !cmn.FileExists(configFilePath) {
-		// Ask user for moniker
-		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
-		cmn.MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
+		cmn.MustWriteFile(configFilePath, []byte(defaultConfig(defaultMoniker)), 0644)
 	}
 }
 
@@ -53,21 +55,23 @@ func ResetTestRoot(testName string) *Config {
 	rootDir = filepath.Join(rootDir, testName)
 	// Remove ~/.tendermint_test_bak
 	if cmn.FileExists(rootDir + "_bak") {
-		err := os.RemoveAll(rootDir + "_bak")
-		if err != nil {
+		if err := os.RemoveAll(rootDir + "_bak"); err != nil {
 			cmn.PanicSanity(err.Error())
 		}
 	}
 	// Move ~/.tendermint_test to ~/.tendermint_test_bak
 	if cmn.FileExists(rootDir) {
-		err := os.Rename(rootDir, rootDir+"_bak")
-		if err != nil {
+		if err := os.Rename(rootDir, rootDir+"_bak"); err != nil {
 			cmn.PanicSanity(err.Error())
 		}
 	}
 	// Create new dir
-	cmn.EnsureDir(rootDir, 0700)
-	cmn.EnsureDir(rootDir+"/data", 0700)
+	if err := cmn.EnsureDir(rootDir, 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
+	if err := cmn.EnsureDir(rootDir+"/data", 0700); err != nil {
+		cmn.PanicSanity(err.Error())
+	}
 
 	configFilePath := path.Join(rootDir, "config.toml")
 	genesisFilePath := path.Join(rootDir, "genesis.json")
@@ -75,8 +79,7 @@ func ResetTestRoot(testName string) *Config {
 
 	// Write default config file if missing.
 	if !cmn.FileExists(configFilePath) {
-		// Ask user for moniker
-		cmn.MustWriteFile(configFilePath, []byte(testConfig("anonymous")), 0644)
+		cmn.MustWriteFile(configFilePath, []byte(testConfig(defaultMoniker)), 0644)
 	}
 	if !cmn.FileExists(genesisFilePath) {
 		cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644)

config/toml_test.go

@@ -24,7 +24,7 @@ func TestEnsureRoot(t *testing.T) {
 	// setup temp dir for test
 	tmpDir, err := ioutil.TempDir("", "config-test")
 	require.Nil(err)
-	defer os.RemoveAll(tmpDir)
+	defer os.RemoveAll(tmpDir) // nolint: errcheck
 
 	// create root dir
 	EnsureRoot(tmpDir)
@@ -32,7 +32,7 @@ func TestEnsureRoot(t *testing.T) {
 	// make sure config is set properly
 	data, err := ioutil.ReadFile(filepath.Join(tmpDir, "config.toml"))
 	require.Nil(err)
-	assert.Equal([]byte(defaultConfig("anonymous")), data)
+	assert.Equal([]byte(defaultConfig(defaultMoniker)), data)
 
 	ensureFiles(t, tmpDir, "data")
 }
@@ -49,7 +49,7 @@ func TestEnsureTestRoot(t *testing.T) {
 	// make sure config is set properly
 	data, err := ioutil.ReadFile(filepath.Join(rootDir, "config.toml"))
 	require.Nil(err)
-	assert.Equal([]byte(testConfig("anonymous")), data)
+	assert.Equal([]byte(testConfig(defaultMoniker)), data)
 
 	// TODO: make sure the cfg returned and testconfig are the same!
 

consensus/byzantine_test.go

@@ -1,16 +1,17 @@
 package consensus
 
 import (
+	"context"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	crypto "github.com/tendermint/go-crypto"
 	data "github.com/tendermint/go-wire/data"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
-	"github.com/tendermint/tmlibs/events"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 func init() {
@@ -41,7 +42,43 @@ func TestByzantine(t *testing.T) {
 		switches[i].SetLogger(p2pLogger.With("validator", i))
 	}
 
+	eventChans := make([]chan interface{}, N)
 	reactors := make([]p2p.Reactor, N)
+	for i := 0; i < N; i++ {
+		if i == 0 {
+			css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator)
+			// make byzantine
+			css[i].decideProposal = func(j int) func(int64, int) {
+				return func(height int64, round int) {
+					byzantineDecideProposalFunc(t, height, round, css[j], switches[j])
+				}
+			}(i)
+			css[i].doPrevote = func(height int64, round int) {}
+		}
+
+		eventBus := types.NewEventBus()
+		eventBus.SetLogger(logger.With("module", "events", "validator", i))
+		err := eventBus.Start()
+		require.NoError(t, err)
+		defer eventBus.Stop()
+
+		eventChans[i] = make(chan interface{}, 1)
+		err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		require.NoError(t, err)
+
+		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
+		conR.SetLogger(logger.With("validator", i))
+		conR.SetEventBus(eventBus)
+
+		var conRI p2p.Reactor // nolint: gotype, gosimple
+		conRI = conR
+
+		if i == 0 {
+			conRI = NewByzantineReactor(conR)
+		}
+		reactors[i] = conRI
+	}
+
 	defer func() {
 		for _, r := range reactors {
 			if rr, ok := r.(*ByzantineReactor); ok {
@@ -51,40 +88,6 @@ func TestByzantine(t *testing.T) {
 			}
 		}
 	}()
-	eventChans := make([]chan interface{}, N)
-	eventLogger := logger.With("module", "events")
-	for i := 0; i < N; i++ {
-		if i == 0 {
-			css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator)
-			// make byzantine
-			css[i].decideProposal = func(j int) func(int, int) {
-				return func(height, round int) {
-					byzantineDecideProposalFunc(t, height, round, css[j], switches[j])
-				}
-			}(i)
-			css[i].doPrevote = func(height, round int) {}
-		}
-
-		eventSwitch := events.NewEventSwitch()
-		eventSwitch.SetLogger(eventLogger.With("validator", i))
-		_, err := eventSwitch.Start()
-		if err != nil {
-			t.Fatalf("Failed to start switch: %v", err)
-		}
-		eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1)
-
-		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
-		conR.SetLogger(logger.With("validator", i))
-		conR.SetEventSwitch(eventSwitch)
-
-		var conRI p2p.Reactor
-		conRI = conR
-
-		if i == 0 {
-			conRI = NewByzantineReactor(conR)
-		}
-		reactors[i] = conRI
-	}
 
 	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
 		// ignore new switch s, we already made ours
@@ -101,10 +104,10 @@ func TestByzantine(t *testing.T) {
 	// start the state machines
 	byzR := reactors[0].(*ByzantineReactor)
 	s := byzR.reactor.conS.GetState()
-	byzR.reactor.SwitchToConsensus(s)
+	byzR.reactor.SwitchToConsensus(s, 0)
 	for i := 1; i < N; i++ {
 		cr := reactors[i].(*ConsensusReactor)
-		cr.SwitchToConsensus(cr.conS.GetState())
+		cr.SwitchToConsensus(cr.conS.GetState(), 0)
 	}
 
 	// byz proposer sends one block to peers[0]
@@ -149,8 +152,8 @@ func TestByzantine(t *testing.T) {
 	case <-done:
 	case <-tick.C:
 		for i, reactor := range reactors {
-			t.Log(Fmt("Consensus Reactor %v", i))
-			t.Log(Fmt("%v", reactor))
+			t.Log(cmn.Fmt("Consensus Reactor %v", i))
+			t.Log(cmn.Fmt("%v", reactor))
 		}
 		t.Fatalf("Timed out waiting for all validators to commit first block")
 	}
@@ -159,7 +162,7 @@ func TestByzantine(t *testing.T) {
 //-------------------------------
 // byzantine consensus functions
 
-func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusState, sw *p2p.Switch) {
+func byzantineDecideProposalFunc(t *testing.T, height int64, round int, cs *ConsensusState, sw *p2p.Switch) {
 	// byzantine user should create two proposals and try to split the vote.
 	// Avoid sending on internalMsgQueue and running consensus state.
 
@@ -167,13 +170,17 @@ func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusS
 	block1, blockParts1 := cs.createProposalBlock()
 	polRound, polBlockID := cs.Votes.POLInfo()
 	proposal1 := types.NewProposal(height, round, blockParts1.Header(), polRound, polBlockID)
-	cs.privValidator.SignProposal(cs.state.ChainID, proposal1) // byzantine doesnt err
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal1); err != nil {
+		t.Error(err)
+	}
 
 	// Create a new proposal block from state/txs from the mempool.
 	block2, blockParts2 := cs.createProposalBlock()
 	polRound, polBlockID = cs.Votes.POLInfo()
 	proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID)
-	cs.privValidator.SignProposal(cs.state.ChainID, proposal2) // byzantine doesnt err
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal2); err != nil {
+		t.Error(err)
+	}
 
 	block1Hash := block1.Hash()
 	block2Hash := block2.Hash()
@@ -190,7 +197,7 @@ func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusS
 	}
 }
 
-func sendProposalAndParts(height, round int, cs *ConsensusState, peer p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) {
+func sendProposalAndParts(height int64, round int, cs *ConsensusState, peer p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) {
 	// proposal
 	msg := &ProposalMessage{Proposal: proposal}
 	peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
@@ -220,7 +227,7 @@ func sendProposalAndParts(height, round int, cs *ConsensusState, peer p2p.Peer,
 // byzantine consensus reactor
 
 type ByzantineReactor struct {
-	Service
+	cmn.Service
 	reactor *ConsensusReactor
 }
 
@@ -286,15 +293,15 @@ func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote
 }
 
 func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) (err error) {
-	proposal.Signature, err = privVal.Sign(types.SignBytes(chainID, proposal))
+	proposal.Signature, _ = privVal.Sign(types.SignBytes(chainID, proposal))
 	return nil
 }
 
 func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) (err error) {
-	heartbeat.Signature, err = privVal.Sign(types.SignBytes(chainID, heartbeat))
+	heartbeat.Signature, _ = privVal.Sign(types.SignBytes(chainID, heartbeat))
 	return nil
 }
 
 func (privVal *ByzantinePrivValidator) String() string {
-	return Fmt("PrivValidator{%X}", privVal.GetAddress())
+	return cmn.Fmt("PrivValidator{%X}", privVal.GetAddress())
 }

consensus/common.go

@@ -1,35 +0,0 @@
-package consensus
-
-import (
-	"github.com/tendermint/tendermint/types"
-)
-
-// XXX: WARNING: these functions can halt the consensus as firing events is synchronous.
-// Make sure to read off the channels, and in the case of subscribeToEventRespond, to write back on it
-
-// NOTE: if chanCap=0, this blocks on the event being consumed
-func subscribeToEvent(evsw types.EventSwitch, receiver, eventID string, chanCap int) chan interface{} {
-	// listen for event
-	ch := make(chan interface{}, chanCap)
-	types.AddListenerForEvent(evsw, receiver, eventID, func(data types.TMEventData) {
-		ch <- data
-	})
-	return ch
-}
-
-// NOTE: this blocks on receiving a response after the event is consumed
-func subscribeToEventRespond(evsw types.EventSwitch, receiver, eventID string) chan interface{} {
-	// listen for event
-	ch := make(chan interface{})
-	types.AddListenerForEvent(evsw, receiver, eventID, func(data types.TMEventData) {
-		ch <- data
-		<-ch
-	})
-	return ch
-}
-
-func discardFromChan(ch chan interface{}, n int) {
-	for i := 0; i < n; i++ {
-		<-ch
-	}
-}

consensus/common_test.go

@@ -2,6 +2,7 @@ package consensus
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,11 +16,12 @@ import (
 	abci "github.com/tendermint/abci/types"
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
+	cstypes "github.com/tendermint/tendermint/consensus/types"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
 
@@ -29,12 +31,16 @@ import (
 	"github.com/go-kit/kit/log/term"
 )
 
+const (
+	testSubscriber = "test-client"
+)
+
 // genesis, chain_id, priv_val
 var config *cfg.Config // NOTE: must be reset for each _test.go file
 var ensureTimeout = time.Second * 2
 
 func ensureDir(dir string, mode os.FileMode) {
-	if err := EnsureDir(dir, mode); err != nil {
+	if err := cmn.EnsureDir(dir, mode); err != nil {
 		panic(err)
 	}
 }
@@ -48,12 +54,12 @@ func ResetConfig(name string) *cfg.Config {
 
 type validatorStub struct {
 	Index  int // Validator index. NOTE: we don't assume validator set changes.
-	Height int
+	Height int64
 	Round  int
 	types.PrivValidator
 }
 
-var testMinPower = 10
+var testMinPower int64 = 10
 
 func NewValidatorStub(privValidator types.PrivValidator, valIndex int) *validatorStub {
 	return &validatorStub{
@@ -107,13 +113,13 @@ func incrementRound(vss ...*validatorStub) {
 //-------------------------------------------------------------------------------
 // Functions for transitioning the consensus state
 
-func startTestRound(cs *ConsensusState, height, round int) {
+func startTestRound(cs *ConsensusState, height int64, round int) {
 	cs.enterNewRound(height, round)
 	cs.startRoutines(0)
 }
 
 // Create proposal block from cs1 but sign it with vs
-func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) (proposal *types.Proposal, block *types.Block) {
+func decideProposal(cs1 *ConsensusState, vs *validatorStub, height int64, round int) (proposal *types.Proposal, block *types.Block) {
 	block, blockParts := cs1.createProposalBlock()
 	if block == nil { // on error
 		panic("error creating proposal block")
@@ -207,11 +213,14 @@ func validatePrevoteAndPrecommit(t *testing.T, cs *ConsensusState, thisRound, lo
 
 // genesis
 func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} {
-	voteCh0 := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1)
+	voteCh0 := make(chan interface{})
+	err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryVote, voteCh0)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote))
+	}
 	voteCh := make(chan interface{})
 	go func() {
-		for {
-			v := <-voteCh0
+		for v := range voteCh0 {
 			vote := v.(types.TMEventData).Unwrap().(types.EventDataVote)
 			// we only fire for our own votes
 			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
@@ -230,8 +239,12 @@ func newConsensusState(state *sm.State, pv types.PrivValidator, app abci.Applica
 }
 
 func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState {
-	// Get BlockStore
 	blockDB := dbm.NewMemDB()
+	return newConsensusStateWithConfigAndBlockStore(thisConfig, state, pv, app, blockDB)
+}
+
+func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state *sm.State, pv types.PrivValidator, app abci.Application, blockDB dbm.DB) *ConsensusState {
+	// Get BlockStore
 	blockStore := bc.NewBlockStore(blockDB)
 
 	// one for mempool, one for consensus
@@ -251,10 +264,10 @@ func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv typ
 	cs.SetLogger(log.TestingLogger())
 	cs.SetPrivValidator(pv)
 
-	evsw := types.NewEventSwitch()
-	evsw.SetLogger(log.TestingLogger().With("module", "events"))
-	cs.SetEventSwitch(evsw)
-	evsw.Start()
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+	eventBus.Start()
+	cs.SetEventBus(eventBus)
 	return cs
 }
 
@@ -266,13 +279,13 @@ func loadPrivValidator(config *cfg.Config) *types.PrivValidatorFS {
 	return privValidator
 }
 
-func fixedConsensusStateDummy() *ConsensusState {
+func fixedConsensusStateDummy(config *cfg.Config, logger log.Logger) *ConsensusState {
 	stateDB := dbm.NewMemDB()
 	state, _ := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
-	state.SetLogger(log.TestingLogger().With("module", "state"))
+	state.SetLogger(logger.With("module", "state"))
 	privValidator := loadPrivValidator(config)
 	cs := newConsensusState(state, privValidator, dummy.NewDummyApplication())
-	cs.SetLogger(log.TestingLogger())
+	cs.SetLogger(logger)
 	return cs
 }
 
@@ -296,7 +309,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 
 //-------------------------------------------------------------------------------
 
-func ensureNoNewStep(stepCh chan interface{}) {
+func ensureNoNewStep(stepCh <-chan interface{}) {
 	timer := time.NewTimer(ensureTimeout)
 	select {
 	case <-timer.C:
@@ -306,7 +319,7 @@ func ensureNoNewStep(stepCh chan interface{}) {
 	}
 }
 
-func ensureNewStep(stepCh chan interface{}) {
+func ensureNewStep(stepCh <-chan interface{}) {
 	timer := time.NewTimer(ensureTimeout)
 	select {
 	case <-timer.C:
@@ -341,12 +354,16 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
 		state, _ := sm.MakeGenesisState(db, genDoc)
 		state.SetLogger(logger.With("module", "state", "validator", i))
 		state.Save()
-		thisConfig := ResetConfig(Fmt("%s_%d", testName, i))
+		thisConfig := ResetConfig(cmn.Fmt("%s_%d", testName, i))
 		for _, opt := range configOpts {
 			opt(thisConfig)
 		}
 		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc())
+		app := appFunc()
+		vals := types.TM2PB.Validators(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], app)
 		css[i].SetLogger(logger.With("validator", i))
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
@@ -355,25 +372,30 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
 
 // nPeers = nValidators + nNotValidator
 func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application) []*ConsensusState {
-	genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower))
+	genDoc, privVals := randGenesisDoc(nValidators, false, testMinPower)
 	css := make([]*ConsensusState, nPeers)
+	logger := consensusLogger()
 	for i := 0; i < nPeers; i++ {
 		db := dbm.NewMemDB() // each state needs its own db
 		state, _ := sm.MakeGenesisState(db, genDoc)
-		state.SetLogger(log.TestingLogger().With("module", "state"))
+		state.SetLogger(logger.With("module", "state", "validator", i))
 		state.Save()
-		thisConfig := ResetConfig(Fmt("%s_%d", testName, i))
+		thisConfig := ResetConfig(cmn.Fmt("%s_%d", testName, i))
 		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
 		var privVal types.PrivValidator
 		if i < nValidators {
 			privVal = privVals[i]
 		} else {
-			_, tempFilePath := Tempfile("priv_validator_")
+			_, tempFilePath := cmn.Tempfile("priv_validator_")
 			privVal = types.GenPrivValidatorFS(tempFilePath)
 		}
 
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc())
-		css[i].SetLogger(log.TestingLogger())
+		app := appFunc()
+		vals := types.TM2PB.Validators(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, app)
+		css[i].SetLogger(logger.With("validator", i))
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
 	return css
@@ -442,12 +464,12 @@ type mockTicker struct {
 	fired    bool
 }
 
-func (m *mockTicker) Start() (bool, error) {
-	return true, nil
+func (m *mockTicker) Start() error {
+	return nil
 }
 
-func (m *mockTicker) Stop() bool {
-	return true
+func (m *mockTicker) Stop() error {
+	return nil
 }
 
 func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) {
@@ -456,7 +478,7 @@ func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) {
 	if m.onlyOnce && m.fired {
 		return
 	}
-	if ti.Step == RoundStepNewHeight {
+	if ti.Step == cstypes.RoundStepNewHeight {
 		m.c <- ti
 		m.fired = true
 	}

consensus/mempool_test.go

@@ -2,13 +2,17 @@ package consensus
 
 import (
 	"encoding/binary"
+	"fmt"
 	"testing"
 	"time"
 
-	abci "github.com/tendermint/abci/types"
-	"github.com/tendermint/tendermint/types"
+	"github.com/stretchr/testify/assert"
 
-	. "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/abci/example/code"
+	abci "github.com/tendermint/abci/types"
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/types"
 )
 
 func init() {
@@ -22,16 +26,15 @@ func TestNoProgressUntilTxsAvailable(t *testing.T) {
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
 	cs.mempool.EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	startTestRound(cs, height, round)
 
 	ensureNewStep(newBlockCh) // first block gets committed
 	ensureNoNewStep(newBlockCh)
-	deliverTxsRange(cs, 0, 2)
+	deliverTxsRange(cs, 0, 1)
 	ensureNewStep(newBlockCh) // commit txs
 	ensureNewStep(newBlockCh) // commit updated app hash
 	ensureNoNewStep(newBlockCh)
-
 }
 
 func TestProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
@@ -41,7 +44,7 @@ func TestProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
 	cs.mempool.EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	startTestRound(cs, height, round)
 
 	ensureNewStep(newBlockCh)   // first block gets committed
@@ -56,9 +59,9 @@ func TestProgressInHigherRound(t *testing.T) {
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
 	cs.mempool.EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
-	newRoundCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewRound(), 1)
-	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
 	cs.setProposal = func(proposal *types.Proposal) error {
 		if cs.Height == 2 && cs.Round == 0 {
 			// dont set the proposal in round 0 so we timeout and
@@ -73,7 +76,7 @@ func TestProgressInHigherRound(t *testing.T) {
 	ensureNewStep(newRoundCh) // first round at first height
 	ensureNewStep(newBlockCh) // first block gets committed
 	ensureNewStep(newRoundCh) // first round at next height
-	deliverTxsRange(cs, 0, 2) // we deliver txs, but dont set a proposal so we get the next round
+	deliverTxsRange(cs, 0, 1) // we deliver txs, but dont set a proposal so we get the next round
 	<-timeoutCh
 	ensureNewStep(newRoundCh) // wait for the next round
 	ensureNewStep(newBlockCh) // now we can commit the block
@@ -86,17 +89,16 @@ func deliverTxsRange(cs *ConsensusState, start, end int) {
 		binary.BigEndian.PutUint64(txBytes, uint64(i))
 		err := cs.mempool.CheckTx(txBytes, nil)
 		if err != nil {
-			panic(Fmt("Error after CheckTx: %v", err))
+			panic(cmn.Fmt("Error after CheckTx: %v", err))
 		}
 	}
 }
 
 func TestTxConcurrentWithCommit(t *testing.T) {
-
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusState(state, privVals[0], NewCounterApplication())
 	height, round := cs.Height, cs.Round
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 
 	NTxs := 10000
 	go deliverTxsRange(cs, 0, NTxs)
@@ -121,41 +123,43 @@ func TestRmBadTx(t *testing.T) {
 	// increment the counter by 1
 	txBytes := make([]byte, 8)
 	binary.BigEndian.PutUint64(txBytes, uint64(0))
-	app.DeliverTx(txBytes)
-	app.Commit()
 
-	ch := make(chan struct{})
-	cbCh := make(chan struct{})
+	resDeliver := app.DeliverTx(txBytes)
+	assert.False(t, resDeliver.IsErr(), cmn.Fmt("expected no error. got %v", resDeliver))
+
+	resCommit := app.Commit()
+	assert.False(t, resCommit.IsErr(), cmn.Fmt("expected no error. got %v", resCommit))
+
+	emptyMempoolCh := make(chan struct{})
+	checkTxRespCh := make(chan struct{})
 	go func() {
 		// Try to send the tx through the mempool.
 		// CheckTx should not err, but the app should return a bad abci code
 		// and the tx should get removed from the pool
 		err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) {
-			if r.GetCheckTx().Code != abci.CodeType_BadNonce {
+			if r.GetCheckTx().Code != code.CodeTypeBadNonce {
 				t.Fatalf("expected checktx to return bad nonce, got %v", r)
 			}
-			cbCh <- struct{}{}
+			checkTxRespCh <- struct{}{}
 		})
 		if err != nil {
-			t.Fatal("Error after CheckTx: %v", err)
+			t.Fatalf("Error after CheckTx: %v", err)
 		}
 
 		// check for the tx
 		for {
-			time.Sleep(time.Second)
 			txs := cs.mempool.Reap(1)
 			if len(txs) == 0 {
-				ch <- struct{}{}
-				return
+				emptyMempoolCh <- struct{}{}
 			}
-
+			time.Sleep(10 * time.Millisecond)
 		}
 	}()
 
 	// Wait until the tx returns
 	ticker := time.After(time.Second * 5)
 	select {
-	case <-cbCh:
+	case <-checkTxRespCh:
 		// success
 	case <-ticker:
 		t.Fatalf("Timed out waiting for tx to return")
@@ -164,7 +168,7 @@ func TestRmBadTx(t *testing.T) {
 	// Wait until the tx is removed
 	ticker = time.After(time.Second * 5)
 	select {
-	case <-ch:
+	case <-emptyMempoolCh:
 		// success
 	case <-ticker:
 		t.Fatalf("Timed out waiting for tx to be removed")
@@ -184,36 +188,44 @@ func NewCounterApplication() *CounterApplication {
 }
 
 func (app *CounterApplication) Info(req abci.RequestInfo) abci.ResponseInfo {
-	return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)}
+	return abci.ResponseInfo{Data: cmn.Fmt("txs:%v", app.txCount)}
 }
 
-func (app *CounterApplication) DeliverTx(tx []byte) abci.Result {
-	return runTx(tx, &app.txCount)
+func (app *CounterApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx {
+	txValue := txAsUint64(tx)
+	if txValue != uint64(app.txCount) {
+		return abci.ResponseDeliverTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)}
+	}
+	app.txCount += 1
+	return abci.ResponseDeliverTx{Code: code.CodeTypeOK}
 }
 
-func (app *CounterApplication) CheckTx(tx []byte) abci.Result {
-	return runTx(tx, &app.mempoolTxCount)
+func (app *CounterApplication) CheckTx(tx []byte) abci.ResponseCheckTx {
+	txValue := txAsUint64(tx)
+	if txValue != uint64(app.mempoolTxCount) {
+		return abci.ResponseCheckTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)}
+	}
+	app.mempoolTxCount += 1
+	return abci.ResponseCheckTx{Code: code.CodeTypeOK}
 }
 
-func runTx(tx []byte, countPtr *int) abci.Result {
-	count := *countPtr
+func txAsUint64(tx []byte) uint64 {
 	tx8 := make([]byte, 8)
 	copy(tx8[len(tx8)-len(tx):], tx)
-	txValue := binary.BigEndian.Uint64(tx8)
-	if txValue != uint64(count) {
-		return abci.ErrBadNonce.AppendLog(Fmt("Invalid nonce. Expected %v, got %v", count, txValue))
-	}
-	*countPtr += 1
-	return abci.OK
+	return binary.BigEndian.Uint64(tx8)
 }
 
-func (app *CounterApplication) Commit() abci.Result {
+func (app *CounterApplication) Commit() abci.ResponseCommit {
 	app.mempoolTxCount = app.txCount
 	if app.txCount == 0 {
-		return abci.OK
+		return abci.ResponseCommit{Code: code.CodeTypeOK}
 	} else {
 		hash := make([]byte, 8)
 		binary.BigEndian.PutUint64(hash, uint64(app.txCount))
-		return abci.NewResultOK(hash, "")
+		return abci.ResponseCommit{Code: code.CodeTypeOK, Data: hash}
 	}
 }

consensus/reactor.go

@@ -2,16 +2,19 @@ package consensus
 
 import (
 	"bytes"
-	"errors"
+	"context"
 	"fmt"
 	"reflect"
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	wire "github.com/tendermint/go-wire"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
 
+	cstypes "github.com/tendermint/tendermint/consensus/types"
 	"github.com/tendermint/tendermint/p2p"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -33,10 +36,10 @@ type ConsensusReactor struct {
 	p2p.BaseReactor // BaseService + p2p.Switch
 
 	conS *ConsensusState
-	evsw types.EventSwitch
 
 	mtx      sync.RWMutex
 	fastSync bool
+	eventBus *types.EventBus
 }
 
 // NewConsensusReactor returns a new ConsensusReactor with the given consensusState.
@@ -52,18 +55,22 @@ func NewConsensusReactor(consensusState *ConsensusState, fastSync bool) *Consens
 // OnStart implements BaseService.
 func (conR *ConsensusReactor) OnStart() error {
 	conR.Logger.Info("ConsensusReactor ", "fastSync", conR.FastSync())
-	conR.BaseReactor.OnStart()
+	if err := conR.BaseReactor.OnStart(); err != nil {
+		return err
+	}
 
-	// callbacks for broadcasting new steps and votes to peers
-	// upon their respective events (ie. uses evsw)
-	conR.registerEventCallbacks()
+	err := conR.startBroadcastRoutine()
+	if err != nil {
+		return err
+	}
 
 	if !conR.FastSync() {
-		_, err := conR.conS.Start()
+		err := conR.conS.Start()
 		if err != nil {
 			return err
 		}
 	}
+
 	return nil
 }
 
@@ -75,7 +82,7 @@ func (conR *ConsensusReactor) OnStop() {
 
 // SwitchToConsensus switches from fast_sync mode to consensus mode.
 // It resets the state, turns off fast_sync, and starts the consensus state-machine
-func (conR *ConsensusReactor) SwitchToConsensus(state *sm.State) {
+func (conR *ConsensusReactor) SwitchToConsensus(state *sm.State, blocksSynced int) {
 	conR.Logger.Info("SwitchToConsensus")
 	conR.conS.reconstructLastCommit(state)
 	// NOTE: The line below causes broadcastNewRoundStepRoutine() to
@@ -86,31 +93,38 @@ func (conR *ConsensusReactor) SwitchToConsensus(state *sm.State) {
 	conR.fastSync = false
 	conR.mtx.Unlock()
 
-	conR.conS.Start()
+	if blocksSynced > 0 {
+		// dont bother with the WAL if we fast synced
+		conR.conS.doWALCatchup = false
+	}
+	err := conR.conS.Start()
+	if err != nil {
+		conR.Logger.Error("Error starting conS", "err", err)
+	}
 }
 
 // GetChannels implements Reactor
 func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor {
 	// TODO optimize
 	return []*p2p.ChannelDescriptor{
-		&p2p.ChannelDescriptor{
+		{
 			ID:                StateChannel,
 			Priority:          5,
 			SendQueueCapacity: 100,
 		},
-		&p2p.ChannelDescriptor{
+		{
 			ID:                 DataChannel, // maybe split between gossiping current block and catchup stuff
 			Priority:           10,          // once we gossip the whole block there's nothing left to send until next height or round
 			SendQueueCapacity:  100,
 			RecvBufferCapacity: 50 * 4096,
 		},
-		&p2p.ChannelDescriptor{
+		{
 			ID:                 VoteChannel,
 			Priority:           5,
 			SendQueueCapacity:  100,
 			RecvBufferCapacity: 100 * 100,
 		},
-		&p2p.ChannelDescriptor{
+		{
 			ID:                 VoteSetBitsChannel,
 			Priority:           1,
 			SendQueueCapacity:  2,
@@ -301,10 +315,10 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 	}
 }
 
-// SetEventSwitch implements events.Eventable
-func (conR *ConsensusReactor) SetEventSwitch(evsw types.EventSwitch) {
-	conR.evsw = evsw
-	conR.conS.SetEventSwitch(evsw)
+// SetEventBus sets event bus.
+func (conR *ConsensusReactor) SetEventBus(b *types.EventBus) {
+	conR.eventBus = b
+	conR.conS.SetEventBus(b)
 }
 
 // FastSync returns whether the consensus reactor is in fast-sync mode.
@@ -316,24 +330,60 @@ func (conR *ConsensusReactor) FastSync() bool {
 
 //--------------------------------------
 
-// Listens for new steps and votes,
-// broadcasting the result to peers
-func (conR *ConsensusReactor) registerEventCallbacks() {
+// startBroadcastRoutine subscribes for new round steps, votes and proposal
+// heartbeats using the event bus and starts a go routine to broadcasts events
+// to peers upon receiving them.
+func (conR *ConsensusReactor) startBroadcastRoutine() error {
+	const subscriber = "consensus-reactor"
+	ctx := context.Background()
 
-	types.AddListenerForEvent(conR.evsw, "conR", types.EventStringNewRoundStep(), func(data types.TMEventData) {
-		rs := data.Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
-		conR.broadcastNewRoundStep(rs)
-	})
+	// new round steps
+	stepsCh := make(chan interface{})
+	err := conR.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, stepsCh)
+	if err != nil {
+		return errors.Wrapf(err, "failed to subscribe %s to %s", subscriber, types.EventQueryNewRoundStep)
+	}
 
-	types.AddListenerForEvent(conR.evsw, "conR", types.EventStringVote(), func(data types.TMEventData) {
-		edv := data.Unwrap().(types.EventDataVote)
-		conR.broadcastHasVoteMessage(edv.Vote)
-	})
+	// votes
+	votesCh := make(chan interface{})
+	err = conR.eventBus.Subscribe(ctx, subscriber, types.EventQueryVote, votesCh)
+	if err != nil {
+		return errors.Wrapf(err, "failed to subscribe %s to %s", subscriber, types.EventQueryVote)
+	}
 
-	types.AddListenerForEvent(conR.evsw, "conR", types.EventStringProposalHeartbeat(), func(data types.TMEventData) {
-		heartbeat := data.Unwrap().(types.EventDataProposalHeartbeat)
-		conR.broadcastProposalHeartbeatMessage(heartbeat)
-	})
+	// proposal heartbeats
+	heartbeatsCh := make(chan interface{})
+	err = conR.eventBus.Subscribe(ctx, subscriber, types.EventQueryProposalHeartbeat, heartbeatsCh)
+	if err != nil {
+		return errors.Wrapf(err, "failed to subscribe %s to %s", subscriber, types.EventQueryProposalHeartbeat)
+	}
+
+	go func() {
+		for {
+			select {
+			case data, ok := <-stepsCh:
+				if ok { // a receive from a closed channel returns the zero value immediately
+					edrs := data.(types.TMEventData).Unwrap().(types.EventDataRoundState)
+					conR.broadcastNewRoundStep(edrs.RoundState.(*cstypes.RoundState))
+				}
+			case data, ok := <-votesCh:
+				if ok {
+					edv := data.(types.TMEventData).Unwrap().(types.EventDataVote)
+					conR.broadcastHasVoteMessage(edv.Vote)
+				}
+			case data, ok := <-heartbeatsCh:
+				if ok {
+					edph := data.(types.TMEventData).Unwrap().(types.EventDataProposalHeartbeat)
+					conR.broadcastProposalHeartbeatMessage(edph)
+				}
+			case <-conR.Quit:
+				conR.eventBus.UnsubscribeAll(ctx, subscriber)
+				return
+			}
+		}
+	}()
+
+	return nil
 }
 
 func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(heartbeat types.EventDataProposalHeartbeat) {
@@ -344,8 +394,7 @@ func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(heartbeat types.
 	conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{msg})
 }
 
-func (conR *ConsensusReactor) broadcastNewRoundStep(rs *RoundState) {
-
+func (conR *ConsensusReactor) broadcastNewRoundStep(rs *cstypes.RoundState) {
 	nrsMsg, csMsg := makeRoundStepMessages(rs)
 	if nrsMsg != nil {
 		conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{nrsMsg})
@@ -381,7 +430,7 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) {
 	*/
 }
 
-func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg *CommitStepMessage) {
+func makeRoundStepMessages(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage, csMsg *CommitStepMessage) {
 	nrsMsg = &NewRoundStepMessage{
 		Height: rs.Height,
 		Round:  rs.Round,
@@ -389,7 +438,7 @@ func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg *
 		SecondsSinceStartTime: int(time.Since(rs.StartTime).Seconds()),
 		LastCommitRound:       rs.LastCommit.Round(),
 	}
-	if rs.Step == RoundStepCommit {
+	if rs.Step == cstypes.RoundStepCommit {
 		csMsg = &CommitStepMessage{
 			Height:           rs.Height,
 			BlockPartsHeader: rs.ProposalBlockParts.Header(),
@@ -443,6 +492,18 @@ OUTER_LOOP:
 		// If the peer is on a previous height, help catch up.
 		if (0 < prs.Height) && (prs.Height < rs.Height) {
 			heightLogger := logger.With("height", prs.Height)
+
+			// if we never received the commit message from the peer, the block parts wont be initialized
+			if prs.ProposalBlockParts == nil {
+				blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height)
+				if blockMeta == nil {
+					cmn.PanicCrisis(cmn.Fmt("Failed to load block %d when blockStore is at %d",
+						prs.Height, conR.conS.blockStore.Height()))
+				}
+				ps.InitProposalBlockParts(blockMeta.BlockID.PartsHeader)
+				// continue the loop since prs is a copy and not effected by this initialization
+				continue OUTER_LOOP
+			}
 			conR.gossipDataForCatchup(heightLogger, rs, prs, ps, peer)
 			continue OUTER_LOOP
 		}
@@ -491,8 +552,8 @@ OUTER_LOOP:
 	}
 }
 
-func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *RoundState,
-	prs *PeerRoundState, ps *PeerState, peer p2p.Peer) {
+func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *cstypes.RoundState,
+	prs *cstypes.PeerRoundState, ps *PeerState, peer p2p.Peer) {
 
 	if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok {
 		// Ensure that the peer's PartSetHeader is correct
@@ -522,9 +583,11 @@ func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *RoundS
 			Round:  prs.Round,  // Not our height, so it doesn't matter.
 			Part:   part,
 		}
-		logger.Debug("Sending block part for catchup", "round", prs.Round)
+		logger.Debug("Sending block part for catchup", "round", prs.Round, "index", index)
 		if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
 			ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+		} else {
+			logger.Debug("Sending block part for catchup failed")
 		}
 		return
 	} else {
@@ -606,24 +669,24 @@ OUTER_LOOP:
 	}
 }
 
-func (conR *ConsensusReactor) gossipVotesForHeight(logger log.Logger, rs *RoundState, prs *PeerRoundState, ps *PeerState) bool {
+func (conR *ConsensusReactor) gossipVotesForHeight(logger log.Logger, rs *cstypes.RoundState, prs *cstypes.PeerRoundState, ps *PeerState) bool {
 
 	// If there are lastCommits to send...
-	if prs.Step == RoundStepNewHeight {
+	if prs.Step == cstypes.RoundStepNewHeight {
 		if ps.PickSendVote(rs.LastCommit) {
 			logger.Debug("Picked rs.LastCommit to send")
 			return true
 		}
 	}
 	// If there are prevotes to send...
-	if prs.Step <= RoundStepPrevote && prs.Round != -1 && prs.Round <= rs.Round {
+	if prs.Step <= cstypes.RoundStepPrevote && prs.Round != -1 && prs.Round <= rs.Round {
 		if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) {
 			logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round)
 			return true
 		}
 	}
 	// If there are precommits to send...
-	if prs.Step <= RoundStepPrecommit && prs.Round != -1 && prs.Round <= rs.Round {
+	if prs.Step <= cstypes.RoundStepPrecommit && prs.Round != -1 && prs.Round <= rs.Round {
 		if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) {
 			logger.Debug("Picked rs.Precommits(prs.Round) to send", "round", prs.Round)
 			return true
@@ -752,54 +815,6 @@ func (conR *ConsensusReactor) StringIndented(indent string) string {
 
 //-----------------------------------------------------------------------------
 
-// PeerRoundState contains the known state of a peer.
-// NOTE: Read-only when returned by PeerState.GetRoundState().
-type PeerRoundState struct {
-	Height                   int                 // Height peer is at
-	Round                    int                 // Round peer is at, -1 if unknown.
-	Step                     RoundStepType       // Step peer is at
-	StartTime                time.Time           // Estimated start of round 0 at this height
-	Proposal                 bool                // True if peer has proposal for this round
-	ProposalBlockPartsHeader types.PartSetHeader //
-	ProposalBlockParts       *cmn.BitArray       //
-	ProposalPOLRound         int                 // Proposal's POL round. -1 if none.
-	ProposalPOL              *cmn.BitArray       // nil until ProposalPOLMessage received.
-	Prevotes                 *cmn.BitArray       // All votes peer has for this round
-	Precommits               *cmn.BitArray       // All precommits peer has for this round
-	LastCommitRound          int                 // Round of commit for last height. -1 if none.
-	LastCommit               *cmn.BitArray       // All commit precommits of commit for last height.
-	CatchupCommitRound       int                 // Round that we have commit for. Not necessarily unique. -1 if none.
-	CatchupCommit            *cmn.BitArray       // All commit precommits peer has for this height & CatchupCommitRound
-}
-
-// String returns a string representation of the PeerRoundState
-func (prs PeerRoundState) String() string {
-	return prs.StringIndented("")
-}
-
-// StringIndented returns a string representation of the PeerRoundState
-func (prs PeerRoundState) StringIndented(indent string) string {
-	return fmt.Sprintf(`PeerRoundState{
-%s  %v/%v/%v @%v
-%s  Proposal %v -> %v
-%s  POL      %v (round %v)
-%s  Prevotes   %v
-%s  Precommits %v
-%s  LastCommit %v (round %v)
-%s  Catchup    %v (round %v)
-%s}`,
-		indent, prs.Height, prs.Round, prs.Step, prs.StartTime,
-		indent, prs.ProposalBlockPartsHeader, prs.ProposalBlockParts,
-		indent, prs.ProposalPOL, prs.ProposalPOLRound,
-		indent, prs.Prevotes,
-		indent, prs.Precommits,
-		indent, prs.LastCommit, prs.LastCommitRound,
-		indent, prs.CatchupCommit, prs.CatchupCommitRound,
-		indent)
-}
-
-//-----------------------------------------------------------------------------
-
 var (
 	ErrPeerStateHeightRegression = errors.New("Error peer state height regression")
 	ErrPeerStateInvalidStartTime = errors.New("Error peer state invalid startTime")
@@ -812,7 +827,7 @@ type PeerState struct {
 	logger log.Logger
 
 	mtx sync.Mutex
-	PeerRoundState
+	cstypes.PeerRoundState
 }
 
 // NewPeerState returns a new PeerState for the given Peer
@@ -820,7 +835,7 @@ func NewPeerState(peer p2p.Peer) *PeerState {
 	return &PeerState{
 		Peer:   peer,
 		logger: log.NewNopLogger(),
-		PeerRoundState: PeerRoundState{
+		PeerRoundState: cstypes.PeerRoundState{
 			Round:              -1,
 			ProposalPOLRound:   -1,
 			LastCommitRound:    -1,
@@ -836,7 +851,7 @@ func (ps *PeerState) SetLogger(logger log.Logger) *PeerState {
 
 // GetRoundState returns an atomic snapshot of the PeerRoundState.
 // There's no point in mutating it since it won't change PeerState.
-func (ps *PeerState) GetRoundState() *PeerRoundState {
+func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
 
@@ -846,7 +861,7 @@ func (ps *PeerState) GetRoundState() *PeerRoundState {
 
 // GetHeight returns an atomic snapshot of the PeerRoundState's height
 // used by the mempool to ensure peers are caught up before broadcasting new txs
-func (ps *PeerState) GetHeight() int {
+func (ps *PeerState) GetHeight() int64 {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
 	return ps.PeerRoundState.Height
@@ -871,8 +886,21 @@ func (ps *PeerState) SetHasProposal(proposal *types.Proposal) {
 	ps.ProposalPOL = nil // Nil until ProposalPOLMessage received.
 }
 
+// InitProposalBlockParts initializes the peer's proposal block parts header and bit array.
+func (ps *PeerState) InitProposalBlockParts(partsHeader types.PartSetHeader) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.ProposalBlockParts != nil {
+		return
+	}
+
+	ps.ProposalBlockPartsHeader = partsHeader
+	ps.ProposalBlockParts = cmn.NewBitArray(partsHeader.Total)
+}
+
 // SetHasProposalBlockPart sets the given block part index as known for the peer.
-func (ps *PeerState) SetHasProposalBlockPart(height int, round int, index int) {
+func (ps *PeerState) SetHasProposalBlockPart(height int64, round int, index int) {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
 
@@ -923,7 +951,7 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote
 	return nil, false
 }
 
-func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *cmn.BitArray {
+func (ps *PeerState) getVoteBitArray(height int64, round int, type_ byte) *cmn.BitArray {
 	if !types.IsVoteTypeValid(type_) {
 		return nil
 	}
@@ -970,7 +998,7 @@ func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *cmn.BitArra
 }
 
 // 'round': A round for which we have a +2/3 commit.
-func (ps *PeerState) ensureCatchupCommitRound(height, round int, numValidators int) {
+func (ps *PeerState) ensureCatchupCommitRound(height int64, round int, numValidators int) {
 	if ps.Height != height {
 		return
 	}
@@ -996,13 +1024,13 @@ func (ps *PeerState) ensureCatchupCommitRound(height, round int, numValidators i
 // what votes this peer has received.
 // NOTE: It's important to make sure that numValidators actually matches
 // what the node sees as the number of validators for height.
-func (ps *PeerState) EnsureVoteBitArrays(height int, numValidators int) {
+func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
 	ps.ensureVoteBitArrays(height, numValidators)
 }
 
-func (ps *PeerState) ensureVoteBitArrays(height int, numValidators int) {
+func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) {
 	if ps.Height == height {
 		if ps.Prevotes == nil {
 			ps.Prevotes = cmn.NewBitArray(numValidators)
@@ -1031,9 +1059,9 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) {
 	ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex)
 }
 
-func (ps *PeerState) setHasVote(height int, round int, type_ byte, index int) {
-	logger := ps.logger.With("peerRound", ps.Round, "height", height, "round", round)
-	logger.Debug("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index)
+func (ps *PeerState) setHasVote(height int64, round int, type_ byte, index int) {
+	logger := ps.logger.With("peerH/R", cmn.Fmt("%d/%d", ps.Height, ps.Round), "H/R", cmn.Fmt("%d/%d", height, round))
+	logger.Debug("setHasVote", "type", type_, "index", index)
 
 	// NOTE: some may be nil BitArrays -> no side effects.
 	psVotes := ps.getVoteBitArray(height, round, type_)
@@ -1225,9 +1253,9 @@ func DecodeMessage(bz []byte) (msgType byte, msg ConsensusMessage, err error) {
 // NewRoundStepMessage is sent for every step taken in the ConsensusState.
 // For every height/round/step transition
 type NewRoundStepMessage struct {
-	Height                int
+	Height                int64
 	Round                 int
-	Step                  RoundStepType
+	Step                  cstypes.RoundStepType
 	SecondsSinceStartTime int
 	LastCommitRound       int
 }
@@ -1242,7 +1270,7 @@ func (m *NewRoundStepMessage) String() string {
 
 // CommitStepMessage is sent when a block is committed.
 type CommitStepMessage struct {
-	Height           int
+	Height           int64
 	BlockPartsHeader types.PartSetHeader
 	BlockParts       *cmn.BitArray
 }
@@ -1268,7 +1296,7 @@ func (m *ProposalMessage) String() string {
 
 // ProposalPOLMessage is sent when a previous proposal is re-proposed.
 type ProposalPOLMessage struct {
-	Height           int
+	Height           int64
 	ProposalPOLRound int
 	ProposalPOL      *cmn.BitArray
 }
@@ -1282,7 +1310,7 @@ func (m *ProposalPOLMessage) String() string {
 
 // BlockPartMessage is sent when gossipping a piece of the proposed block.
 type BlockPartMessage struct {
-	Height int
+	Height int64
 	Round  int
 	Part   *types.Part
 }
@@ -1308,7 +1336,7 @@ func (m *VoteMessage) String() string {
 
 // HasVoteMessage is sent to indicate that a particular vote has been received.
 type HasVoteMessage struct {
-	Height int
+	Height int64
 	Round  int
 	Type   byte
 	Index  int
@@ -1323,7 +1351,7 @@ func (m *HasVoteMessage) String() string {
 
 // VoteSetMaj23Message is sent to indicate that a given BlockID has seen +2/3 votes.
 type VoteSetMaj23Message struct {
-	Height  int
+	Height  int64
 	Round   int
 	Type    byte
 	BlockID types.BlockID
@@ -1338,7 +1366,7 @@ func (m *VoteSetMaj23Message) String() string {
 
 // VoteSetBitsMessage is sent to communicate the bit-array of votes seen for the BlockID.
 type VoteSetBitsMessage struct {
-	Height  int
+	Height  int64
 	Round   int
 	Type    byte
 	BlockID types.BlockID

consensus/reactor_test.go

@@ -1,17 +1,21 @@
 package consensus
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"runtime/pprof"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/tendermint/abci/example/dummy"
-	"github.com/tendermint/tmlibs/events"
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
+
+	"github.com/stretchr/testify/require"
 )
 
 func init() {
@@ -21,27 +25,30 @@ func init() {
 //----------------------------------------------
 // in-process testnets
 
-func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEventRespond bool) ([]*ConsensusReactor, []chan interface{}) {
+func startConsensusNet(t *testing.T, css []*ConsensusState, N int) ([]*ConsensusReactor, []chan interface{}, []*types.EventBus) {
 	reactors := make([]*ConsensusReactor, N)
 	eventChans := make([]chan interface{}, N)
+	eventBuses := make([]*types.EventBus, N)
 	logger := consensusLogger()
 	for i := 0; i < N; i++ {
+		/*thisLogger, err := tmflags.ParseLogLevel("consensus:info,*:error", logger, "info")
+		if err != nil {	t.Fatal(err)}*/
+		thisLogger := logger
+
 		reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states
-		reactors[i].SetLogger(logger.With("validator", i))
+		reactors[i].conS.SetLogger(thisLogger.With("validator", i))
+		reactors[i].SetLogger(thisLogger.With("validator", i))
 
-		eventSwitch := events.NewEventSwitch()
-		eventSwitch.SetLogger(logger.With("module", "events", "validator", i))
-		_, err := eventSwitch.Start()
-		if err != nil {
-			t.Fatalf("Failed to start switch: %v", err)
-		}
+		eventBuses[i] = types.NewEventBus()
+		eventBuses[i].SetLogger(thisLogger.With("module", "events", "validator", i))
+		err := eventBuses[i].Start()
+		require.NoError(t, err)
 
-		reactors[i].SetEventSwitch(eventSwitch)
-		if subscribeEventRespond {
-			eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock())
-		} else {
-			eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1)
-		}
+		reactors[i].SetEventBus(eventBuses[i])
+
+		eventChans[i] = make(chan interface{}, 1)
+		err = eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		require.NoError(t, err)
 	}
 	// make connected switches and start all reactors
 	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
@@ -52,25 +59,29 @@ func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEven
 	// now that everyone is connected,  start the state machines
 	// If we started the state machines before everyone was connected,
 	// we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors
+	// TODO: is this still true with new pubsub?
 	for i := 0; i < N; i++ {
 		s := reactors[i].conS.GetState()
-		reactors[i].SwitchToConsensus(s)
+		reactors[i].SwitchToConsensus(s, 0)
 	}
-	return reactors, eventChans
+	return reactors, eventChans, eventBuses
 }
 
-func stopConsensusNet(reactors []*ConsensusReactor) {
+func stopConsensusNet(reactors []*ConsensusReactor, eventBuses []*types.EventBus) {
 	for _, r := range reactors {
 		r.Switch.Stop()
 	}
+	for _, b := range eventBuses {
+		b.Stop()
+	}
 }
 
 // Ensure a testnet makes blocks
 func TestReactor(t *testing.T) {
 	N := 4
 	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
-	reactors, eventChans := startConsensusNet(t, css, N, false)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(reactors, eventBuses)
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
 		<-eventChans[j]
@@ -85,11 +96,14 @@ func TestReactorProposalHeartbeats(t *testing.T) {
 		func(c *cfg.Config) {
 			c.Consensus.CreateEmptyBlocks = false
 		})
-	reactors, eventChans := startConsensusNet(t, css, N, false)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(reactors, eventBuses)
 	heartbeatChans := make([]chan interface{}, N)
+	var err error
 	for i := 0; i < N; i++ {
-		heartbeatChans[i] = subscribeToEvent(css[i].evsw, "tester", types.EventStringProposalHeartbeat(), 1)
+		heartbeatChans[i] = make(chan interface{}, 1)
+		err = eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryProposalHeartbeat, heartbeatChans[i])
+		require.NoError(t, err)
 	}
 	// wait till everyone sends a proposal heartbeat
 	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
@@ -98,7 +112,9 @@ func TestReactorProposalHeartbeats(t *testing.T) {
 	}, css)
 
 	// send a tx
-	css[3].mempool.CheckTx([]byte{1, 2, 3}, nil)
+	if err := css[3].mempool.CheckTx([]byte{1, 2, 3}, nil); err != nil {
+		//t.Fatal(err)
+	}
 
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
@@ -113,8 +129,8 @@ func TestReactorProposalHeartbeats(t *testing.T) {
 func TestVotingPowerChange(t *testing.T) {
 	nVals := 4
 	css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentDummy)
-	reactors, eventChans := startConsensusNet(t, css, nVals, true)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, nVals)
+	defer stopConsensusNet(reactors, eventBuses)
 
 	// map of active validators
 	activeVals := make(map[string]struct{})
@@ -125,7 +141,6 @@ func TestVotingPowerChange(t *testing.T) {
 	// wait till everyone makes block 1
 	timeoutWaitGroup(t, nVals, func(wg *sync.WaitGroup, j int) {
 		<-eventChans[j]
-		eventChans[j] <- struct{}{}
 		wg.Done()
 	}, css)
 
@@ -174,8 +189,9 @@ func TestValidatorSetChanges(t *testing.T) {
 	nPeers := 7
 	nVals := 4
 	css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy)
-	reactors, eventChans := startConsensusNet(t, css, nPeers, true)
-	defer stopConsensusNet(reactors)
+
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, nPeers)
+	defer stopConsensusNet(reactors, eventBuses)
 
 	// map of active validators
 	activeVals := make(map[string]struct{})
@@ -186,7 +202,6 @@ func TestValidatorSetChanges(t *testing.T) {
 	// wait till everyone makes block 1
 	timeoutWaitGroup(t, nPeers, func(wg *sync.WaitGroup, j int) {
 		<-eventChans[j]
-		eventChans[j] <- struct{}{}
 		wg.Done()
 	}, css)
 
@@ -194,7 +209,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	t.Log("---------------------------- Testing adding one validator")
 
 	newValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
-	newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower))
+	newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), testMinPower)
 
 	// wait till everyone makes block 2
 	// ensure the commit includes all validators
@@ -214,7 +229,7 @@ func TestValidatorSetChanges(t *testing.T) {
 
 	// wait till everyone makes block 5
 	// it includes the commit for block 4, which should have the updated validator set
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 
 	//---------------------------------------------------------------------------
 	t.Log("---------------------------- Testing changing the voting power of one validator")
@@ -226,7 +241,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 
 	if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
 		t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].GetRoundState().LastValidators.TotalVotingPower())
@@ -236,17 +251,17 @@ func TestValidatorSetChanges(t *testing.T) {
 	t.Log("---------------------------- Testing adding two validators at once")
 
 	newValidatorPubKey2 := css[nVals+1].privValidator.GetPubKey()
-	newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower))
+	newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), testMinPower)
 
 	newValidatorPubKey3 := css[nVals+2].privValidator.GetPubKey()
-	newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower))
+	newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), testMinPower)
 
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 	activeVals[string(newValidatorPubKey2.Address())] = struct{}{}
 	activeVals[string(newValidatorPubKey3.Address())] = struct{}{}
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 
 	//---------------------------------------------------------------------------
 	t.Log("---------------------------- Testing removing two validators at once")
@@ -259,7 +274,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 	delete(activeVals, string(newValidatorPubKey2.Address()))
 	delete(activeVals, string(newValidatorPubKey3.Address()))
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
 }
 
 // Check we can make blocks with skip_timeout_commit=false
@@ -271,8 +286,8 @@ func TestReactorWithTimeoutCommit(t *testing.T) {
 		css[i].config.SkipTimeoutCommit = false
 	}
 
-	reactors, eventChans := startConsensusNet(t, css, N-1, false)
-	defer stopConsensusNet(reactors)
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N-1)
+	defer stopConsensusNet(reactors, eventBuses)
 
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N-1, func(wg *sync.WaitGroup, j int) {
@@ -283,19 +298,50 @@ func TestReactorWithTimeoutCommit(t *testing.T) {
 
 func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
 	timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) {
-		newBlockI := <-eventChans[j]
+		defer wg.Done()
+
+		newBlockI, ok := <-eventChans[j]
+		if !ok {
+			return
+		}
 		newBlock := newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block
-		t.Logf("[WARN] Got block height=%v validator=%v", newBlock.Height, j)
+		t.Logf("Got block height=%v validator=%v", newBlock.Height, j)
 		err := validateBlock(newBlock, activeVals)
 		if err != nil {
 			t.Fatal(err)
 		}
 		for _, tx := range txs {
-			css[j].mempool.CheckTx(tx, nil)
+			if err = css[j].mempool.CheckTx(tx, nil); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}, css)
+}
+
+func waitForBlockWithUpdatedValsAndValidateIt(t *testing.T, n int, updatedVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState) {
+	timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) {
+		defer wg.Done()
+
+		var newBlock *types.Block
+	LOOP:
+		for {
+			newBlockI, ok := <-eventChans[j]
+			if !ok {
+				return
+			}
+			newBlock = newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block
+			if newBlock.LastCommit.Size() == len(updatedVals) {
+				t.Logf("Block with new validators height=%v validator=%v", newBlock.Height, j)
+				break LOOP
+			} else {
+				t.Logf("Block with no new validators height=%v validator=%v. Skipping...", newBlock.Height, j)
+			}
 		}
 
-		eventChans[j] <- struct{}{}
-		wg.Done()
+		err := validateBlock(newBlock, updatedVals)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}, css)
 }
 
@@ -326,15 +372,20 @@ func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int), css []*
 		close(done)
 	}()
 
+	// we're running many nodes in-process, possibly in in a virtual machine,
+	// and spewing debug messages - making a block could take a while,
+	timeout := time.Second * 60
+
 	select {
 	case <-done:
-	case <-time.After(time.Second * 10):
+	case <-time.After(timeout):
 		for i, cs := range css {
-			fmt.Println("#################")
-			fmt.Println("Validator", i)
-			fmt.Println(cs.GetRoundState())
-			fmt.Println("")
+			t.Log("#################")
+			t.Log("Validator", i)
+			t.Log(cs.GetRoundState())
+			t.Log("")
 		}
+		pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
 		panic("Timed out waiting for all validators to commit a block")
 	}
 }

consensus/replay.go

@@ -4,15 +4,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"reflect"
-	"strconv"
-	"strings"
+	//"strconv"
+	//"strings"
 	"time"
 
 	abci "github.com/tendermint/abci/types"
-	wire "github.com/tendermint/go-wire"
-	auto "github.com/tendermint/tmlibs/autofile"
+	//auto "github.com/tendermint/tmlibs/autofile"
 	cmn "github.com/tendermint/tmlibs/common"
 	"github.com/tendermint/tmlibs/log"
 
@@ -22,6 +22,8 @@ import (
 	"github.com/tendermint/tendermint/version"
 )
 
+var crc32c = crc32.MakeTable(crc32.Castagnoli)
+
 // Functionality to replay blocks and messages on recovery from a crash.
 // There are two general failure scenarios: failure during consensus, and failure while applying the block.
 // The former is handled by the WAL, the latter by the proxyApp Handshake on restart,
@@ -35,18 +37,11 @@ import (
 // as if it were received in receiveRoutine
 // Lines that start with "#" are ignored.
 // NOTE: receiveRoutine should not be running
-func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan interface{}) error {
-	// Skip over empty and meta lines
-	if len(msgBytes) == 0 || msgBytes[0] == '#' {
+func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan interface{}) error {
+	// skip meta messages
+	if _, ok := msg.Msg.(EndHeightMessage); ok {
 		return nil
 	}
-	var err error
-	var msg TimedWALMessage
-	wire.ReadJSON(&msg, msgBytes, &err)
-	if err != nil {
-		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
-		return fmt.Errorf("Error reading json data: %v", err)
-	}
 
 	// for logging
 	switch m := msg.Msg.(type) {
@@ -95,8 +90,7 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte
 
 // replay only those messages since the last block.
 // timeoutRoutine should run concurrently to read off tickChan
-func (cs *ConsensusState) catchupReplay(csHeight int) error {
-
+func (cs *ConsensusState) catchupReplay(csHeight int64) error {
 	// set replayMode
 	cs.replayMode = true
 	defer func() { cs.replayMode = false }()
@@ -104,42 +98,47 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 	// Ensure that ENDHEIGHT for this height doesn't exist
 	// NOTE: This is just a sanity check. As far as we know things work fine without it,
 	// and Handshake could reuse ConsensusState if it weren't for this check (since we can crash after writing ENDHEIGHT).
-	gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight))
+	gr, found, err := cs.wal.SearchForEndHeight(csHeight)
+	if err != nil {
+		return err
+	}
 	if gr != nil {
-		gr.Close()
+		if err := gr.Close(); err != nil {
+			return err
+		}
 	}
 	if found {
-		return errors.New(cmn.Fmt("WAL should not contain #ENDHEIGHT %d.", csHeight))
+		return fmt.Errorf("WAL should not contain #ENDHEIGHT %d.", csHeight)
 	}
 
 	// Search for last height marker
-	gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1))
+	gr, found, err = cs.wal.SearchForEndHeight(csHeight - 1)
 	if err == io.EOF {
 		cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", csHeight-1)
 	} else if err != nil {
 		return err
-	} else {
-		defer gr.Close()
 	}
 	if !found {
 		return errors.New(cmn.Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1))
 	}
+	defer gr.Close() // nolint: errcheck
 
 	cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight)
 
+	var msg *TimedWALMessage
+	dec := WALDecoder{gr}
+
 	for {
-		line, err := gr.ReadLine()
-		if err != nil {
-			if err == io.EOF {
-				break
-			} else {
-				return err
-			}
+		msg, err = dec.Decode()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
 		}
 		// NOTE: since the priv key is set when the msgs are received
 		// it will attempt to eg double sign but we can just ignore it
 		// since the votes will be replayed and we'll get to the next step
-		if err := cs.readReplayMessage([]byte(line), nil); err != nil {
+		if err := cs.readReplayMessage(msg, nil); err != nil {
 			return err
 		}
 	}
@@ -151,7 +150,8 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 
 // Parses marker lines of the form:
 // #ENDHEIGHT: 12345
-func makeHeightSearchFunc(height int) auto.SearchFunc {
+/*
+func makeHeightSearchFunc(height int64) auto.SearchFunc {
 	return func(line string) (int, error) {
 		line = strings.TrimRight(line, "\n")
 		parts := strings.Split(line, " ")
@@ -170,7 +170,7 @@ func makeHeightSearchFunc(height int) auto.SearchFunc {
 			return -1, nil
 		}
 	}
-}
+}*/
 
 //----------------------------------------------
 // Recover from failure during block processing
@@ -205,7 +205,10 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 		return errors.New(cmn.Fmt("Error calling Info: %v", err))
 	}
 
-	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
+	blockHeight := int64(res.LastBlockHeight)
+	if blockHeight < 0 {
+		return fmt.Errorf("Got a negative last block height (%d) from the app", blockHeight)
+	}
 	appHash := res.LastBlockAppHash
 
 	h.logger.Info("ABCI Handshake", "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
@@ -227,7 +230,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 
 // Replay all blocks since appBlockHeight and ensure the result matches the current state.
 // Returns the final AppHash or an error
-func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
+func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int64, proxyApp proxy.AppConns) ([]byte, error) {
 
 	storeBlockHeight := h.store.Height()
 	stateBlockHeight := h.state.LastBlockHeight
@@ -236,7 +239,9 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain
 	if appBlockHeight == 0 {
 		validators := types.TM2PB.Validators(h.state.Validators)
-		proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators})
+		if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil {
+			return nil, err
+		}
 	}
 
 	// First handle edge cases and constraints on the storeBlockHeight
@@ -300,7 +305,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 	return nil, nil
 }
 
-func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) {
+func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int64, mutateState bool) ([]byte, error) {
 	// App is further behind than it should be, so we need to replay blocks.
 	// We replay all blocks from appBlockHeight+1.
 	//
@@ -336,14 +341,13 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store
 }
 
 // ApplyBlock on the proxyApp with the last block.
-func (h *Handshaker) replayBlock(height int, proxyApp proxy.AppConnConsensus) ([]byte, error) {
+func (h *Handshaker) replayBlock(height int64, proxyApp proxy.AppConnConsensus) ([]byte, error) {
 	mempool := types.MockMempool{}
 
-	var eventCache types.Fireable // nil
 	block := h.store.LoadBlock(height)
 	meta := h.store.LoadBlockMeta(height)
 
-	if err := h.state.ApplyBlock(eventCache, proxyApp, block, meta.BlockID.PartsHeader, mempool); err != nil {
+	if err := h.state.ApplyBlock(types.NopEventBus{}, proxyApp, block, meta.BlockID.PartsHeader, mempool); err != nil {
 		return nil, err
 	}
 
@@ -355,7 +359,6 @@ func (h *Handshaker) replayBlock(height int, proxyApp proxy.AppConnConsensus) ([
 func (h *Handshaker) checkAppHash(appHash []byte) error {
 	if !bytes.Equal(h.state.AppHash, appHash) {
 		panic(errors.New(cmn.Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)).Error())
-		return nil
 	}
 	return nil
 }
@@ -370,7 +373,10 @@ func newMockProxyApp(appHash []byte, abciResponses *sm.ABCIResponses) proxy.AppC
 		abciResponses: abciResponses,
 	})
 	cli, _ := clientCreator.NewABCIClient()
-	cli.Start()
+	err := cli.Start()
+	if err != nil {
+		panic(err)
+	}
 	return proxy.NewAppConnConsensus(cli)
 }
 
@@ -382,21 +388,17 @@ type mockProxyApp struct {
 	abciResponses *sm.ABCIResponses
 }
 
-func (mock *mockProxyApp) DeliverTx(tx []byte) abci.Result {
+func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx {
 	r := mock.abciResponses.DeliverTx[mock.txCount]
 	mock.txCount += 1
-	return abci.Result{
-		r.Code,
-		r.Data,
-		r.Log,
-	}
+	return *r
 }
 
-func (mock *mockProxyApp) EndBlock(height uint64) abci.ResponseEndBlock {
+func (mock *mockProxyApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
 	mock.txCount = 0
-	return mock.abciResponses.EndBlock
+	return *mock.abciResponses.EndBlock
 }
 
-func (mock *mockProxyApp) Commit() abci.Result {
-	return abci.NewResultOK(mock.appHash, "")
+func (mock *mockProxyApp) Commit() abci.ResponseCommit {
+	return abci.ResponseCommit{Code: abci.CodeTypeOK, Data: mock.appHash}
 }

consensus/replay_file.go

@@ -2,12 +2,15 @@ package consensus
 
 import (
 	"bufio"
-	"errors"
+	"context"
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/proxy"
@@ -17,6 +20,11 @@ import (
 	dbm "github.com/tendermint/tmlibs/db"
 )
 
+const (
+	// event bus subscriber
+	subscriber = "replay-file"
+)
+
 //--------------------------------------------------------
 // replay messages interactively or all at once
 
@@ -41,24 +49,39 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 	cs.startForReplay()
 
 	// ensure all new step events are regenerated as expected
-	newStepCh := subscribeToEvent(cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+	newStepCh := make(chan interface{}, 1)
+
+	ctx := context.Background()
+	err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, newStepCh)
+	if err != nil {
+		return errors.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)
+	}
+	defer cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep)
 
 	// just open the file for reading, no need to use wal
-	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
+	fp, err := os.OpenFile(file, os.O_RDONLY, 0600)
 	if err != nil {
 		return err
 	}
 
 	pb := newPlayback(file, fp, cs, cs.state.Copy())
-	defer pb.fp.Close()
+	defer pb.fp.Close() // nolint: errcheck
 
 	var nextN int // apply N msgs in a row
-	for pb.scanner.Scan() {
+	var msg *TimedWALMessage
+	for {
 		if nextN == 0 && console {
 			nextN = pb.replayConsoleLoop()
 		}
 
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {
 			return err
 		}
 
@@ -76,9 +99,9 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 type playback struct {
 	cs *ConsensusState
 
-	fp      *os.File
-	scanner *bufio.Scanner
-	count   int // how many lines/msgs into the file are we
+	fp    *os.File
+	dec   *WALDecoder
+	count int // how many lines/msgs into the file are we
 
 	// replays can be reset to beginning
 	fileName     string    // so we can close/reopen the file
@@ -91,33 +114,41 @@ func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm.
 		fp:           fp,
 		fileName:     fileName,
 		genesisState: genState,
-		scanner:      bufio.NewScanner(fp),
+		dec:          NewWALDecoder(fp),
 	}
 }
 
 // go back count steps by resetting the state and running (pb.count - count) steps
 func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
-
 	pb.cs.Stop()
 	pb.cs.Wait()
 
 	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool)
-	newCS.SetEventSwitch(pb.cs.evsw)
+	newCS.SetEventBus(pb.cs.eventBus)
 	newCS.startForReplay()
 
-	pb.fp.Close()
-	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0666)
+	if err := pb.fp.Close(); err != nil {
+		return err
+	}
+	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0600)
 	if err != nil {
 		return err
 	}
 	pb.fp = fp
-	pb.scanner = bufio.NewScanner(fp)
+	pb.dec = NewWALDecoder(fp)
 	count = pb.count - count
 	fmt.Printf("Reseting from %d to %d\n", pb.count, count)
 	pb.count = 0
 	pb.cs = newCS
-	for i := 0; pb.scanner.Scan() && i < count; i++ {
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+	var msg *TimedWALMessage
+	for i := 0; i < count; i++ {
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {
 			return err
 		}
 		pb.count += 1
@@ -180,10 +211,20 @@ func (pb *playback) replayConsoleLoop() int {
 			// NOTE: "back" is not supported in the state machine design,
 			// so we restart and replay up to
 
+			ctx := context.Background()
 			// ensure all new step events are regenerated as expected
-			newStepCh := subscribeToEvent(pb.cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+			newStepCh := make(chan interface{}, 1)
+
+			err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, newStepCh)
+			if err != nil {
+				cmn.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep))
+			}
+			defer pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep)
+
 			if len(tokens) == 1 {
-				pb.replayReset(1, newStepCh)
+				if err := pb.replayReset(1, newStepCh); err != nil {
+					pb.cs.Logger.Error("Replay reset error", "err", err)
+				}
 			} else {
 				i, err := strconv.Atoi(tokens[1])
 				if err != nil {
@@ -191,7 +232,9 @@ func (pb *playback) replayConsoleLoop() int {
 				} else if i > pb.count {
 					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
 				} else {
-					pb.replayReset(i, newStepCh)
+					if err := pb.replayReset(i, newStepCh); err != nil {
+						pb.cs.Logger.Error("Replay reset error", "err", err)
+					}
 				}
 			}
 
@@ -249,19 +292,18 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 	// Create proxyAppConn connection (consensus, mempool, query)
 	clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
 	proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore))
-	_, err = proxyApp.Start()
+	err = proxyApp.Start()
 	if err != nil {
 		cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err))
 	}
 
-	// Make event switch
-	eventSwitch := types.NewEventSwitch()
-	if _, err := eventSwitch.Start(); err != nil {
-		cmn.Exit(cmn.Fmt("Failed to start event switch: %v", err))
+	eventBus := types.NewEventBus()
+	if err := eventBus.Start(); err != nil {
+		cmn.Exit(cmn.Fmt("Failed to start event bus: %v", err))
 	}
 
 	consensusState := NewConsensusState(csConfig, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{})
 
-	consensusState.SetEventSwitch(eventSwitch)
+	consensusState.SetEventBus(eventBus)
 	return consensusState
 }

consensus/replay_test.go

@@ -2,20 +2,24 @@ package consensus
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
-	"strings"
+	"runtime"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/tendermint/abci/example/dummy"
 	abci "github.com/tendermint/abci/types"
 	crypto "github.com/tendermint/go-crypto"
 	wire "github.com/tendermint/go-wire"
+	auto "github.com/tendermint/tmlibs/autofile"
 	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
 
@@ -26,8 +30,10 @@ import (
 	"github.com/tendermint/tmlibs/log"
 )
 
+var consensusReplayConfig *cfg.Config
+
 func init() {
-	config = ResetConfig("consensus_replay_test")
+	consensusReplayConfig = ResetConfig("consensus_replay_test")
 }
 
 // These tests ensure we can always recover from failure at any part of the consensus process.
@@ -40,11 +46,10 @@ func init() {
 
 // NOTE: Files in this dir are generated by running the `build.sh` therein.
 // It's a simple way to generate wals for a single block, or multiple blocks, with random transactions,
-// and different part sizes. The output is not deterministic, and the stepChanges may need to be adjusted
-// after running it (eg. sometimes small_block2 will have 5 block parts, sometimes 6).
+// and different part sizes. The output is not deterministic.
 // It should only have to be re-run if there is some breaking change to the consensus data structures (eg. blocks, votes)
 // or to the behaviour of the app (eg. computes app hash differently)
-var data_dir = path.Join(cmn.GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
+var data_dir = path.Join(cmn.GoPath(), "src/github.com/tendermint/tendermint/consensus", "test_data")
 
 //------------------------------------------------------------------------------------------
 // WAL Tests
@@ -53,221 +58,218 @@ var data_dir = path.Join(cmn.GoPath, "src/github.com/tendermint/tendermint/conse
 // and which ones we need the wal for - then we'd also be able to only flush the
 // wal writer when we need to, instead of with every message.
 
-// the priv validator changes step at these lines for a block with 1 val and 1 part
-var baseStepChanges = []int{3, 6, 8}
+func startNewConsensusStateAndWaitForBlock(t *testing.T, lastBlockHeight int64, blockDB dbm.DB, stateDB dbm.DB) {
+	logger := log.TestingLogger()
+	state, _ := sm.GetState(stateDB, consensusReplayConfig.GenesisFile())
+	state.SetLogger(logger.With("module", "state"))
+	privValidator := loadPrivValidator(consensusReplayConfig)
+	cs := newConsensusStateWithConfigAndBlockStore(consensusReplayConfig, state, privValidator, dummy.NewDummyApplication(), blockDB)
+	cs.SetLogger(logger)
 
-// test recovery from each line in each testCase
-var testCases = []*testCase{
-	newTestCase("empty_block", baseStepChanges),   // empty block (has 1 block part)
-	newTestCase("small_block1", baseStepChanges),  // small block with txs in 1 block part
-	newTestCase("small_block2", []int{3, 11, 13}), // small block with txs across 6 smaller block parts
-}
+	bytes, _ := ioutil.ReadFile(cs.config.WalFile())
+	// fmt.Printf("====== WAL: \n\r%s\n", bytes)
+	t.Logf("====== WAL: \n\r%s\n", bytes)
 
-type testCase struct {
-	name    string
-	log     string       //full cs wal
-	stepMap map[int]int8 // map lines of log to privval step
+	err := cs.Start()
+	require.NoError(t, err)
+	defer func() {
+		cs.Stop()
+	}()
 
-	proposeLine   int
-	prevoteLine   int
-	precommitLine int
-}
-
-func newTestCase(name string, stepChanges []int) *testCase {
-	if len(stepChanges) != 3 {
-		panic(cmn.Fmt("a full wal has 3 step changes! Got array %v", stepChanges))
-	}
-	return &testCase{
-		name:    name,
-		log:     readWAL(path.Join(data_dir, name+".cswal")),
-		stepMap: newMapFromChanges(stepChanges),
-
-		proposeLine:   stepChanges[0],
-		prevoteLine:   stepChanges[1],
-		precommitLine: stepChanges[2],
-	}
-}
-
-func newMapFromChanges(changes []int) map[int]int8 {
-	changes = append(changes, changes[2]+1) // so we add the last step change to the map
-	m := make(map[int]int8)
-	var count int
-	for changeNum, nextChange := range changes {
-		for ; count < nextChange; count++ {
-			m[count] = int8(changeNum)
-		}
-	}
-	return m
-}
-
-func readWAL(p string) string {
-	b, err := ioutil.ReadFile(p)
-	if err != nil {
-		panic(err)
-	}
-	return string(b)
-}
-
-func writeWAL(walMsgs string) string {
-	tempDir := os.TempDir()
-	walDir := path.Join(tempDir, "/wal"+cmn.RandStr(12))
-	walFile := path.Join(walDir, "wal")
-	// Create WAL directory
-	err := cmn.EnsureDir(walDir, 0700)
-	if err != nil {
-		panic(err)
-	}
-	// Write the needed WAL to file
-	err = cmn.WriteFile(walFile, []byte(walMsgs), 0600)
-	if err != nil {
-		panic(err)
-	}
-	return walFile
-}
-
-func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
-	after := time.After(time.Second * 10)
+	// This is just a signal that we haven't halted; its not something contained
+	// in the WAL itself. Assuming the consensus state is running, replay of any
+	// WAL, including the empty one, should eventually be followed by a new
+	// block, or else something is wrong.
+	newBlockCh := make(chan interface{}, 1)
+	err = cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, newBlockCh)
+	require.NoError(t, err)
 	select {
 	case <-newBlockCh:
-	case <-after:
-		panic(cmn.Fmt("Timed out waiting for new block for case '%s' line %d", thisCase.name, i))
+	case <-time.After(10 * time.Second):
+		t.Fatalf("Timed out waiting for new block (see trace above)")
 	}
 }
 
-func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{},
-	thisCase *testCase, i int) {
-
-	cs.config.SetWalFile(walFile)
-	started, err := cs.Start()
-	if err != nil {
-		t.Fatalf("Cannot start consensus: %v", err)
-	}
-	if !started {
-		t.Error("Consensus did not start")
-	}
-	// Wait to make a new block.
-	// This is just a signal that we haven't halted; its not something contained in the WAL itself.
-	// Assuming the consensus state is running, replay of any WAL, including the empty one,
-	// should eventually be followed by a new block, or else something is wrong
-	waitForBlock(newBlockCh, thisCase, i)
-	cs.evsw.Stop()
-	cs.Stop()
-LOOP:
+func sendTxs(cs *ConsensusState, ctx context.Context) {
+	i := 0
 	for {
 		select {
-		case <-newBlockCh:
+		case <-ctx.Done():
+			return
 		default:
-			break LOOP
-		}
-	}
-	cs.Wait()
-}
-
-func toPV(pv types.PrivValidator) *types.PrivValidatorFS {
-	return pv.(*types.PrivValidatorFS)
-}
-
-func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) {
-	t.Log("-------------------------------------")
-	t.Logf("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter)
-
-	lineStep := nLines
-	if crashAfter {
-		lineStep -= 1
-	}
-
-	split := strings.Split(thisCase.log, "\n")
-	lastMsg := split[nLines]
-
-	// we write those lines up to (not including) one with the signature
-	walFile := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
-
-	cs := fixedConsensusStateDummy()
-
-	// set the last step according to when we crashed vs the wal
-	toPV(cs.privValidator).LastHeight = 1 // first block
-	toPV(cs.privValidator).LastStep = thisCase.stepMap[lineStep]
-
-	t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).LastStep)
-
-	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
-
-	return cs, newBlockCh, lastMsg, walFile
-}
-
-func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
-	var err error
-	var msg TimedWALMessage
-	wire.ReadJSON(&msg, []byte(walMsg), &err)
-	if err != nil {
-		t.Fatalf("Error reading json data: %v", err)
-	}
-	return msg
-}
-
-//-----------------------------------------------
-// Test the log at every iteration, and set the privVal last step
-// as if the log was written after signing, before the crash
-
-func TestWALCrashAfterWrite(t *testing.T) {
-	for _, thisCase := range testCases {
-		split := strings.Split(thisCase.log, "\n")
-		for i := 0; i < len(split)-1; i++ {
-			cs, newBlockCh, _, walFile := setupReplayTest(t, thisCase, i+1, true)
-			runReplayTest(t, cs, walFile, newBlockCh, thisCase, i+1)
+			cs.mempool.CheckTx([]byte{byte(i)}, nil)
+			i++
 		}
 	}
 }
 
-//-----------------------------------------------
-// Test the log as if we crashed after signing but before writing.
-// This relies on privValidator.LastSignature being set
+// TestWALCrash uses crashing WAL to test we can recover from any WAL failure.
+func TestWALCrash(t *testing.T) {
+	testCases := []struct {
+		name         string
+		initFn       func(*ConsensusState, context.Context)
+		heightToStop int64
+	}{
+		{"empty block",
+			func(cs *ConsensusState, ctx context.Context) {},
+			1},
+		{"block with a smaller part size",
+			func(cs *ConsensusState, ctx context.Context) {
+				// XXX: is there a better way to change BlockPartSizeBytes?
+				params := cs.state.Params
+				params.BlockPartSizeBytes = 512
+				cs.state.Params = params
+				sendTxs(cs, ctx)
+			},
+			1},
+		{"many non-empty blocks",
+			sendTxs,
+			3},
+	}
 
-func TestWALCrashBeforeWritePropose(t *testing.T) {
-	for _, thisCase := range testCases {
-		lineNum := thisCase.proposeLine
-		// setup replay test where last message is a proposal
-		cs, newBlockCh, proposalMsg, walFile := setupReplayTest(t, thisCase, lineNum, false)
-		msg := readTimedWALMessage(t, proposalMsg)
-		proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage)
-		// Set LastSig
-		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal)
-		toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature
-		runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			crashWALandCheckLiveness(t, tc.initFn, tc.heightToStop)
+		})
 	}
 }
 
-func TestWALCrashBeforeWritePrevote(t *testing.T) {
-	for _, thisCase := range testCases {
-		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal())
+func crashWALandCheckLiveness(t *testing.T, initFn func(*ConsensusState, context.Context), heightToStop int64) {
+	walPaniced := make(chan error)
+	crashingWal := &crashingWAL{panicCh: walPaniced, heightToStop: heightToStop}
+
+	i := 1
+LOOP:
+	for {
+		// fmt.Printf("====== LOOP %d\n", i)
+		t.Logf("====== LOOP %d\n", i)
+
+		// create consensus state from a clean slate
+		logger := log.NewNopLogger()
+		stateDB := dbm.NewMemDB()
+		state, _ := sm.MakeGenesisStateFromFile(stateDB, consensusReplayConfig.GenesisFile())
+		state.SetLogger(logger.With("module", "state"))
+		privValidator := loadPrivValidator(consensusReplayConfig)
+		blockDB := dbm.NewMemDB()
+		cs := newConsensusStateWithConfigAndBlockStore(consensusReplayConfig, state, privValidator, dummy.NewDummyApplication(), blockDB)
+		cs.SetLogger(logger)
+
+		// start sending transactions
+		ctx, cancel := context.WithCancel(context.Background())
+		go initFn(cs, ctx)
+
+		// clean up WAL file from the previous iteration
+		walFile := cs.config.WalFile()
+		os.Remove(walFile)
+
+		// set crashing WAL
+		csWal, err := cs.OpenWAL(walFile)
+		require.NoError(t, err)
+		crashingWal.next = csWal
+		// reset the message counter
+		crashingWal.msgIndex = 1
+		cs.wal = crashingWal
+
+		// start consensus state
+		err = cs.Start()
+		require.NoError(t, err)
+
+		i++
+
+		select {
+		case err := <-walPaniced:
+			t.Logf("WAL paniced: %v", err)
+
+			// make sure we can make blocks after a crash
+			startNewConsensusStateAndWaitForBlock(t, cs.Height, blockDB, stateDB)
+
+			// stop consensus state and transactions sender (initFn)
+			cs.Stop()
+			cancel()
+
+			// if we reached the required height, exit
+			if _, ok := err.(ReachedHeightToStopError); ok {
+				break LOOP
+			}
+		case <-time.After(10 * time.Second):
+			t.Fatal("WAL did not panic for 10 seconds (check the log)")
+		}
 	}
 }
 
-func TestWALCrashBeforeWritePrecommit(t *testing.T) {
-	for _, thisCase := range testCases {
-		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka())
+// crashingWAL is a WAL which crashes or rather simulates a crash during Save
+// (before and after). It remembers a message for which we last panicked
+// (lastPanicedForMsgIndex), so we don't panic for it in subsequent iterations.
+type crashingWAL struct {
+	next         WAL
+	panicCh      chan error
+	heightToStop int64
+
+	msgIndex               int // current message index
+	lastPanicedForMsgIndex int // last message for which we panicked
+}
+
+// WALWriteError indicates a WAL crash.
+type WALWriteError struct {
+	msg string
+}
+
+func (e WALWriteError) Error() string {
+	return e.msg
+}
+
+// ReachedHeightToStopError indicates we've reached the required consensus
+// height and may exit.
+type ReachedHeightToStopError struct {
+	height int64
+}
+
+func (e ReachedHeightToStopError) Error() string {
+	return fmt.Sprintf("reached height to stop %d", e.height)
+}
+
+// Save simulate WAL's crashing by sending an error to the panicCh and then
+// exiting the cs.receiveRoutine.
+func (w *crashingWAL) Save(m WALMessage) {
+	if endMsg, ok := m.(EndHeightMessage); ok {
+		if endMsg.Height == w.heightToStop {
+			w.panicCh <- ReachedHeightToStopError{endMsg.Height}
+			runtime.Goexit()
+		} else {
+			w.next.Save(m)
+		}
+		return
+	}
+
+	if w.msgIndex > w.lastPanicedForMsgIndex {
+		w.lastPanicedForMsgIndex = w.msgIndex
+		_, file, line, _ := runtime.Caller(1)
+		w.panicCh <- WALWriteError{fmt.Sprintf("failed to write %T to WAL (fileline: %s:%d)", m, file, line)}
+		runtime.Goexit()
+	} else {
+		w.msgIndex++
+		w.next.Save(m)
 	}
 }
 
-func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) {
-	// setup replay test where last message is a vote
-	cs, newBlockCh, voteMsg, walFile := setupReplayTest(t, thisCase, lineNum, false)
-	types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) {
-		msg := readTimedWALMessage(t, voteMsg)
-		vote := msg.Msg.(msgInfo).Msg.(*VoteMessage)
-		// Set LastSig
-		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote)
-		toPV(cs.privValidator).LastSignature = vote.Vote.Signature
-	})
-	runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
+func (w *crashingWAL) Group() *auto.Group { return w.next.Group() }
+func (w *crashingWAL) SearchForEndHeight(height int64) (gr *auto.GroupReader, found bool, err error) {
+	return w.next.SearchForEndHeight(height)
 }
 
+func (w *crashingWAL) Start() error { return w.next.Start() }
+func (w *crashingWAL) Stop() error  { return w.next.Stop() }
+func (w *crashingWAL) Wait()        { w.next.Wait() }
+
 //------------------------------------------------------------------------------------------
 // Handshake Tests
 
+const (
+	NUM_BLOCKS = 6
+)
+
 var (
-	NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal
-	mempool    = types.MockMempool{}
+	mempool = types.MockMempool{}
 )
 
 //---------------------------------------
@@ -306,16 +308,30 @@ func TestHandshakeReplayNone(t *testing.T) {
 	}
 }
 
+func tempWALWithData(data []byte) string {
+	walFile, err := ioutil.TempFile("", "wal")
+	if err != nil {
+		panic(fmt.Errorf("failed to create temp WAL file: %v", err))
+	}
+	_, err = walFile.Write(data)
+	if err != nil {
+		panic(fmt.Errorf("failed to write to temp WAL file: %v", err))
+	}
+	if err := walFile.Close(); err != nil {
+		panic(fmt.Errorf("failed to close temp WAL file: %v", err))
+	}
+	return walFile.Name()
+}
+
 // Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
 func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	config := ResetConfig("proxy_test_")
 
-	// copy the many_blocks file
-	walBody, err := cmn.ReadFile(path.Join(data_dir, "many_blocks.cswal"))
+	walBody, err := WALWithNBlocks(NUM_BLOCKS)
 	if err != nil {
 		t.Fatal(err)
 	}
-	walFile := writeWAL(string(walBody))
+	walFile := tempWALWithData(walBody)
 	config.Consensus.SetWalFile(walFile)
 
 	privVal := types.LoadPrivValidatorFS(config.PrivValidatorFile())
@@ -325,7 +341,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 		t.Fatal(err)
 	}
 	wal.SetLogger(log.TestingLogger())
-	if _, err := wal.Start(); err != nil {
+	if err := wal.Start(); err != nil {
 		t.Fatal(err)
 	}
 	chain, commits, err := makeBlockchainFromWAL(wal)
@@ -354,7 +370,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	// now start the app using the handshake - it should sync
 	handshaker := NewHandshaker(state, store)
 	proxyApp := proxy.NewAppConns(clientCreator2, handshaker)
-	if _, err := proxyApp.Start(); err != nil {
+	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
 
@@ -382,8 +398,8 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 }
 
 func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) {
-	testPartSize := st.Params().BlockPartSizeBytes
-	err := st.ApplyBlock(nil, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool)
+	testPartSize := st.Params.BlockPartSizeBytes
+	err := st.ApplyBlock(types.NopEventBus{}, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool)
 	if err != nil {
 		panic(err)
 	}
@@ -392,12 +408,14 @@ func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) {
 func buildAppStateFromChain(proxyApp proxy.AppConns,
 	state *sm.State, chain []*types.Block, nBlocks int, mode uint) {
 	// start a new app without handshake, play nBlocks blocks
-	if _, err := proxyApp.Start(); err != nil {
+	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
 
 	validators := types.TM2PB.Validators(state.Validators)
-	proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators})
+	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil {
+		panic(err)
+	}
 
 	defer proxyApp.Stop()
 	switch mode {
@@ -425,13 +443,15 @@ func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.B
 	// run the whole chain against this client to build up the tendermint state
 	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "1")))
 	proxyApp := proxy.NewAppConns(clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
-	if _, err := proxyApp.Start(); err != nil {
+	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
 	defer proxyApp.Stop()
 
 	validators := types.TM2PB.Validators(state.Validators)
-	proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators})
+	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil {
+		panic(err)
+	}
 
 	var latestAppHash []byte
 
@@ -463,36 +483,33 @@ func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.B
 //--------------------------
 // utils for making blocks
 
-func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
+func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 	// Search for height marker
-	gr, found, err := wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(0))
+	gr, found, err := wal.SearchForEndHeight(0)
 	if err != nil {
 		return nil, nil, err
 	}
 	if !found {
 		return nil, nil, errors.New(cmn.Fmt("WAL does not contain height %d.", 1))
 	}
-	defer gr.Close()
+	defer gr.Close() // nolint: errcheck
 
 	// log.Notice("Build a blockchain by reading from the WAL")
 
 	var blockParts *types.PartSet
 	var blocks []*types.Block
 	var commits []*types.Commit
-	for {
-		line, err := gr.ReadLine()
-		if err != nil {
-			if err == io.EOF {
-				break
-			} else {
-				return nil, nil, err
-			}
-		}
 
-		piece, err := readPieceFromWAL([]byte(line))
-		if err != nil {
+	dec := NewWALDecoder(gr)
+	for {
+		msg, err := dec.Decode()
+		if err == io.EOF {
+			break
+		} else if err != nil {
 			return nil, nil, err
 		}
+
+		piece := readPieceFromWAL(msg)
 		if piece == nil {
 			continue
 		}
@@ -528,17 +545,10 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
 	return blocks, commits, nil
 }
 
-func readPieceFromWAL(msgBytes []byte) (interface{}, error) {
-	// Skip over empty and meta lines
-	if len(msgBytes) == 0 || msgBytes[0] == '#' {
-		return nil, nil
-	}
-	var err error
-	var msg TimedWALMessage
-	wire.ReadJSON(&msg, msgBytes, &err)
-	if err != nil {
-		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
-		return nil, fmt.Errorf("Error reading json data: %v", err)
+func readPieceFromWAL(msg *TimedWALMessage) interface{} {
+	// skip meta messages
+	if _, ok := msg.Msg.(EndHeightMessage); ok {
+		return nil
 	}
 
 	// for logging
@@ -546,14 +556,15 @@ func readPieceFromWAL(msgBytes []byte) (interface{}, error) {
 	case msgInfo:
 		switch msg := m.Msg.(type) {
 		case *ProposalMessage:
-			return &msg.Proposal.BlockPartsHeader, nil
+			return &msg.Proposal.BlockPartsHeader
 		case *BlockPartMessage:
-			return msg.Part, nil
+			return msg.Part
 		case *VoteMessage:
-			return msg.Vote, nil
+			return msg.Vote
 		}
 	}
-	return nil, nil
+
+	return nil
 }
 
 // fresh state and mock store
@@ -562,7 +573,7 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBl
 	state, _ := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
 	state.SetLogger(log.TestingLogger().With("module", "state"))
 
-	store := NewMockBlockStore(config, state.Params())
+	store := NewMockBlockStore(config, state.Params)
 	return state, store
 }
 
@@ -581,21 +592,21 @@ func NewMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBl
 	return &mockBlockStore{config, params, nil, nil}
 }
 
-func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
-func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
-func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
+func (bs *mockBlockStore) Height() int64                       { return int64(len(bs.chain)) }
+func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain[height-1] }
+func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
 	block := bs.chain[height-1]
 	return &types.BlockMeta{
 		BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.params.BlockPartSizeBytes).Header()},
 		Header:  block.Header,
 	}
 }
-func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
+func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil }
 func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
 }
-func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
+func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit {
 	return bs.commits[height-1]
 }
-func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
+func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit {
 	return bs.commits[height-1]
 }

consensus/state.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"path/filepath"
 	"reflect"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -16,6 +16,7 @@ import (
 	"github.com/tendermint/tmlibs/log"
 
 	cfg "github.com/tendermint/tendermint/config"
+	cstypes "github.com/tendermint/tendermint/consensus/types"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -38,124 +39,6 @@ var (
 	ErrVoteHeightMismatch       = errors.New("Error vote height mismatch")
 )
 
-//-----------------------------------------------------------------------------
-// RoundStepType enum type
-
-// RoundStepType enumerates the state of the consensus state machine
-type RoundStepType uint8 // These must be numeric, ordered.
-
-const (
-	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
-	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
-	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
-	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
-	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
-	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
-	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
-	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
-	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
-)
-
-// String returns a string
-func (rs RoundStepType) String() string {
-	switch rs {
-	case RoundStepNewHeight:
-		return "RoundStepNewHeight"
-	case RoundStepNewRound:
-		return "RoundStepNewRound"
-	case RoundStepPropose:
-		return "RoundStepPropose"
-	case RoundStepPrevote:
-		return "RoundStepPrevote"
-	case RoundStepPrevoteWait:
-		return "RoundStepPrevoteWait"
-	case RoundStepPrecommit:
-		return "RoundStepPrecommit"
-	case RoundStepPrecommitWait:
-		return "RoundStepPrecommitWait"
-	case RoundStepCommit:
-		return "RoundStepCommit"
-	default:
-		return "RoundStepUnknown" // Cannot panic.
-	}
-}
-
-//-----------------------------------------------------------------------------
-
-// RoundState defines the internal consensus state.
-// It is Immutable when returned from ConsensusState.GetRoundState()
-// TODO: Actually, only the top pointer is copied,
-// so access to field pointers is still racey
-type RoundState struct {
-	Height             int // Height we are working on
-	Round              int
-	Step               RoundStepType
-	StartTime          time.Time
-	CommitTime         time.Time // Subjective time when +2/3 precommits for Block at Round were found
-	Validators         *types.ValidatorSet
-	Proposal           *types.Proposal
-	ProposalBlock      *types.Block
-	ProposalBlockParts *types.PartSet
-	LockedRound        int
-	LockedBlock        *types.Block
-	LockedBlockParts   *types.PartSet
-	Votes              *HeightVoteSet
-	CommitRound        int            //
-	LastCommit         *types.VoteSet // Last precommits at Height-1
-	LastValidators     *types.ValidatorSet
-}
-
-// RoundStateEvent returns the H/R/S of the RoundState as an event.
-func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
-	edrs := types.EventDataRoundState{
-		Height:     rs.Height,
-		Round:      rs.Round,
-		Step:       rs.Step.String(),
-		RoundState: rs,
-	}
-	return edrs
-}
-
-// String returns a string
-func (rs *RoundState) String() string {
-	return rs.StringIndented("")
-}
-
-// StringIndented returns a string
-func (rs *RoundState) StringIndented(indent string) string {
-	return fmt.Sprintf(`RoundState{
-%s  H:%v R:%v S:%v
-%s  StartTime:     %v
-%s  CommitTime:    %v
-%s  Validators:    %v
-%s  Proposal:      %v
-%s  ProposalBlock: %v %v
-%s  LockedRound:   %v
-%s  LockedBlock:   %v %v
-%s  Votes:         %v
-%s  LastCommit: %v
-%s  LastValidators:    %v
-%s}`,
-		indent, rs.Height, rs.Round, rs.Step,
-		indent, rs.StartTime,
-		indent, rs.CommitTime,
-		indent, rs.Validators.StringIndented(indent+"    "),
-		indent, rs.Proposal,
-		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
-		indent, rs.LockedRound,
-		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
-		indent, rs.Votes.StringIndented(indent+"    "),
-		indent, rs.LastCommit.StringShort(),
-		indent, rs.LastValidators.StringIndented(indent+"    "),
-		indent)
-}
-
-// StringShort returns a string
-func (rs *RoundState) StringShort() string {
-	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
-		rs.Height, rs.Round, rs.Step, rs.StartTime)
-}
-
 //-----------------------------------------------------------------------------
 
 var (
@@ -170,10 +53,10 @@ type msgInfo struct {
 
 // internally generated messages which may update the state
 type timeoutInfo struct {
-	Duration time.Duration `json:"duration"`
-	Height   int           `json:"height"`
-	Round    int           `json:"round"`
-	Step     RoundStepType `json:"step"`
+	Duration time.Duration         `json:"duration"`
+	Height   int64                 `json:"height"`
+	Round    int                   `json:"round"`
+	Step     cstypes.RoundStepType `json:"step"`
 }
 
 func (ti *timeoutInfo) String() string {
@@ -198,7 +81,7 @@ type ConsensusState struct {
 
 	// internal state
 	mtx sync.Mutex
-	RoundState
+	cstypes.RoundState
 	state *sm.State // State until height-1.
 
 	// state changes may be triggered by msgs from peers,
@@ -207,21 +90,22 @@ type ConsensusState struct {
 	internalMsgQueue chan msgInfo
 	timeoutTicker    TimeoutTicker
 
-	// we use PubSub to trigger msg broadcasts in the reactor,
+	// we use eventBus to trigger msg broadcasts in the reactor,
 	// and to notify external subscribers, eg. through a websocket
-	evsw types.EventSwitch
+	eventBus *types.EventBus
 
 	// a Write-Ahead Log ensures we can recover from any kind of crash
 	// and helps us avoid signing conflicting votes
-	wal        *WAL
-	replayMode bool // so we don't log signing errors during replay
+	wal          WAL
+	replayMode   bool // so we don't log signing errors during replay
+	doWALCatchup bool // determines if we even try to do the catchup
 
 	// for tests where we want to limit the number of transitions the state makes
 	nSteps int
 
 	// some functions can be overwritten for testing
-	decideProposal func(height, round int)
-	doPrevote      func(height, round int)
+	decideProposal func(height int64, round int)
+	doPrevote      func(height int64, round int)
 	setProposal    func(proposal *types.Proposal) error
 
 	// closed when we finish shutting down
@@ -239,6 +123,8 @@ func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppCon
 		internalMsgQueue: make(chan msgInfo, msgQueueSize),
 		timeoutTicker:    NewTimeoutTicker(),
 		done:             make(chan struct{}),
+		doWALCatchup:     true,
+		wal:              nilWAL{},
 	}
 	// set function defaults (may be overwritten before calling Start)
 	cs.decideProposal = cs.defaultDecideProposal
@@ -262,9 +148,9 @@ func (cs *ConsensusState) SetLogger(l log.Logger) {
 	cs.timeoutTicker.SetLogger(l)
 }
 
-// SetEventSwitch implements events.Eventable
-func (cs *ConsensusState) SetEventSwitch(evsw types.EventSwitch) {
-	cs.evsw = evsw
+// SetEventBus sets event bus.
+func (cs *ConsensusState) SetEventBus(b *types.EventBus) {
+	cs.eventBus = b
 }
 
 // String returns a string.
@@ -281,19 +167,19 @@ func (cs *ConsensusState) GetState() *sm.State {
 }
 
 // GetRoundState returns a copy of the internal consensus state.
-func (cs *ConsensusState) GetRoundState() *RoundState {
+func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	return cs.getRoundState()
 }
 
-func (cs *ConsensusState) getRoundState() *RoundState {
+func (cs *ConsensusState) getRoundState() *cstypes.RoundState {
 	rs := cs.RoundState // copy
 	return &rs
 }
 
 // GetValidators returns a copy of the current validators.
-func (cs *ConsensusState) GetValidators() (int, []*types.Validator) {
+func (cs *ConsensusState) GetValidators() (int64, []*types.Validator) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators
@@ -314,7 +200,7 @@ func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) {
 }
 
 // LoadCommit loads the commit for a given height.
-func (cs *ConsensusState) LoadCommit(height int) *types.Commit {
+func (cs *ConsensusState) LoadCommit(height int64) *types.Commit {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	if height == cs.blockStore.Height() {
@@ -326,26 +212,36 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit {
 // OnStart implements cmn.Service.
 // It loads the latest state via the WAL, and starts the timeout and receive routines.
 func (cs *ConsensusState) OnStart() error {
-
-	walFile := cs.config.WalFile()
-	if err := cs.OpenWAL(walFile); err != nil {
-		cs.Logger.Error("Error loading ConsensusState wal", "err", err.Error())
-		return err
+	// we may set the WAL in testing before calling Start,
+	// so only OpenWAL if its still the nilWAL
+	if _, ok := cs.wal.(nilWAL); ok {
+		walFile := cs.config.WalFile()
+		wal, err := cs.OpenWAL(walFile)
+		if err != nil {
+			cs.Logger.Error("Error loading ConsensusState wal", "err", err.Error())
+			return err
+		}
+		cs.wal = wal
 	}
 
 	// we need the timeoutRoutine for replay so
-	//  we don't block on the tick chan.
+	// we don't block on the tick chan.
 	// NOTE: we will get a build up of garbage go routines
-	//  firing on the tockChan until the receiveRoutine is started
-	//  to deal with them (by that point, at most one will be valid)
-	cs.timeoutTicker.Start()
+	// firing on the tockChan until the receiveRoutine is started
+	// to deal with them (by that point, at most one will be valid)
+	err := cs.timeoutTicker.Start()
+	if err != nil {
+		return err
+	}
 
 	// we may have lost some votes if the process crashed
 	// reload from consensus log to catchup
-	if err := cs.catchupReplay(cs.Height); err != nil {
-		cs.Logger.Error("Error on catchup replay. Proceeding to start ConsensusState anyway", "err", err.Error())
-		// NOTE: if we ever do return an error here,
-		// make sure to stop the timeoutTicker
+	if cs.doWALCatchup {
+		if err := cs.catchupReplay(cs.Height); err != nil {
+			cs.Logger.Error("Error on catchup replay. Proceeding to start ConsensusState anyway", "err", err.Error())
+			// NOTE: if we ever do return an error here,
+			// make sure to stop the timeoutTicker
+		}
 	}
 
 	// now start the receiveRoutine
@@ -361,7 +257,11 @@ func (cs *ConsensusState) OnStart() error {
 // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
 // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
 func (cs *ConsensusState) startRoutines(maxSteps int) {
-	cs.timeoutTicker.Start()
+	err := cs.timeoutTicker.Start()
+	if err != nil {
+		cs.Logger.Error("Error starting timeout ticker", "err", err)
+		return
+	}
 	go cs.receiveRoutine(maxSteps)
 }
 
@@ -372,7 +272,7 @@ func (cs *ConsensusState) OnStop() {
 	cs.timeoutTicker.Stop()
 
 	// Make BaseService.Wait() wait until cs.wal.Wait()
-	if cs.wal != nil && cs.IsRunning() {
+	if cs.IsRunning() {
 		cs.wal.Wait()
 	}
 }
@@ -385,25 +285,17 @@ func (cs *ConsensusState) Wait() {
 }
 
 // OpenWAL opens a file to log all consensus messages and timeouts for deterministic accountability
-func (cs *ConsensusState) OpenWAL(walFile string) (err error) {
-	err = cmn.EnsureDir(filepath.Dir(walFile), 0700)
-	if err != nil {
-		cs.Logger.Error("Error ensuring ConsensusState wal dir", "err", err.Error())
-		return err
-	}
-
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+func (cs *ConsensusState) OpenWAL(walFile string) (WAL, error) {
 	wal, err := NewWAL(walFile, cs.config.WalLight)
 	if err != nil {
-		return err
+		cs.Logger.Error("Failed to open WAL for consensus state", "wal", walFile, "err", err)
+		return nil, err
 	}
 	wal.SetLogger(cs.Logger.With("wal", walFile))
-	if _, err := wal.Start(); err != nil {
-		return err
+	if err := wal.Start(); err != nil {
+		return nil, err
 	}
-	cs.wal = wal
-	return nil
+	return wal, nil
 }
 
 //------------------------------------------------------------
@@ -439,7 +331,7 @@ func (cs *ConsensusState) SetProposal(proposal *types.Proposal, peerKey string)
 }
 
 // AddProposalBlockPart inputs a part of the proposal block.
-func (cs *ConsensusState) AddProposalBlockPart(height, round int, part *types.Part, peerKey string) error {
+func (cs *ConsensusState) AddProposalBlockPart(height int64, round int, part *types.Part, peerKey string) error {
 
 	if peerKey == "" {
 		cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""}
@@ -453,35 +345,39 @@ func (cs *ConsensusState) AddProposalBlockPart(height, round int, part *types.Pa
 
 // SetProposalAndBlock inputs the proposal and all block parts.
 func (cs *ConsensusState) SetProposalAndBlock(proposal *types.Proposal, block *types.Block, parts *types.PartSet, peerKey string) error {
-	cs.SetProposal(proposal, peerKey)
+	if err := cs.SetProposal(proposal, peerKey); err != nil {
+		return err
+	}
 	for i := 0; i < parts.Total(); i++ {
 		part := parts.GetPart(i)
-		cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerKey)
+		if err := cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerKey); err != nil {
+			return err
+		}
 	}
-	return nil // TODO errors
+	return nil
 }
 
 //------------------------------------------------------------
 // internal functions for managing the state
 
-func (cs *ConsensusState) updateHeight(height int) {
+func (cs *ConsensusState) updateHeight(height int64) {
 	cs.Height = height
 }
 
-func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) {
+func (cs *ConsensusState) updateRoundStep(round int, step cstypes.RoundStepType) {
 	cs.Round = round
 	cs.Step = step
 }
 
 // enterNewRound(height, 0) at cs.StartTime.
-func (cs *ConsensusState) scheduleRound0(rs *RoundState) {
+func (cs *ConsensusState) scheduleRound0(rs *cstypes.RoundState) {
 	//cs.Logger.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
-	sleepDuration := rs.StartTime.Sub(time.Now())
-	cs.scheduleTimeout(sleepDuration, rs.Height, 0, RoundStepNewHeight)
+	sleepDuration := rs.StartTime.Sub(time.Now()) // nolint: gotype, gosimple
+	cs.scheduleTimeout(sleepDuration, rs.Height, 0, cstypes.RoundStepNewHeight)
 }
 
 // Attempt to schedule a timeout (by sending timeoutInfo on the tickChan)
-func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height, round int, step RoundStepType) {
+func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height int64, round int, step cstypes.RoundStepType) {
 	cs.timeoutTicker.ScheduleTimeout(timeoutInfo{duration, height, round, step})
 }
 
@@ -506,7 +402,7 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
 		return
 	}
 	seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight)
-	lastPrecommits := types.NewVoteSet(cs.state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators)
+	lastPrecommits := types.NewVoteSet(state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators)
 	for _, precommit := range seenCommit.Precommits {
 		if precommit == nil {
 			continue
@@ -523,7 +419,7 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
 }
 
 // Updates ConsensusState and increments height to match that of state.
-// The round becomes 0 and cs.Step becomes RoundStepNewHeight.
+// The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight.
 func (cs *ConsensusState) updateToState(state *sm.State) {
 	if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
 		cmn.PanicSanity(cmn.Fmt("updateToState() expected state height of %v but found %v",
@@ -559,7 +455,7 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 
 	// RoundState fields
 	cs.updateHeight(height)
-	cs.updateRoundStep(0, RoundStepNewHeight)
+	cs.updateRoundStep(0, cstypes.RoundStepNewHeight)
 	if cs.CommitTime.IsZero() {
 		// "Now" makes it easier to sync up dev nodes.
 		// We add timeoutCommit to allow transactions
@@ -577,7 +473,7 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 	cs.LockedRound = 0
 	cs.LockedBlock = nil
 	cs.LockedBlockParts = nil
-	cs.Votes = NewHeightVoteSet(state.ChainID, height, validators)
+	cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators)
 	cs.CommitRound = -1
 	cs.LastCommit = lastPrecommits
 	cs.LastValidators = state.LastValidators
@@ -592,9 +488,9 @@ func (cs *ConsensusState) newStep() {
 	rs := cs.RoundStateEvent()
 	cs.wal.Save(rs)
 	cs.nSteps += 1
-	// newStep is called by updateToStep in NewConsensusState before the evsw is set!
-	if cs.evsw != nil {
-		types.FireEventNewRoundStep(cs.evsw, rs)
+	// newStep is called by updateToStep in NewConsensusState before the eventBus is set!
+	if cs.eventBus != nil {
+		cs.eventBus.PublishEventNewRoundStep(rs)
 	}
 }
 
@@ -609,7 +505,7 @@ func (cs *ConsensusState) newStep() {
 func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 	defer func() {
 		if r := recover(); r != nil {
-			cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r)
+			cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack()))
 		}
 	}()
 
@@ -648,9 +544,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 			// priv_val tracks LastSig
 
 			// close wal now that we're done writing to it
-			if cs.wal != nil {
-				cs.wal.Stop()
-			}
+			cs.wal.Stop()
 
 			close(cs.done)
 			return
@@ -698,7 +592,7 @@ func (cs *ConsensusState) handleMsg(mi msgInfo) {
 	}
 }
 
-func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
+func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) {
 	cs.Logger.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
 
 	// timeouts must be for current height, round, step
@@ -712,20 +606,20 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
 	defer cs.mtx.Unlock()
 
 	switch ti.Step {
-	case RoundStepNewHeight:
+	case cstypes.RoundStepNewHeight:
 		// NewRound event fired from enterNewRound.
 		// XXX: should we fire timeout here (for timeout commit)?
 		cs.enterNewRound(ti.Height, 0)
-	case RoundStepNewRound:
+	case cstypes.RoundStepNewRound:
 		cs.enterPropose(ti.Height, 0)
-	case RoundStepPropose:
-		types.FireEventTimeoutPropose(cs.evsw, cs.RoundStateEvent())
+	case cstypes.RoundStepPropose:
+		cs.eventBus.PublishEventTimeoutPropose(cs.RoundStateEvent())
 		cs.enterPrevote(ti.Height, ti.Round)
-	case RoundStepPrevoteWait:
-		types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
+	case cstypes.RoundStepPrevoteWait:
+		cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent())
 		cs.enterPrecommit(ti.Height, ti.Round)
-	case RoundStepPrecommitWait:
-		types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
+	case cstypes.RoundStepPrecommitWait:
+		cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent())
 		cs.enterNewRound(ti.Height, ti.Round+1)
 	default:
 		panic(cmn.Fmt("Invalid timeout step: %v", ti.Step))
@@ -733,7 +627,7 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
 
 }
 
-func (cs *ConsensusState) handleTxsAvailable(height int) {
+func (cs *ConsensusState) handleTxsAvailable(height int64) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	// we only need to do this for round 0
@@ -750,8 +644,8 @@ func (cs *ConsensusState) handleTxsAvailable(height int) {
 // Enter: +2/3 precommits for nil at (height,round-1)
 // Enter: +2/3 prevotes any or +2/3 precommits for block or any from (height, round)
 // NOTE: cs.StartTime was already set for height.
-func (cs *ConsensusState) enterNewRound(height int, round int) {
-	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != RoundStepNewHeight) {
+func (cs *ConsensusState) enterNewRound(height int64, round int) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != cstypes.RoundStepNewHeight) {
 		cs.Logger.Debug(cmn.Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -772,7 +666,7 @@ func (cs *ConsensusState) enterNewRound(height int, round int) {
 	// Setup new round
 	// we don't fire newStep for this step,
 	// but we fire an event, so update the round step first
-	cs.updateRoundStep(round, RoundStepNewRound)
+	cs.updateRoundStep(round, cstypes.RoundStepNewRound)
 	cs.Validators = validators
 	if round == 0 {
 		// We've already reset these upon new height,
@@ -785,7 +679,7 @@ func (cs *ConsensusState) enterNewRound(height int, round int) {
 	}
 	cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
 
-	types.FireEventNewRound(cs.evsw, cs.RoundStateEvent())
+	cs.eventBus.PublishEventNewRound(cs.RoundStateEvent())
 
 	// Wait for txs to be available in the mempool
 	// before we enterPropose in round 0. If the last block changed the app hash,
@@ -793,7 +687,7 @@ func (cs *ConsensusState) enterNewRound(height int, round int) {
 	waitForTxs := cs.config.WaitForTxs() && round == 0 && !cs.needProofBlock(height)
 	if waitForTxs {
 		if cs.config.CreateEmptyBlocksInterval > 0 {
-			cs.scheduleTimeout(cs.config.EmptyBlocksInterval(), height, round, RoundStepNewRound)
+			cs.scheduleTimeout(cs.config.EmptyBlocksInterval(), height, round, cstypes.RoundStepNewRound)
 		}
 		go cs.proposalHeartbeat(height, round)
 	} else {
@@ -803,19 +697,16 @@ func (cs *ConsensusState) enterNewRound(height int, round int) {
 
 // needProofBlock returns true on the first height (so the genesis app hash is signed right away)
 // and where the last block (height-1) caused the app hash to change
-func (cs *ConsensusState) needProofBlock(height int) bool {
+func (cs *ConsensusState) needProofBlock(height int64) bool {
 	if height == 1 {
 		return true
 	}
 
 	lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1)
-	if !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash) {
-		return true
-	}
-	return false
+	return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash)
 }
 
-func (cs *ConsensusState) proposalHeartbeat(height, round int) {
+func (cs *ConsensusState) proposalHeartbeat(height int64, round int) {
 	counter := 0
 	addr := cs.privValidator.GetAddress()
 	valIndex, v := cs.Validators.GetByAddress(addr)
@@ -823,10 +714,11 @@ func (cs *ConsensusState) proposalHeartbeat(height, round int) {
 		// not a validator
 		valIndex = -1
 	}
+	chainID := cs.state.ChainID
 	for {
 		rs := cs.GetRoundState()
 		// if we've already moved on, no need to send more heartbeats
-		if rs.Step > RoundStepNewRound || rs.Round > round || rs.Height > height {
+		if rs.Step > cstypes.RoundStepNewRound || rs.Round > round || rs.Height > height {
 			return
 		}
 		heartbeat := &types.Heartbeat{
@@ -836,9 +728,8 @@ func (cs *ConsensusState) proposalHeartbeat(height, round int) {
 			ValidatorAddress: addr,
 			ValidatorIndex:   valIndex,
 		}
-		cs.privValidator.SignHeartbeat(cs.state.ChainID, heartbeat)
-		heartbeatEvent := types.EventDataProposalHeartbeat{heartbeat}
-		types.FireEventProposalHeartbeat(cs.evsw, heartbeatEvent)
+		cs.privValidator.SignHeartbeat(chainID, heartbeat)
+		cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat})
 		counter += 1
 		time.Sleep(proposalHeartbeatIntervalSeconds * time.Second)
 	}
@@ -847,8 +738,8 @@ func (cs *ConsensusState) proposalHeartbeat(height, round int) {
 // Enter (CreateEmptyBlocks): from enterNewRound(height,round)
 // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ): after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval
 // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool
-func (cs *ConsensusState) enterPropose(height int, round int) {
-	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
+func (cs *ConsensusState) enterPropose(height int64, round int) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPropose <= cs.Step) {
 		cs.Logger.Debug(cmn.Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -856,7 +747,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) {
 
 	defer func() {
 		// Done enterPropose:
-		cs.updateRoundStep(round, RoundStepPropose)
+		cs.updateRoundStep(round, cstypes.RoundStepPropose)
 		cs.newStep()
 
 		// If we have the whole proposal + POL, then goto Prevote now.
@@ -868,7 +759,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) {
 	}()
 
 	// If we don't get the proposal and all block parts quick enough, enterPrevote
-	cs.scheduleTimeout(cs.config.Propose(round), height, round, RoundStepPropose)
+	cs.scheduleTimeout(cs.config.Propose(round), height, round, cstypes.RoundStepPropose)
 
 	// Nothing more to do if we're not a validator
 	if cs.privValidator == nil {
@@ -894,7 +785,7 @@ func (cs *ConsensusState) isProposer() bool {
 	return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress())
 }
 
-func (cs *ConsensusState) defaultDecideProposal(height, round int) {
+func (cs *ConsensusState) defaultDecideProposal(height int64, round int) {
 	var block *types.Block
 	var blockParts *types.PartSet
 
@@ -913,8 +804,7 @@ func (cs *ConsensusState) defaultDecideProposal(height, round int) {
 	// Make proposal
 	polRound, polBlockID := cs.Votes.POLInfo()
 	proposal := types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID)
-	err := cs.privValidator.SignProposal(cs.state.ChainID, proposal)
-	if err == nil {
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal); err == nil {
 		// Set fields
 		/*  fields set by setProposal and addBlockPart
 		cs.Proposal = proposal
@@ -973,10 +863,9 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 
 	// Mempool validated transactions
 	txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs)
-
 	return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit,
 		cs.state.LastBlockID, cs.state.Validators.Hash(),
-		cs.state.AppHash, cs.state.Params().BlockPartSizeBytes)
+		cs.state.AppHash, cs.state.Params.BlockPartSizeBytes)
 }
 
 // Enter: `timeoutPropose` after entering Propose.
@@ -984,21 +873,21 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 // Enter: any +2/3 prevotes for future round.
 // Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
 // Otherwise vote nil.
-func (cs *ConsensusState) enterPrevote(height int, round int) {
-	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
+func (cs *ConsensusState) enterPrevote(height int64, round int) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevote <= cs.Step) {
 		cs.Logger.Debug(cmn.Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 
 	defer func() {
 		// Done enterPrevote:
-		cs.updateRoundStep(round, RoundStepPrevote)
+		cs.updateRoundStep(round, cstypes.RoundStepPrevote)
 		cs.newStep()
 	}()
 
 	// fire event for how we got here
 	if cs.isProposalComplete() {
-		types.FireEventCompleteProposal(cs.evsw, cs.RoundStateEvent())
+		cs.eventBus.PublishEventCompleteProposal(cs.RoundStateEvent())
 	} else {
 		// we received +2/3 prevotes for a future round
 		// TODO: catchup event?
@@ -1013,7 +902,7 @@ func (cs *ConsensusState) enterPrevote(height int, round int) {
 	// (so we have more time to try and collect +2/3 prevotes for a single block)
 }
 
-func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
+func (cs *ConsensusState) defaultDoPrevote(height int64, round int) {
 	logger := cs.Logger.With("height", height, "round", round)
 	// If a block is locked, prevote that.
 	if cs.LockedBlock != nil {
@@ -1046,8 +935,8 @@ func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
 }
 
 // Enter: any +2/3 prevotes at next round.
-func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
-	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
+func (cs *ConsensusState) enterPrevoteWait(height int64, round int) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevoteWait <= cs.Step) {
 		cs.Logger.Debug(cmn.Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -1058,12 +947,12 @@ func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
 
 	defer func() {
 		// Done enterPrevoteWait:
-		cs.updateRoundStep(round, RoundStepPrevoteWait)
+		cs.updateRoundStep(round, cstypes.RoundStepPrevoteWait)
 		cs.newStep()
 	}()
 
 	// Wait for some more prevotes; enterPrecommit
-	cs.scheduleTimeout(cs.config.Prevote(round), height, round, RoundStepPrevoteWait)
+	cs.scheduleTimeout(cs.config.Prevote(round), height, round, cstypes.RoundStepPrevoteWait)
 }
 
 // Enter: `timeoutPrevote` after any +2/3 prevotes.
@@ -1072,8 +961,8 @@ func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
 // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round)
 // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil,
 // else, precommit nil otherwise.
-func (cs *ConsensusState) enterPrecommit(height int, round int) {
-	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
+func (cs *ConsensusState) enterPrecommit(height int64, round int) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommit <= cs.Step) {
 		cs.Logger.Debug(cmn.Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -1082,7 +971,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 
 	defer func() {
 		// Done enterPrecommit:
-		cs.updateRoundStep(round, RoundStepPrecommit)
+		cs.updateRoundStep(round, cstypes.RoundStepPrecommit)
 		cs.newStep()
 	}()
 
@@ -1100,7 +989,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 	}
 
 	// At this point +2/3 prevoted for a particular block or nil
-	types.FireEventPolka(cs.evsw, cs.RoundStateEvent())
+	cs.eventBus.PublishEventPolka(cs.RoundStateEvent())
 
 	// the latest POLRound should be this round
 	polRound, _ := cs.Votes.POLInfo()
@@ -1117,7 +1006,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 			cs.LockedRound = 0
 			cs.LockedBlock = nil
 			cs.LockedBlockParts = nil
-			types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
+			cs.eventBus.PublishEventUnlock(cs.RoundStateEvent())
 		}
 		cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
 		return
@@ -1129,7 +1018,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 	if cs.LockedBlock.HashesTo(blockID.Hash) {
 		cs.Logger.Info("enterPrecommit: +2/3 prevoted locked block. Relocking")
 		cs.LockedRound = round
-		types.FireEventRelock(cs.evsw, cs.RoundStateEvent())
+		cs.eventBus.PublishEventRelock(cs.RoundStateEvent())
 		cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
 		return
 	}
@@ -1144,7 +1033,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 		cs.LockedRound = round
 		cs.LockedBlock = cs.ProposalBlock
 		cs.LockedBlockParts = cs.ProposalBlockParts
-		types.FireEventLock(cs.evsw, cs.RoundStateEvent())
+		cs.eventBus.PublishEventLock(cs.RoundStateEvent())
 		cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
 		return
 	}
@@ -1160,13 +1049,13 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 		cs.ProposalBlock = nil
 		cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
 	}
-	types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
+	cs.eventBus.PublishEventUnlock(cs.RoundStateEvent())
 	cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
 }
 
 // Enter: any +2/3 precommits for next round.
-func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
-	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
+func (cs *ConsensusState) enterPrecommitWait(height int64, round int) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommitWait <= cs.Step) {
 		cs.Logger.Debug(cmn.Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -1177,18 +1066,18 @@ func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
 
 	defer func() {
 		// Done enterPrecommitWait:
-		cs.updateRoundStep(round, RoundStepPrecommitWait)
+		cs.updateRoundStep(round, cstypes.RoundStepPrecommitWait)
 		cs.newStep()
 	}()
 
 	// Wait for some more precommits; enterNewRound
-	cs.scheduleTimeout(cs.config.Precommit(round), height, round, RoundStepPrecommitWait)
+	cs.scheduleTimeout(cs.config.Precommit(round), height, round, cstypes.RoundStepPrecommitWait)
 
 }
 
 // Enter: +2/3 precommits for block
-func (cs *ConsensusState) enterCommit(height int, commitRound int) {
-	if cs.Height != height || RoundStepCommit <= cs.Step {
+func (cs *ConsensusState) enterCommit(height int64, commitRound int) {
+	if cs.Height != height || cstypes.RoundStepCommit <= cs.Step {
 		cs.Logger.Debug(cmn.Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -1197,7 +1086,7 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) {
 	defer func() {
 		// Done enterCommit:
 		// keep cs.Round the same, commitRound points to the right Precommits set.
-		cs.updateRoundStep(cs.Round, RoundStepCommit)
+		cs.updateRoundStep(cs.Round, cstypes.RoundStepCommit)
 		cs.CommitRound = commitRound
 		cs.CommitTime = time.Now()
 		cs.newStep()
@@ -1233,7 +1122,7 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) {
 }
 
 // If we have the block AND +2/3 commits for it, finalize.
-func (cs *ConsensusState) tryFinalizeCommit(height int) {
+func (cs *ConsensusState) tryFinalizeCommit(height int64) {
 	if cs.Height != height {
 		cmn.PanicSanity(cmn.Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
 	}
@@ -1246,7 +1135,7 @@ func (cs *ConsensusState) tryFinalizeCommit(height int) {
 	if !cs.ProposalBlock.HashesTo(blockID.Hash) {
 		// TODO: this happens every time if we're not a validator (ugly logs)
 		// TODO: ^^ wait, why does it matter that we're a validator?
-		cs.Logger.Error("Attempt to finalize failed. We don't have the commit block.", "height", height, "proposal-block", cs.ProposalBlock.Hash(), "commit-block", blockID.Hash)
+		cs.Logger.Info("Attempt to finalize failed. We don't have the commit block.", "height", height, "proposal-block", cs.ProposalBlock.Hash(), "commit-block", blockID.Hash)
 		return
 	}
 
@@ -1254,9 +1143,9 @@ func (cs *ConsensusState) tryFinalizeCommit(height int) {
 	cs.finalizeCommit(height)
 }
 
-// Increment height and goto RoundStepNewHeight
-func (cs *ConsensusState) finalizeCommit(height int) {
-	if cs.Height != height || cs.Step != RoundStepCommit {
+// Increment height and goto cstypes.RoundStepNewHeight
+func (cs *ConsensusState) finalizeCommit(height int64) {
+	if cs.Height != height || cs.Step != cstypes.RoundStepCommit {
 		cs.Logger.Debug(cmn.Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -1304,23 +1193,25 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	// WAL replay for blocks with an #ENDHEIGHT
 	// As is, ConsensusState should not be started again
 	// until we successfully call ApplyBlock (ie. here or in Handshake after restart)
-	if cs.wal != nil {
-		cs.wal.writeEndHeight(height)
-	}
+	cs.wal.Save(EndHeightMessage{height})
 
 	fail.Fail() // XXX
 
 	// Create a copy of the state for staging
 	// and an event cache for txs
 	stateCopy := cs.state.Copy()
-	eventCache := types.NewEventCache(cs.evsw)
+	txEventBuffer := types.NewTxEventBuffer(cs.eventBus, block.NumTxs)
 
 	// Execute and commit the block, update and save the state, and update the mempool.
 	// All calls to the proxyAppConn come here.
 	// NOTE: the block.AppHash wont reflect these txs until the next block
-	err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
+	err := stateCopy.ApplyBlock(txEventBuffer, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
 	if err != nil {
 		cs.Logger.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "err", err)
+		err := cmn.Kill()
+		if err != nil {
+			cs.Logger.Error("Failed to kill this process - please do so manually", "err", err)
+		}
 		return
 	}
 
@@ -1333,9 +1224,12 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	//	* Fire before persisting state, in ApplyBlock
 	//	* Fire on start up if we haven't written any new WAL msgs
 	//   Both options mean we may fire more than once. Is that fine ?
-	types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block})
-	types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header})
-	eventCache.Flush()
+	cs.eventBus.PublishEventNewBlock(types.EventDataNewBlock{block})
+	cs.eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{block.Header})
+	err = txEventBuffer.Flush()
+	if err != nil {
+		cs.Logger.Error("Failed to flush event buffer", "err", err)
+	}
 
 	fail.Fail() // XXX
 
@@ -1350,7 +1244,7 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 
 	// By here,
 	// * cs.Height has been increment to height+1
-	// * cs.Step is now RoundStepNewHeight
+	// * cs.Step is now cstypes.RoundStepNewHeight
 	// * cs.StartTime is set to when we will start round0.
 }
 
@@ -1368,8 +1262,8 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
 		return nil
 	}
 
-	// We don't care about the proposal if we're already in RoundStepCommit.
-	if RoundStepCommit <= cs.Step {
+	// We don't care about the proposal if we're already in cstypes.RoundStepCommit.
+	if cstypes.RoundStepCommit <= cs.Step {
 		return nil
 	}
 
@@ -1391,7 +1285,7 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
 
 // NOTE: block is not necessarily valid.
 // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block.
-func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, verify bool) (added bool, err error) {
+func (cs *ConsensusState) addProposalBlockPart(height int64, part *types.Part, verify bool) (added bool, err error) {
 	// Blocks might be reused, so round mismatch is OK
 	if cs.Height != height {
 		return false, nil
@@ -1411,13 +1305,13 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, ver
 		var n int
 		var err error
 		cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(),
-			cs.state.Params().BlockSizeParams.MaxBytes, &n, &err).(*types.Block)
+			cs.state.Params.BlockSizeParams.MaxBytes, &n, &err).(*types.Block)
 		// NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
 		cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
-		if cs.Step == RoundStepPropose && cs.isProposalComplete() {
+		if cs.Step == cstypes.RoundStepPropose && cs.isProposalComplete() {
 			// Move onto the next step
 			cs.enterPrevote(height, cs.Round)
-		} else if cs.Step == RoundStepCommit {
+		} else if cs.Step == cstypes.RoundStepCommit {
 			// If we're waiting on the proposal block...
 			cs.tryFinalizeCommit(height)
 		}
@@ -1462,7 +1356,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 	// A precommit for the previous height?
 	// These come in while we wait timeoutCommit
 	if vote.Height+1 == cs.Height {
-		if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) {
+		if !(cs.Step == cstypes.RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) {
 			// TODO: give the reason ..
 			// fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.")
 			return added, ErrVoteHeightMismatch
@@ -1470,12 +1364,12 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 		added, err = cs.LastCommit.AddVote(vote)
 		if added {
 			cs.Logger.Info(cmn.Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
-			types.FireEventVote(cs.evsw, types.EventDataVote{vote})
+			cs.eventBus.PublishEventVote(types.EventDataVote{vote})
 
 			// if we can skip timeoutCommit and have all the votes now,
 			if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() {
 				// go straight to new round (skip timeout commit)
-				// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
+				// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, cstypes.RoundStepNewHeight)
 				cs.enterNewRound(cs.Height, 0)
 			}
 		}
@@ -1488,7 +1382,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 		height := cs.Height
 		added, err = cs.Votes.AddVote(vote, peerKey)
 		if added {
-			types.FireEventVote(cs.evsw, types.EventDataVote{vote})
+			cs.eventBus.PublishEventVote(types.EventDataVote{vote})
 
 			switch vote.Type {
 			case types.VoteTypePrevote:
@@ -1506,7 +1400,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 						cs.LockedRound = 0
 						cs.LockedBlock = nil
 						cs.LockedBlockParts = nil
-						types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
+						cs.eventBus.PublishEventUnlock(cs.RoundStateEvent())
 					}
 				}
 				if cs.Round <= vote.Round && prevotes.HasTwoThirdsAny() {
@@ -1539,7 +1433,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 						if cs.config.SkipTimeoutCommit && precommits.HasAll() {
 							// if we have all the votes now,
 							// go straight to new round (skip timeout commit)
-							// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
+							// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, cstypes.RoundStepNewHeight)
 							cs.enterNewRound(cs.Height, 0)
 						}
 
@@ -1600,7 +1494,7 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part
 
 //---------------------------------------------------------
 
-func CompareHRS(h1, r1 int, s1 RoundStepType, h2, r2 int, s2 RoundStepType) int {
+func CompareHRS(h1 int64, r1 int, s1 cstypes.RoundStepType, h2 int64, r2 int, s2 cstypes.RoundStepType) int {
 	if h1 < h2 {
 		return -1
 	} else if h1 > h2 {

consensus/state_test.go

@@ -2,12 +2,16 @@ package consensus
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"testing"
 	"time"
 
+	cstypes "github.com/tendermint/tendermint/consensus/types"
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+	tmpubsub "github.com/tendermint/tmlibs/pubsub"
 )
 
 func init() {
@@ -55,8 +59,8 @@ func TestProposerSelection0(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	height, round := cs1.Height, cs1.Round
 
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
 
 	startTestRound(cs1, height, round)
 
@@ -80,7 +84,7 @@ func TestProposerSelection0(t *testing.T) {
 
 	prop = cs1.GetRoundState().Validators.GetProposer()
 	if !bytes.Equal(prop.Address, vss[1].GetAddress()) {
-		panic(Fmt("expected proposer to be validator %d. Got %X", 1, prop.Address))
+		panic(cmn.Fmt("expected proposer to be validator %d. Got %X", 1, prop.Address))
 	}
 }
 
@@ -88,7 +92,7 @@ func TestProposerSelection0(t *testing.T) {
 func TestProposerSelection2(t *testing.T) {
 	cs1, vss := randConsensusState(4) // test needs more work for more than 3 validators
 
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 
 	// this time we jump in at round 2
 	incrementRound(vss[1:]...)
@@ -101,7 +105,7 @@ func TestProposerSelection2(t *testing.T) {
 	for i := 0; i < len(vss); i++ {
 		prop := cs1.GetRoundState().Validators.GetProposer()
 		if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].GetAddress()) {
-			panic(Fmt("expected proposer to be validator %d. Got %X", (i+2)%len(vss), prop.Address))
+			panic(cmn.Fmt("expected proposer to be validator %d. Got %X", (i+2)%len(vss), prop.Address))
 		}
 
 		rs := cs1.GetRoundState()
@@ -120,7 +124,7 @@ func TestEnterProposeNoPrivValidator(t *testing.T) {
 	height, round := cs.Height, cs.Round
 
 	// Listen for propose timeout event
-	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
 
 	startTestRound(cs, height, round)
 
@@ -145,8 +149,8 @@ func TestEnterProposeYesPrivValidator(t *testing.T) {
 
 	// Listen for propose timeout event
 
-	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	proposalCh := subscribeToEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), 1)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
+	proposalCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal)
 
 	cs.enterNewRound(height, round)
 	cs.startRoutines(3)
@@ -180,10 +184,10 @@ func TestBadProposal(t *testing.T) {
 	height, round := cs1.Height, cs1.Round
 	vs2 := vss[1]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
 
 	propBlock, _ := cs1.createProposalBlock() //changeProposer(t, cs1, vs2)
 
@@ -205,7 +209,9 @@ func TestBadProposal(t *testing.T) {
 	}
 
 	// set the proposal block
-	cs1.SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	// start the machine
 	startTestRound(cs1, height, round)
@@ -237,9 +243,17 @@ func TestFullRound1(t *testing.T) {
 	cs, vss := randConsensusState(1)
 	height, round := cs.Height, cs.Round
 
-	voteCh := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 0)
-	propCh := subscribeToEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	newRoundCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewRound(), 1)
+	// NOTE: buffer capacity of 0 ensures we can validate prevote and last commit
+	// before consensus can move to the next height (and cause a race condition)
+	cs.eventBus.Stop()
+	eventBus := types.NewEventBusWithBufferCapacity(0)
+	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+	cs.SetEventBus(eventBus)
+	eventBus.Start()
+
+	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
+	propCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal)
+	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
 
 	startTestRound(cs, height, round)
 
@@ -247,11 +261,9 @@ func TestFullRound1(t *testing.T) {
 
 	// grab proposal
 	re := <-propCh
-	propBlockHash := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState).ProposalBlock.Hash()
+	propBlockHash := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState).ProposalBlock.Hash()
 
 	<-voteCh // wait for prevote
-	// NOTE: voteChan cap of 0 ensures we can complete this
-	// before consensus can move to the next height (and cause a race condition)
 	validatePrevote(t, cs, round, vss[0], propBlockHash)
 
 	<-voteCh // wait for precommit
@@ -267,7 +279,7 @@ func TestFullRoundNil(t *testing.T) {
 	cs, vss := randConsensusState(1)
 	height, round := cs.Height, cs.Round
 
-	voteCh := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1)
+	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
 
 	cs.enterPrevote(height, round)
 	cs.startRoutines(4)
@@ -286,8 +298,8 @@ func TestFullRound2(t *testing.T) {
 	vs2 := vss[1]
 	height, round := cs1.Height, cs1.Round
 
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
-	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, height, round)
@@ -327,13 +339,13 @@ func TestLockNoPOL(t *testing.T) {
 	vs2 := vss[1]
 	height := cs1.Height
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 
 	/*
 		Round1 (cs1, B) // B B // B B2
@@ -344,7 +356,7 @@ func TestLockNoPOL(t *testing.T) {
 	cs1.startRoutines(0)
 
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -384,7 +396,7 @@ func TestLockNoPOL(t *testing.T) {
 
 	// now we're on a new round and not the proposer, so wait for timeout
 	re = <-timeoutProposeCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	if rs.ProposalBlock != nil {
 		panic("Expected proposal block to be nil")
@@ -428,11 +440,11 @@ func TestLockNoPOL(t *testing.T) {
 	incrementRound(vs2)
 
 	re = <-proposalCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	// now we're on a new round and are the proposer
 	if !bytes.Equal(rs.ProposalBlock.Hash(), rs.LockedBlock.Hash()) {
-		panic(Fmt("Expected proposal block to be locked block. Got %v, Expected %v", rs.ProposalBlock, rs.LockedBlock))
+		panic(cmn.Fmt("Expected proposal block to be locked block. Got %v, Expected %v", rs.ProposalBlock, rs.LockedBlock))
 	}
 
 	<-voteCh // prevote
@@ -468,7 +480,9 @@ func TestLockNoPOL(t *testing.T) {
 
 	// now we're on a new round and not the proposer
 	// so set the proposal block
-	cs1.SetProposalAndBlock(prop, propBlock, propBlock.MakePartSet(partSize), "")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlock.MakePartSet(partSize), ""); err != nil {
+		t.Fatal(err)
+	}
 
 	<-proposalCh
 	<-voteCh // prevote
@@ -493,14 +507,14 @@ func TestLockPOLRelock(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlockHeader(), 1)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader)
 
 	// everything done from perspective of cs1
 
@@ -515,7 +529,7 @@ func TestLockPOLRelock(t *testing.T) {
 
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -545,7 +559,9 @@ func TestLockPOLRelock(t *testing.T) {
 	<-timeoutWaitCh
 
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
-	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	<-newRoundCh
 	t.Log("### ONTO ROUND 1")
@@ -591,7 +607,7 @@ func TestLockPOLRelock(t *testing.T) {
 	be := <-newBlockCh
 	b := be.(types.TMEventData).Unwrap().(types.EventDataNewBlockHeader)
 	re = <-newRoundCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	if rs.Height != 2 {
 		panic("Expected height to increment")
 	}
@@ -606,13 +622,13 @@ func TestLockPOLUnlock(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// everything done from perspective of cs1
@@ -627,7 +643,7 @@ func TestLockPOLUnlock(t *testing.T) {
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -653,11 +669,13 @@ func TestLockPOLUnlock(t *testing.T) {
 
 	// timeout to new round
 	re = <-timeoutWaitCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	lockedBlockHash := rs.LockedBlock.Hash()
 
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
-	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	<-newRoundCh
 	t.Log("#### ONTO ROUND 1")
@@ -701,19 +719,19 @@ func TestLockPOLSafety1(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	propBlock := rs.ProposalBlock
 
 	<-voteCh // prevote
@@ -744,7 +762,9 @@ func TestLockPOLSafety1(t *testing.T) {
 	incrementRound(vs2, vs3, vs4)
 
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
-	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	<-newRoundCh
 	t.Log("### ONTO ROUND 1")
@@ -761,7 +781,7 @@ func TestLockPOLSafety1(t *testing.T) {
 		re = <-proposalCh
 	}
 
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	if rs.LockedBlock != nil {
 		panic("we should not be locked!")
@@ -801,7 +821,7 @@ func TestLockPOLSafety1(t *testing.T) {
 	// we should prevote what we're locked on
 	validatePrevote(t, cs1, 2, vss[0], propBlockHash)
 
-	newStepCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRoundStep(), 1)
+	newStepCh := subscribe(cs1.eventBus, types.EventQueryNewRoundStep)
 
 	// add prevotes from the earlier round
 	addVotes(cs1, prevotes...)
@@ -822,13 +842,13 @@ func TestLockPOLSafety2(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// the block for R0: gets polkad but we miss it
@@ -848,7 +868,7 @@ func TestLockPOLSafety2(t *testing.T) {
 
 	incrementRound(vs2, vs3, vs4)
 
-	cs1.updateRoundStep(0, RoundStepPrecommitWait)
+	cs1.updateRoundStep(0, cstypes.RoundStepPrecommitWait)
 
 	t.Log("### ONTO Round 1")
 	// jump in at round 1
@@ -856,7 +876,9 @@ func TestLockPOLSafety2(t *testing.T) {
 	startTestRound(cs1, height, 1)
 	<-newRoundCh
 
-	cs1.SetProposalAndBlock(prop1, propBlock1, propBlockParts1, "some peer")
+	if err := cs1.SetProposalAndBlock(prop1, propBlock1, propBlockParts1, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 	<-proposalCh
 
 	<-voteCh // prevote
@@ -881,7 +903,9 @@ func TestLockPOLSafety2(t *testing.T) {
 	if err := vs3.SignProposal(config.ChainID, newProp); err != nil {
 		t.Fatal(err)
 	}
-	cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer")
+	if err := cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer"); err != nil {
+		t.Fatal(err)
+	}
 
 	// Add the pol votes
 	addVotes(cs1, prevotes...)
@@ -918,9 +942,9 @@ func TestSlashingPrevotes(t *testing.T) {
 	vs2 := vss[1]
 
 
-	proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1)
-	newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
@@ -929,7 +953,7 @@ func TestSlashingPrevotes(t *testing.T) {
 	re := <-proposalCh
 	<-voteCh // prevote
 
-	rs := re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	// we should now be stuck in limbo forever, waiting for more prevotes
 	// add one for a different block should cause us to go into prevote wait
@@ -953,9 +977,9 @@ func TestSlashingPrecommits(t *testing.T) {
 	vs2 := vss[1]
 
 
-	proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1)
-	newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
@@ -997,19 +1021,19 @@ func TestHalt1(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := cs1.state.Params().BlockPartSizeBytes
+	partSize := cs1.state.Params.BlockPartSizeBytes
 
-	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
-	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
-	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
-	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1)
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
 	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 	propBlock := rs.ProposalBlock
 	propBlockParts := propBlock.MakePartSet(partSize)
 
@@ -1032,7 +1056,7 @@ func TestHalt1(t *testing.T) {
 	// timeout to new round
 	<-timeoutWaitCh
 	re = <-newRoundCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	t.Log("### ONTO ROUND 1")
 	/*Round2
@@ -1050,9 +1074,26 @@ func TestHalt1(t *testing.T) {
 	// receiving that precommit should take us straight to commit
 	<-newBlockCh
 	re = <-newRoundCh
-	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*cstypes.RoundState)
 
 	if rs.Height != 2 {
 		panic("expected height to increment")
 	}
 }
+
+// subscribe subscribes test client to the given query and returns a channel with cap = 1.
+func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan interface{} {
+	out := make(chan interface{}, 1)
+	err := eventBus.Subscribe(context.Background(), testSubscriber, q, out)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, q))
+	}
+	return out
+}
+
+// discardFromChan reads n values from the channel.
+func discardFromChan(ch <-chan interface{}, n int) {
+	for i := 0; i < n; i++ {
+		<-ch
+	}
+}

consensus/test_data/README.md

@@ -1,36 +0,0 @@
-# Generating test data
-
-To generate the data, run `build.sh`. See that script for more details.
-
-Make sure to adjust the stepChanges in the testCases if the number of messages changes.
-This sometimes happens for the `small_block2.cswal`, where the number of block parts changes between 4 and 5.
-
-If you need to change the signatures, you can use a script as follows:
-The privBytes comes from `config/tendermint_test/...`:
-
-```
-package main
-
-import (
-	"encoding/hex"
-	"fmt"
-
-	"github.com/tendermint/go-crypto"
-)
-
-func main() {
-	signBytes, err := hex.DecodeString("7B22636861696E5F6964223A2274656E6465726D696E745F74657374222C22766F7465223A7B22626C6F636B5F68617368223A2242453544373939433846353044354645383533364334333932464443384537423342313830373638222C22626C6F636B5F70617274735F686561646572223A506172745365747B543A31204236323237323535464632307D2C22686569676874223A312C22726F756E64223A302C2274797065223A327D7D")
-	if err != nil {
-		panic(err)
-	}
-	privBytes, err := hex.DecodeString("27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8")
-	if err != nil {
-		panic(err)
-	}
-	privKey := crypto.PrivKeyEd25519{}
-	copy(privKey[:], privBytes)
-	signature := privKey.Sign(signBytes)
-	fmt.Printf("Signature Bytes: %X\n", signature.Bytes())
-}
-```
-

consensus/test_data/build.sh

@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-
-# XXX: removes tendermint dir
-
-cd "$GOPATH/src/github.com/tendermint/tendermint" || exit 1
-
-# Make sure we have a tendermint command.
-if ! hash tendermint 2>/dev/null; then
-	make install
-fi
-
-# specify a dir to copy
-# TODO: eventually we should replace with `tendermint init --test`
-DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test
-
-TMHOME="$HOME/.tendermint"
-rm -rf "$TMHOME"
-cp -r "$DIR_TO_COPY" "$TMHOME"
-cp $TMHOME/config.toml $TMHOME/config.toml.bak
-
-function reset(){
-	tendermint unsafe_reset_all
-	cp $TMHOME/config.toml.bak $TMHOME/config.toml
-}
-
-reset
-
-# empty block
-function empty_block(){
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 5
-killall tendermint
-
-# /q would print up to and including the match, then quit.
-# /Q doesn't include the match.
-# http://unix.stackexchange.com/questions/11305/grep-show-all-the-file-up-to-the-match
-sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/empty_block.cswal
-
-reset
-}
-
-# many blocks
-function many_blocks(){
-bash scripts/txs/random.sh 1000 36657 &> /dev/null &
-PID=$!
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 7
-killall tendermint
-kill -9 $PID
-
-sed '/ENDHEIGHT: 6/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/many_blocks.cswal
-
-reset
-}
-
-
-# small block 1
-function small_block1(){
-bash scripts/txs/random.sh 1000 36657 &> /dev/null &
-PID=$!
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 10
-killall tendermint
-kill -9 $PID
-
-sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block1.cswal
-
-reset
-}
-
-
-# small block 2 (part size = 512)
-function small_block2(){
-echo "" >> ~/.tendermint/config.toml
-echo "block_part_size = 512" >> ~/.tendermint/config.toml
-bash scripts/txs/random.sh 1000 36657 &> /dev/null &
-PID=$!
-tendermint node --proxy_app=persistent_dummy &> /dev/null &
-sleep 5
-killall tendermint
-kill -9 $PID
-
-sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block2.cswal
-
-reset
-}
-
-
-
-case "$1" in
-	"small_block1")
-		small_block1
-		;;
-	"small_block2")
-		small_block2
-		;;
-	"empty_block")
-		empty_block
-		;;
-	"many_blocks")
-		many_blocks
-		;;
-	*)
-		small_block1
-		small_block2
-		empty_block
-		many_blocks
-esac
-
-

consensus/test_data/empty_block.cswal

@@ -1,10 +0,0 @@
-#ENDHEIGHT: 0
-{"time":"2017-04-27T22:24:01.346Z","msg":[3,{"duration":972946821,"height":1,"round":0,"step":1}]}
-{"time":"2017-04-27T22:24:01.349Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
-{"time":"2017-04-27T22:24:01.349Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"E785764AED6D92D7CC65C0A3A4ED9C8465198A05142C3E6C7F3EF601FDCD3A604900B77B7B87C046221EF99FD038A960398385BD5BBAA50EE4F86DE757B8F704"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.350Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114B96165CF4496C00000000000000114354594CBFC1A7BCA1AD0050ED6AA010023EADA390001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.351Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
-{"time":"2017-04-27T22:24:01.351Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"F3BBFBE7E4A5D619E2C498C3D1B912883786DD71","parts":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"}},"signature":[1,"35C937C78D061ECDC3770982A1330C9AA7F6FEF00835C43DEB50B8FCF69A3EEF221E675EE5E469114F64E4FBBABA414EB9170E1025FC47D3F0EADE46767D2E00"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.352Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
-{"time":"2017-04-27T22:24:01.352Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"F3BBFBE7E4A5D619E2C498C3D1B912883786DD71","parts":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"}},"signature":[1,"D1A7D27FCD5D352F3A3EDA8DE368520BC5B796662E32BCD8D91CDB8209A88DAF37CB7C4C93143D3C12B37C1435229268098CFFD0AD1400D88DA7606454692301"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:24:01.352Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}

consensus/test_data/small_block2.cswal

@@ -1,15 +0,0 @@
-#ENDHEIGHT: 0
-{"time":"2017-04-27T22:23:56.310Z","msg":[3,{"duration":969732098,"height":1,"round":0,"step":1}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"7624F6E943B7A207E16D1FA87EA099BD924E930F98E7DECBC01DB37735C619409588A67C2EABA9845FD6B80FDB65ECFCDA5F0DEFCEF74B8C34DB8E0540480203"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114B96164A30A118001620000000001141F6753D22BACA2180B1EADD722434EB28444D91D0114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010162011A3631363236333634333133303344363436333632363133313330011A3631363236333634333133313344363436333632363133313331011A3631363236333634333133323344363436333632363133313332011A3631363236333634333133333344363436333632363133313333011A3631363236333634333133343344363436333632363133313334011A3631363236333634333133353344363436333632363133313335011A3631363236333634333133363344363436333632363133313336011A3631363236333634333133373344363436333632363133313337011A3631363236333634333133383344363436333632363133313338011A3631363236333634333133393344363436333632363133313339011A3631363236333634333233303344363436333632363133323330011A3631363236333634333233313344363436333632363133323331011A3631363236333634333233323344363436333632363133323332011A3631363236333634333233333344363436333632363133323333011A3631363236333634333233343344363436333632363133323334011A36313632363336","proof":{"aunts":["49F4B71E3D7C457415069E2EA916DB12F67AA8D0","D35A72BEDAAAAC17045D7BFAAFA94C2EC0B0A4C2","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"34333233353344363436333632363133323335011A3631363236333634333233363344363436333632363133323336011A3631363236333634333233373344363436333632363133323337011A3631363236333634333233383344363436333632363133323338011A3631363236333634333233393344363436333632363133323339011A3631363236333634333333303344363436333632363133333330011A3631363236333634333333313344363436333632363133333331011A3631363236333634333333323344363436333632363133333332011A3631363236333634333333333344363436333632363133333333011A3631363236333634333333343344363436333632363133333334011A3631363236333634333333353344363436333632363133333335011A3631363236333634333333363344363436333632363133333336011A3631363236333634333333373344363436333632363133333337011A3631363236333634333333383344363436333632363133333338011A3631363236333634333333393344363436333632363133333339011A3631363236333634333433303344363436333632363133343330011A3631363236333634333433313344363436333632363133343331011A3631363236333634333433323344363436333632363133343332011A363136323633363433343333334436","proof":{"aunts":["5AD2A9A1A49A1FD6EF83F05FA4588F800B29DEF1","D35A72BEDAAAAC17045D7BFAAFA94C2EC0B0A4C2","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"3436333632363133343333011A3631363236333634333433343344363436333632363133343334011A3631363236333634333433353344363436333632363133343335011A3631363236333634333433363344363436333632363133343336011A3631363236333634333433373344363436333632363133343337011A3631363236333634333433383344363436333632363133343338011A3631363236333634333433393344363436333632363133343339011A3631363236333634333533303344363436333632363133353330011A3631363236333634333533313344363436333632363133353331011A3631363236333634333533323344363436333632363133353332011A3631363236333634333533333344363436333632363133353333011A3631363236333634333533343344363436333632363133353334011A3631363236333634333533353344363436333632363133353335011A3631363236333634333533363344363436333632363133353336011A3631363236333634333533373344363436333632363133353337011A3631363236333634333533383344363436333632363133353338011A3631363236333634333533393344363436333632363133353339011A3631363236333634333633303344363436333632363133363330011A3631363236333634333633313344363436333632363133","proof":{"aunts":["8B5786C3D871EE37B0F4B2DECAC39E157340DFBE","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"363331011A3631363236333634333633323344363436333632363133363332011A3631363236333634333633333344363436333632363133363333011A3631363236333634333633343344363436333632363133363334011A3631363236333634333633353344363436333632363133363335011A3631363236333634333633363344363436333632363133363336011A3631363236333634333633373344363436333632363133363337011A3631363236333634333633383344363436333632363133363338011A3631363236333634333633393344363436333632363133363339011A3631363236333634333733303344363436333632363133373330011A3631363236333634333733313344363436333632363133373331011A3631363236333634333733323344363436333632363133373332011A3631363236333634333733333344363436333632363133373333011A3631363236333634333733343344363436333632363133373334011A3631363236333634333733353344363436333632363133373335011A3631363236333634333733363344363436333632363133373336011A3631363236333634333733373344363436333632363133373337011A3631363236333634333733383344363436333632363133373338011A3631363236333634333733393344363436333632363133373339011A363136","proof":{"aunts":["56097661A1B2707588100586B3B1C2C8A51057D1","6DE889147DF528EEB5F7422E95DC45900CAFB619","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"3236333634333833303344363436333632363133383330011A3631363236333634333833313344363436333632363133383331011A3631363236333634333833323344363436333632363133383332011A3631363236333634333833333344363436333632363133383333011A3631363236333634333833343344363436333632363133383334011A3631363236333634333833353344363436333632363133383335011A3631363236333634333833363344363436333632363133383336011A3631363236333634333833373344363436333632363133383337011A3631363236333634333833383344363436333632363133383338011A3631363236333634333833393344363436333632363133383339011A3631363236333634333933303344363436333632363133393330011A3631363236333634333933313344363436333632363133393331011A3631363236333634333933323344363436333632363133393332011A3631363236333634333933333344363436333632363133393333011A3631363236333634333933343344363436333632363133393334011A3631363236333634333933353344363436333632363133393335011A3631363236333634333933363344363436333632363133393336011A3631363236333634333933373344363436333632363133393337011A3631363236333634333933","proof":{"aunts":["081D3DC5F11850851D5F0D760B98EE87BFA6B8B0","6DE889147DF528EEB5F7422E95DC45900CAFB619","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.313Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":5,"bytes":"383344363436333632363133393338011A3631363236333634333933393344363436333632363133393339011E363136323633363433313330333033443634363336323631333133303330011E363136323633363433313330333133443634363336323631333133303331011E363136323633363433313330333233443634363336323631333133303332011E363136323633363433313330333333443634363336323631333133303333011E363136323633363433313330333433443634363336323631333133303334011E363136323633363433313330333533443634363336323631333133303335011E363136323633363433313330333633443634363336323631333133303336011E3631363236333634333133303337334436343633363236313331333033370100000000","proof":{"aunts":["6AA912328C2B52EFA0ECE71F523E137E400EC484","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.314Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
-{"time":"2017-04-27T22:23:56.314Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"62371CF72F8662378691706DB256C833CF1AF81B","parts":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"}},"signature":[1,"255906FAAA50C84E85DABF7DE73468E4F95DB4E46F598848145926E2FAD77CA682BF07E09E2F3EC81FFBD9A036B67914A3C02F819B69248D777AEBA792725907"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.315Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
-{"time":"2017-04-27T22:23:56.315Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"62371CF72F8662378691706DB256C833CF1AF81B","parts":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"}},"signature":[1,"056CC15C748434D0A59B64B45CB56EDC1A437A426E68FA63DC7D61A7C17B0F768F207D81340D129A57C5A64195F8AFDD03B6BF28D7B2286290D61BCE88FCA304"]}}],"peer_key":""}]}
-{"time":"2017-04-27T22:23:56.316Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}

consensus/ticker.go

@@ -15,8 +15,8 @@ var (
 // conditional on the height/round/step in the timeoutInfo.
 // The timeoutInfo.Duration may be non-positive.
 type TimeoutTicker interface {
-	Start() (bool, error)
-	Stop() bool
+	Start() error
+	Stop() error
 	Chan() <-chan timeoutInfo       // on which to receive a timeout
 	ScheduleTimeout(ti timeoutInfo) // reset the timer
 
@@ -68,7 +68,7 @@ func (t *timeoutTicker) Chan() <-chan timeoutInfo {
 }
 
 // ScheduleTimeout schedules a new timeout by sending on the internal tickChan.
-// The timeoutRoutine is alwaya available to read from tickChan, so this won't block.
+// The timeoutRoutine is always available to read from tickChan, so this won't block.
 // The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
 func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) {
 	t.tickChan <- ti

consensus/types/height_vote_set.go

@@ -1,11 +1,11 @@
-package consensus
+package types
 
 import (
 	"strings"
 	"sync"
 
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 type RoundVoteSet struct {
@@ -29,7 +29,7 @@ One for their LastCommit round, and another for the official commit round.
 */
 type HeightVoteSet struct {
 	chainID string
-	height  int
+	height  int64
 	valSet  *types.ValidatorSet
 
 	mtx               sync.Mutex
@@ -38,7 +38,7 @@ type HeightVoteSet struct {
 	peerCatchupRounds map[string][]int     // keys: peer.Key; values: at most 2 rounds
 }
 
-func NewHeightVoteSet(chainID string, height int, valSet *types.ValidatorSet) *HeightVoteSet {
+func NewHeightVoteSet(chainID string, height int64, valSet *types.ValidatorSet) *HeightVoteSet {
 	hvs := &HeightVoteSet{
 		chainID: chainID,
 	}
@@ -46,7 +46,7 @@ func NewHeightVoteSet(chainID string, height int, valSet *types.ValidatorSet) *H
 	return hvs
 }
 
-func (hvs *HeightVoteSet) Reset(height int, valSet *types.ValidatorSet) {
+func (hvs *HeightVoteSet) Reset(height int64, valSet *types.ValidatorSet) {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 
@@ -59,7 +59,7 @@ func (hvs *HeightVoteSet) Reset(height int, valSet *types.ValidatorSet) {
 	hvs.round = 0
 }
 
-func (hvs *HeightVoteSet) Height() int {
+func (hvs *HeightVoteSet) Height() int64 {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 	return hvs.height
@@ -76,7 +76,7 @@ func (hvs *HeightVoteSet) SetRound(round int) {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 	if hvs.round != 0 && (round < hvs.round+1) {
-		PanicSanity("SetRound() must increment hvs.round")
+		cmn.PanicSanity("SetRound() must increment hvs.round")
 	}
 	for r := hvs.round + 1; r <= round; r++ {
 		if _, ok := hvs.roundVoteSets[r]; ok {
@@ -89,7 +89,7 @@ func (hvs *HeightVoteSet) SetRound(round int) {
 
 func (hvs *HeightVoteSet) addRound(round int) {
 	if _, ok := hvs.roundVoteSets[round]; ok {
-		PanicSanity("addRound() for an existing round")
+		cmn.PanicSanity("addRound() for an existing round")
 	}
 	// log.Debug("addRound(round)", "round", round)
 	prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, types.VoteTypePrevote, hvs.valSet)
@@ -164,7 +164,7 @@ func (hvs *HeightVoteSet) getVoteSet(round int, type_ byte) *types.VoteSet {
 	case types.VoteTypePrecommit:
 		return rvs.Precommits
 	default:
-		PanicSanity(Fmt("Unexpected vote type %X", type_))
+		cmn.PanicSanity(cmn.Fmt("Unexpected vote type %X", type_))
 		return nil
 	}
 }
@@ -194,7 +194,7 @@ func (hvs *HeightVoteSet) StringIndented(indent string) string {
 		voteSetString = roundVoteSet.Precommits.StringShort()
 		vsStrings = append(vsStrings, voteSetString)
 	}
-	return Fmt(`HeightVoteSet{H:%v R:0~%v
+	return cmn.Fmt(`HeightVoteSet{H:%v R:0~%v
 %s  %v
 %s}`,
 		hvs.height, hvs.round,

consensus/types/height_vote_set_test.go

@@ -1,14 +1,17 @@
-package consensus
+package types
 
 import (
 	"testing"
 
+	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/types"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
+var config *cfg.Config // NOTE: must be reset for each _test.go file
+
 func init() {
-	config = ResetConfig("consensus_height_vote_set_test")
+	config = cfg.ResetTestRoot("consensus_height_vote_set_test")
 }
 
 func TestPeerCatchupRounds(t *testing.T) {
@@ -44,7 +47,7 @@ func TestPeerCatchupRounds(t *testing.T) {
 
 }
 
-func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidatorFS, valIndex int) *types.Vote {
+func makeVoteHR(t *testing.T, height int64, round int, privVals []*types.PrivValidatorFS, valIndex int) *types.Vote {
 	privVal := privVals[valIndex]
 	vote := &types.Vote{
 		ValidatorAddress: privVal.GetAddress(),
@@ -57,7 +60,7 @@ func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidator
 	chainID := config.ChainID
 	err := privVal.SignVote(chainID, vote)
 	if err != nil {
-		panic(Fmt("Error signing vote: %v", err))
+		panic(cmn.Fmt("Error signing vote: %v", err))
 		return nil
 	}
 	return vote

consensus/types/reactor.go

@@ -0,0 +1,57 @@
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+//-----------------------------------------------------------------------------
+
+// PeerRoundState contains the known state of a peer.
+// NOTE: Read-only when returned by PeerState.GetRoundState().
+type PeerRoundState struct {
+	Height                   int64               // Height peer is at
+	Round                    int                 // Round peer is at, -1 if unknown.
+	Step                     RoundStepType       // Step peer is at
+	StartTime                time.Time           // Estimated start of round 0 at this height
+	Proposal                 bool                // True if peer has proposal for this round
+	ProposalBlockPartsHeader types.PartSetHeader //
+	ProposalBlockParts       *cmn.BitArray       //
+	ProposalPOLRound         int                 // Proposal's POL round. -1 if none.
+	ProposalPOL              *cmn.BitArray       // nil until ProposalPOLMessage received.
+	Prevotes                 *cmn.BitArray       // All votes peer has for this round
+	Precommits               *cmn.BitArray       // All precommits peer has for this round
+	LastCommitRound          int                 // Round of commit for last height. -1 if none.
+	LastCommit               *cmn.BitArray       // All commit precommits of commit for last height.
+	CatchupCommitRound       int                 // Round that we have commit for. Not necessarily unique. -1 if none.
+	CatchupCommit            *cmn.BitArray       // All commit precommits peer has for this height & CatchupCommitRound
+}
+
+// String returns a string representation of the PeerRoundState
+func (prs PeerRoundState) String() string {
+	return prs.StringIndented("")
+}
+
+// StringIndented returns a string representation of the PeerRoundState
+func (prs PeerRoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`PeerRoundState{
+%s  %v/%v/%v @%v
+%s  Proposal %v -> %v
+%s  POL      %v (round %v)
+%s  Prevotes   %v
+%s  Precommits %v
+%s  LastCommit %v (round %v)
+%s  Catchup    %v (round %v)
+%s}`,
+		indent, prs.Height, prs.Round, prs.Step, prs.StartTime,
+		indent, prs.ProposalBlockPartsHeader, prs.ProposalBlockParts,
+		indent, prs.ProposalPOL, prs.ProposalPOLRound,
+		indent, prs.Prevotes,
+		indent, prs.Precommits,
+		indent, prs.LastCommit, prs.LastCommitRound,
+		indent, prs.CatchupCommit, prs.CatchupCommitRound,
+		indent)
+}

consensus/types/state.go

@@ -0,0 +1,131 @@
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+// RoundStepType enum type
+
+// RoundStepType enumerates the state of the consensus state machine
+type RoundStepType uint8 // These must be numeric, ordered.
+
+const (
+	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
+	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
+	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
+	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
+	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
+	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
+	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
+	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
+	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
+)
+
+// String returns a string
+func (rs RoundStepType) String() string {
+	switch rs {
+	case RoundStepNewHeight:
+		return "RoundStepNewHeight"
+	case RoundStepNewRound:
+		return "RoundStepNewRound"
+	case RoundStepPropose:
+		return "RoundStepPropose"
+	case RoundStepPrevote:
+		return "RoundStepPrevote"
+	case RoundStepPrevoteWait:
+		return "RoundStepPrevoteWait"
+	case RoundStepPrecommit:
+		return "RoundStepPrecommit"
+	case RoundStepPrecommitWait:
+		return "RoundStepPrecommitWait"
+	case RoundStepCommit:
+		return "RoundStepCommit"
+	default:
+		return "RoundStepUnknown" // Cannot panic.
+	}
+}
+
+//-----------------------------------------------------------------------------
+
+// RoundState defines the internal consensus state.
+// It is Immutable when returned from ConsensusState.GetRoundState()
+// TODO: Actually, only the top pointer is copied,
+// so access to field pointers is still racey
+// NOTE: Not thread safe. Should only be manipulated by functions downstream
+// of the cs.receiveRoutine
+type RoundState struct {
+	Height             int64 // Height we are working on
+	Round              int
+	Step               RoundStepType
+	StartTime          time.Time
+	CommitTime         time.Time // Subjective time when +2/3 precommits for Block at Round were found
+	Validators         *types.ValidatorSet
+	Proposal           *types.Proposal
+	ProposalBlock      *types.Block
+	ProposalBlockParts *types.PartSet
+	LockedRound        int
+	LockedBlock        *types.Block
+	LockedBlockParts   *types.PartSet
+	Votes              *HeightVoteSet
+	CommitRound        int            //
+	LastCommit         *types.VoteSet // Last precommits at Height-1
+	LastValidators     *types.ValidatorSet
+}
+
+// RoundStateEvent returns the H/R/S of the RoundState as an event.
+func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
+	// XXX: copy the RoundState
+	// if we want to avoid this, we may need synchronous events after all
+	rs_ := *rs
+	edrs := types.EventDataRoundState{
+		Height:     rs.Height,
+		Round:      rs.Round,
+		Step:       rs.Step.String(),
+		RoundState: &rs_,
+	}
+	return edrs
+}
+
+// String returns a string
+func (rs *RoundState) String() string {
+	return rs.StringIndented("")
+}
+
+// StringIndented returns a string
+func (rs *RoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`RoundState{
+%s  H:%v R:%v S:%v
+%s  StartTime:     %v
+%s  CommitTime:    %v
+%s  Validators:    %v
+%s  Proposal:      %v
+%s  ProposalBlock: %v %v
+%s  LockedRound:   %v
+%s  LockedBlock:   %v %v
+%s  Votes:         %v
+%s  LastCommit: %v
+%s  LastValidators:    %v
+%s}`,
+		indent, rs.Height, rs.Round, rs.Step,
+		indent, rs.StartTime,
+		indent, rs.CommitTime,
+		indent, rs.Validators.StringIndented(indent+"    "),
+		indent, rs.Proposal,
+		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
+		indent, rs.LockedRound,
+		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
+		indent, rs.Votes.StringIndented(indent+"    "),
+		indent, rs.LastCommit.StringShort(),
+		indent, rs.LastValidators.StringIndented(indent+"    "),
+		indent)
+}
+
+// StringShort returns a string
+func (rs *RoundState) StringShort() string {
+	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
+		rs.Height, rs.Round, rs.Step, rs.StartTime)
+}

consensus/version.go

@@ -1,7 +1,7 @@
 package consensus
 
 import (
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 // kind of arbitrary
@@ -10,4 +10,4 @@ var Major = "0"    //
 var Minor = "2"    // replay refactor
 var Revision = "2" // validation -> commit
 
-var Version = Fmt("v%s/%s.%s.%s", Spec, Major, Minor, Revision)
+var Version = cmn.Fmt("v%s/%s.%s.%s", Spec, Major, Minor, Revision)

consensus/wal.go

@@ -1,22 +1,36 @@
 package consensus
 
 import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"path/filepath"
 	"time"
 
+	"github.com/pkg/errors"
+
 	wire "github.com/tendermint/go-wire"
 	"github.com/tendermint/tendermint/types"
 	auto "github.com/tendermint/tmlibs/autofile"
-	. "github.com/tendermint/tmlibs/common"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 //--------------------------------------------------------
 // types and functions for savings consensus messages
 
 type TimedWALMessage struct {
-	Time time.Time  `json:"time"`
+	Time time.Time  `json:"time"` // for debugging purposes
 	Msg  WALMessage `json:"msg"`
 }
 
+// EndHeightMessage marks the end of the given height inside WAL.
+// @internal used by scripts/wal2json util.
+type EndHeightMessage struct {
+	Height int64 `json:"height"`
+}
+
 type WALMessage interface{}
 
 var _ = wire.RegisterInterface(
@@ -24,81 +38,247 @@ var _ = wire.RegisterInterface(
 	wire.ConcreteType{types.EventDataRoundState{}, 0x01},
 	wire.ConcreteType{msgInfo{}, 0x02},
 	wire.ConcreteType{timeoutInfo{}, 0x03},
+	wire.ConcreteType{EndHeightMessage{}, 0x04},
 )
 
 //--------------------------------------------------------
 // Simple write-ahead logger
 
+// WAL is an interface for any write-ahead logger.
+type WAL interface {
+	Save(WALMessage)
+	Group() *auto.Group
+	SearchForEndHeight(height int64) (gr *auto.GroupReader, found bool, err error)
+
+	Start() error
+	Stop() error
+	Wait()
+}
+
 // Write ahead logger writes msgs to disk before they are processed.
 // Can be used for crash-recovery and deterministic replay
 // TODO: currently the wal is overwritten during replay catchup
 //   give it a mode so it's either reading or appending - must read to end to start appending again
-type WAL struct {
-	BaseService
+type baseWAL struct {
+	cmn.BaseService
 
 	group *auto.Group
 	light bool // ignore block parts
+
+	enc *WALEncoder
 }
 
-func NewWAL(walFile string, light bool) (*WAL, error) {
+func NewWAL(walFile string, light bool) (*baseWAL, error) {
+	err := cmn.EnsureDir(filepath.Dir(walFile), 0700)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to ensure WAL directory is in place")
+	}
+
 	group, err := auto.OpenGroup(walFile)
 	if err != nil {
 		return nil, err
 	}
-	wal := &WAL{
+	wal := &baseWAL{
 		group: group,
 		light: light,
+		enc:   NewWALEncoder(group),
 	}
-	wal.BaseService = *NewBaseService(nil, "WAL", wal)
+	wal.BaseService = *cmn.NewBaseService(nil, "baseWAL", wal)
 	return wal, nil
 }
 
-func (wal *WAL) OnStart() error {
+func (wal *baseWAL) Group() *auto.Group {
+	return wal.group
+}
+
+func (wal *baseWAL) OnStart() error {
 	size, err := wal.group.Head.Size()
 	if err != nil {
 		return err
 	} else if size == 0 {
-		wal.writeEndHeight(0)
+		wal.Save(EndHeightMessage{0})
 	}
-	_, err = wal.group.Start()
+	err = wal.group.Start()
 	return err
 }
 
-func (wal *WAL) OnStop() {
+func (wal *baseWAL) OnStop() {
 	wal.BaseService.OnStop()
 	wal.group.Stop()
 }
 
 // called in newStep and for each pass in receiveRoutine
-func (wal *WAL) Save(wmsg WALMessage) {
+func (wal *baseWAL) Save(msg WALMessage) {
 	if wal == nil {
 		return
 	}
+
 	if wal.light {
 		// in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts)
-		if mi, ok := wmsg.(msgInfo); ok {
+		if mi, ok := msg.(msgInfo); ok {
 			if mi.PeerKey != "" {
 				return
 			}
 		}
 	}
+
 	// Write the wal message
-	var wmsgBytes = wire.JSONBytes(TimedWALMessage{time.Now(), wmsg})
-	err := wal.group.WriteLine(string(wmsgBytes))
+	if err := wal.enc.Encode(&TimedWALMessage{time.Now(), msg}); err != nil {
+		cmn.PanicQ(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg))
+	}
+
+	// TODO: only flush when necessary
+	if err := wal.group.Flush(); err != nil {
+		cmn.PanicQ(cmn.Fmt("Error flushing consensus wal buf to file. Error: %v \n", err))
+	}
+}
+
+// SearchForEndHeight searches for the EndHeightMessage with the height and
+// returns an auto.GroupReader, whenever it was found or not and an error.
+// Group reader will be nil if found equals false.
+//
+// CONTRACT: caller must close group reader.
+func (wal *baseWAL) SearchForEndHeight(height int64) (gr *auto.GroupReader, found bool, err error) {
+	var msg *TimedWALMessage
+
+	// NOTE: starting from the last file in the group because we're usually
+	// searching for the last height. See replay.go
+	min, max := wal.group.MinIndex(), wal.group.MaxIndex()
+	wal.Logger.Debug("Searching for height", "height", height, "min", min, "max", max)
+	for index := max; index >= min; index-- {
+		gr, err = wal.group.NewReader(index)
+		if err != nil {
+			return nil, false, err
+		}
+
+		dec := NewWALDecoder(gr)
+		for {
+			msg, err = dec.Decode()
+			if err == io.EOF {
+				// check next file
+				break
+			}
+			if err != nil {
+				gr.Close()
+				return nil, false, err
+			}
+
+			if m, ok := msg.Msg.(EndHeightMessage); ok {
+				if m.Height == height { // found
+					wal.Logger.Debug("Found", "height", height, "index", index)
+					return gr, true, nil
+				}
+			}
+		}
+		gr.Close()
+	}
+
+	return nil, false, nil
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+// A WALEncoder writes custom-encoded WAL messages to an output stream.
+//
+// Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value (go-wire encoded)
+type WALEncoder struct {
+	wr io.Writer
+}
+
+// NewWALEncoder returns a new encoder that writes to wr.
+func NewWALEncoder(wr io.Writer) *WALEncoder {
+	return &WALEncoder{wr}
+}
+
+// Encode writes the custom encoding of v to the stream.
+func (enc *WALEncoder) Encode(v *TimedWALMessage) error {
+	data := wire.BinaryBytes(v)
+
+	crc := crc32.Checksum(data, crc32c)
+	length := uint32(len(data))
+	totalLength := 8 + int(length)
+
+	msg := make([]byte, totalLength)
+	binary.BigEndian.PutUint32(msg[0:4], crc)
+	binary.BigEndian.PutUint32(msg[4:8], length)
+	copy(msg[8:], data)
+
+	_, err := enc.wr.Write(msg)
+
+	return err
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+// A WALDecoder reads and decodes custom-encoded WAL messages from an input
+// stream. See WALEncoder for the format used.
+//
+// It will also compare the checksums and make sure data size is equal to the
+// length from the header. If that is not the case, error will be returned.
+type WALDecoder struct {
+	rd io.Reader
+}
+
+// NewWALDecoder returns a new decoder that reads from rd.
+func NewWALDecoder(rd io.Reader) *WALDecoder {
+	return &WALDecoder{rd}
+}
+
+// Decode reads the next custom-encoded value from its reader and returns it.
+func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
+	b := make([]byte, 4)
+
+	n, err := dec.rd.Read(b)
+	if err == io.EOF {
+		return nil, err
+	}
 	if err != nil {
-		PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, wmsg))
+		return nil, fmt.Errorf("failed to read checksum: %v", err)
 	}
-	// TODO: only flush when necessary
-	if err := wal.group.Flush(); err != nil {
-		PanicQ(Fmt("Error flushing consensus wal buf to file. Error: %v \n", err))
+	crc := binary.BigEndian.Uint32(b)
+
+	b = make([]byte, 4)
+	n, err = dec.rd.Read(b)
+	if err == io.EOF {
+		return nil, err
 	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read length: %v", err)
+	}
+	length := binary.BigEndian.Uint32(b)
+
+	data := make([]byte, length)
+	n, err = dec.rd.Read(data)
+	if err == io.EOF {
+		return nil, err
+	}
+	if err != nil {
+		return nil, fmt.Errorf("not enough bytes for data: %v (want: %d, read: %v)", err, length, n)
+	}
+
+	// check checksum before decoding data
+	actualCRC := crc32.Checksum(data, crc32c)
+	if actualCRC != crc {
+		return nil, fmt.Errorf("checksums do not match: (read: %v, actual: %v)", crc, actualCRC)
+	}
+
+	var nn int
+	var res *TimedWALMessage // nolint: gosimple
+	res = wire.ReadBinary(&TimedWALMessage{}, bytes.NewBuffer(data), int(length), &nn, &err).(*TimedWALMessage)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode data: %v", err)
+	}
+
+	return res, err
 }
 
-func (wal *WAL) writeEndHeight(height int) {
-	wal.group.WriteLine(Fmt("#ENDHEIGHT: %v", height))
+type nilWAL struct{}
 
-	// TODO: only flush when necessary
-	if err := wal.group.Flush(); err != nil {
-		PanicQ(Fmt("Error flushing consensus wal buf to file. Error: %v \n", err))
-	}
+func (nilWAL) Save(m WALMessage)  {}
+func (nilWAL) Group() *auto.Group { return nil }
+func (nilWAL) SearchForEndHeight(height int64) (gr *auto.GroupReader, found bool, err error) {
+	return nil, false, nil
 }
+func (nilWAL) Start() error { return nil }
+func (nilWAL) Stop() error  { return nil }
+func (nilWAL) Wait()        {}

consensus/wal_generator.go

@@ -0,0 +1,181 @@
+package consensus
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/tendermint/abci/example/dummy"
+	bc "github.com/tendermint/tendermint/blockchain"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+	auto "github.com/tendermint/tmlibs/autofile"
+	"github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+)
+
+// WALWithNBlocks generates a consensus WAL. It does this by spining up a
+// stripped down version of node (proxy app, event bus, consensus state) with a
+// persistent dummy application and special consensus wal instance
+// (byteBufferWAL) and waits until numBlocks are created. Then it returns a WAL
+// content.
+func WALWithNBlocks(numBlocks int) (data []byte, err error) {
+	config := getConfig()
+
+	app := dummy.NewPersistentDummyApplication(filepath.Join(config.DBDir(), "wal_generator"))
+
+	logger := log.NewNopLogger() // log.TestingLogger().With("wal_generator", "wal_generator")
+
+	/////////////////////////////////////////////////////////////////////////////
+	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
+	// NOTE: we can't import node package because of circular dependency
+	privValidatorFile := config.PrivValidatorFile()
+	privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile)
+	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read genesis file")
+	}
+	stateDB := db.NewMemDB()
+	blockStoreDB := db.NewMemDB()
+	state, err := sm.MakeGenesisState(stateDB, genDoc)
+	state.SetLogger(logger.With("module", "state"))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make genesis state")
+	}
+	blockStore := bc.NewBlockStore(blockStoreDB)
+	handshaker := NewHandshaker(state, blockStore)
+	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), handshaker)
+	proxyApp.SetLogger(logger.With("module", "proxy"))
+	if err := proxyApp.Start(); err != nil {
+		return nil, errors.Wrap(err, "failed to start proxy app connections")
+	}
+	defer proxyApp.Stop()
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(logger.With("module", "events"))
+	if err := eventBus.Start(); err != nil {
+		return nil, errors.Wrap(err, "failed to start event bus")
+	}
+	mempool := types.MockMempool{}
+	consensusState := NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
+	consensusState.SetLogger(logger)
+	consensusState.SetEventBus(eventBus)
+	if privValidator != nil {
+		consensusState.SetPrivValidator(privValidator)
+	}
+	// END OF COPY PASTE
+	/////////////////////////////////////////////////////////////////////////////
+
+	// set consensus wal to buffered WAL, which will write all incoming msgs to buffer
+	var b bytes.Buffer
+	wr := bufio.NewWriter(&b)
+	numBlocksWritten := make(chan struct{})
+	wal := &byteBufferWAL{enc: NewWALEncoder(wr), heightToStop: int64(numBlocks), signalWhenStopsTo: numBlocksWritten}
+	// see wal.go#103
+	wal.Save(EndHeightMessage{0})
+	consensusState.wal = wal
+
+	if err := consensusState.Start(); err != nil {
+		return nil, errors.Wrap(err, "failed to start consensus state")
+	}
+	defer consensusState.Stop()
+
+	select {
+	case <-numBlocksWritten:
+		wr.Flush()
+		return b.Bytes(), nil
+	case <-time.After(time.Duration(5*numBlocks) * time.Second):
+		return b.Bytes(), fmt.Errorf("waited too long for tendermint to produce %d blocks", numBlocks)
+	}
+}
+
+// f**ing long, but unique for each test
+func makePathname() string {
+	// get path
+	p, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	// fmt.Println(p)
+	sep := string(filepath.Separator)
+	return strings.Replace(p, sep, "_", -1)
+}
+
+func randPort() int {
+	// returns between base and base + spread
+	base, spread := 20000, 20000
+	return base + rand.Intn(spread)
+}
+
+func makeAddrs() (string, string, string) {
+	start := randPort()
+	return fmt.Sprintf("tcp://0.0.0.0:%d", start),
+		fmt.Sprintf("tcp://0.0.0.0:%d", start+1),
+		fmt.Sprintf("tcp://0.0.0.0:%d", start+2)
+}
+
+// getConfig returns a config for test cases
+func getConfig() *cfg.Config {
+	pathname := makePathname()
+	c := cfg.ResetTestRoot(pathname)
+
+	// and we use random ports to run in parallel
+	tm, rpc, grpc := makeAddrs()
+	c.P2P.ListenAddress = tm
+	c.RPC.ListenAddress = rpc
+	c.RPC.GRPCListenAddress = grpc
+	return c
+}
+
+// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops
+// when the heightToStop is reached. Client will be notified via
+// signalWhenStopsTo channel.
+type byteBufferWAL struct {
+	enc               *WALEncoder
+	stopped           bool
+	heightToStop      int64
+	signalWhenStopsTo chan struct{}
+}
+
+// needed for determinism
+var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")
+
+// Save writes message to the internal buffer except when heightToStop is
+// reached, in which case it will signal the caller via signalWhenStopsTo and
+// skip writing.
+func (w *byteBufferWAL) Save(m WALMessage) {
+	if w.stopped {
+		return
+	}
+
+	if endMsg, ok := m.(EndHeightMessage); ok {
+		if endMsg.Height == w.heightToStop {
+			w.signalWhenStopsTo <- struct{}{}
+			w.stopped = true
+			return
+		}
+	}
+
+	err := w.enc.Encode(&TimedWALMessage{fixedTime, m})
+	if err != nil {
+		panic(fmt.Sprintf("failed to encode the msg %v", m))
+	}
+}
+
+func (w *byteBufferWAL) Group() *auto.Group {
+	panic("not implemented")
+}
+func (w *byteBufferWAL) SearchForEndHeight(height int64) (gr *auto.GroupReader, found bool, err error) {
+	return nil, false, nil
+}
+
+func (w *byteBufferWAL) Start() error { return nil }
+func (w *byteBufferWAL) Stop() error  { return nil }
+func (w *byteBufferWAL) Wait()        {}

consensus/wal_test.go

@@ -0,0 +1,132 @@
+package consensus
+
+import (
+	"bytes"
+	"crypto/rand"
+	"sync"
+	"testing"
+	"time"
+
+	wire "github.com/tendermint/go-wire"
+	"github.com/tendermint/tendermint/consensus/types"
+	tmtypes "github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWALEncoderDecoder(t *testing.T) {
+	now := time.Now()
+	msgs := []TimedWALMessage{
+		TimedWALMessage{Time: now, Msg: EndHeightMessage{0}},
+		TimedWALMessage{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
+	}
+
+	b := new(bytes.Buffer)
+
+	for _, msg := range msgs {
+		b.Reset()
+
+		enc := NewWALEncoder(b)
+		err := enc.Encode(&msg)
+		require.NoError(t, err)
+
+		dec := NewWALDecoder(b)
+		decoded, err := dec.Decode()
+		require.NoError(t, err)
+
+		assert.Equal(t, msg.Time.Truncate(time.Millisecond), decoded.Time)
+		assert.Equal(t, msg.Msg, decoded.Msg)
+	}
+}
+
+func TestSearchForEndHeight(t *testing.T) {
+	walBody, err := WALWithNBlocks(6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	walFile := tempWALWithData(walBody)
+
+	wal, err := NewWAL(walFile, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	h := int64(3)
+	gr, found, err := wal.SearchForEndHeight(h)
+	assert.NoError(t, err, cmn.Fmt("expected not to err on height %d", h))
+	assert.True(t, found, cmn.Fmt("expected to find end height for %d", h))
+	assert.NotNil(t, gr, "expected group not to be nil")
+	defer gr.Close()
+
+	dec := NewWALDecoder(gr)
+	msg, err := dec.Decode()
+	assert.NoError(t, err, "expected to decode a message")
+	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
+	assert.True(t, ok, "expected message of type EventDataRoundState")
+	assert.Equal(t, rs.Height, h+1, cmn.Fmt("wrong height"))
+}
+
+var initOnce sync.Once
+
+func registerInterfacesOnce() {
+	initOnce.Do(func() {
+		var _ = wire.RegisterInterface(
+			struct{ WALMessage }{},
+			wire.ConcreteType{[]byte{}, 0x10},
+		)
+	})
+}
+
+func nBytes(n int) []byte {
+	buf := make([]byte, n)
+	n, _ = rand.Read(buf)
+	return buf[:n]
+}
+
+func benchmarkWalDecode(b *testing.B, n int) {
+	registerInterfacesOnce()
+
+	buf := new(bytes.Buffer)
+	enc := NewWALEncoder(buf)
+
+	data := nBytes(n)
+	enc.Encode(&TimedWALMessage{Msg: data, Time: time.Now().Round(time.Second)})
+
+	encoded := buf.Bytes()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		buf.Write(encoded)
+		dec := NewWALDecoder(buf)
+		if _, err := dec.Decode(); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.ReportAllocs()
+}
+
+func BenchmarkWalDecode512B(b *testing.B) {
+	benchmarkWalDecode(b, 512)
+}
+
+func BenchmarkWalDecode10KB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024)
+}
+func BenchmarkWalDecode100KB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024)
+}
+func BenchmarkWalDecode1MB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024)
+}
+func BenchmarkWalDecode10MB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024*1024)
+}
+func BenchmarkWalDecode100MB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024*1024)
+}
+func BenchmarkWalDecode1GB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024*1024)
+}

docs/_static/custom_collapsible_code.css

@@ -0,0 +1,17 @@
+.toggle {
+    padding-bottom: 1em ;
+}
+
+.toggle .header {
+    display: block;
+    clear: both;
+    cursor: pointer;
+}
+
+.toggle .header:after {
+    content: " ▼";
+}
+
+.toggle .header.open:after {
+    content: " ▲";
+}

docs/_static/custom_collapsible_code.js

@@ -0,0 +1,10 @@
+let makeCodeBlocksCollapsible = function() {
+    $(".toggle > *").hide();
+    $(".toggle .header").show();
+    $(".toggle .header").click(function() {
+        $(this).parent().children().not(".header").toggle({"duration": 400});
+        $(this).parent().children(".header").toggleClass("open");
+    });
+};
+// we could use the }(); way if we would have access to jQuery in HEAD, i.e. we would need to force the theme
+// to load jQuery before our custom scripts

docs/_templates/layout.html

@@ -0,0 +1,20 @@
+{% extends "!layout.html" %}
+
+{% set css_files = css_files + ["_static/custom_collapsible_code.css"] %}
+
+# sadly, I didn't find a css style way to add custom JS to a list that is automagically added to head like CSS (above) #}
+{% block extrahead %}
+<script type="text/javascript" src="_static/custom_collapsible_code.js"></script>
+{% endblock %}
+
+{% block footer %}
+<script type="text/javascript">
+    $(document).ready(function() {
+        // using this approach as we don't have access to the jQuery selectors
+        // when executing the function on load in HEAD
+        makeCodeBlocksCollapsible();
+    });
+</script>
+{% endblock %}
+
+

docs/abci-cli.rst

@@ -14,7 +14,7 @@ Next, install the ``abci-cli`` tool and example applications:
 
 ::
 
-    go get -u github.com/tendermint/abci/cmd/...
+    go get -u github.com/tendermint/abci/cmd/abci-cli
 
 If this fails, you may need to use ``glide`` to get vendored
 dependencies:
@@ -24,27 +24,37 @@ dependencies:
     go get github.com/Masterminds/glide
     cd $GOPATH/src/github.com/tendermint/abci
     glide install
-    go install ./cmd/...
+    go install ./cmd/abci-cli
 
-Now run ``abci-cli --help`` to see the list of commands:
+Now run ``abci-cli`` to see the list of commands:
 
 ::
 
-    COMMANDS:
-       batch        Run a batch of ABCI commands against an application
-       console      Start an interactive console for multiple commands
-       echo         Have the application echo a message
-       info         Get some info about the application
-       set_option   Set an option on the application
-       deliver_tx    Append a new tx to application
-       check_tx     Validate a tx
-       commit       Get application Merkle root hash
-       help, h      Shows a list of commands or help for one command
+    Usage:
+      abci-cli [command]
+
+    Available Commands:
+      batch       Run a batch of abci commands against an application
+      check_tx    Validate a tx
+      commit      Commit the application state and return the Merkle root hash
+      console     Start an interactive abci console for multiple commands
+      counter     ABCI demo example
+      deliver_tx  Deliver a new tx to the application
+      dummy       ABCI demo example
+      echo        Have the application echo a message
+      help        Help about any command
+      info        Get some info about the application
+      query       Query the application state
+      set_option  Set an options on the application
+
+    Flags:
+          --abci string      socket or grpc (default "socket")
+          --address string   address of application socket (default "tcp://127.0.0.1:46658")
+      -h, --help             help for abci-cli
+      -v, --verbose          print the command and results as if it were a console session
+
+                                          Use "abci-cli [command] --help" for more information about a command.
 
-    GLOBAL OPTIONS:
-       --address "tcp://127.0.0.1:46658"    address of application socket
-       --help, -h                           show help
-       --version, -v                        print the version
 
 Dummy - First Example
 ---------------------
@@ -61,7 +71,7 @@ Let's start a dummy application, which was installed at the same time as
 
 ::
 
-    dummy
+    abci-cli dummy
 
 In another terminal, run
 
@@ -70,8 +80,19 @@ In another terminal, run
     abci-cli echo hello
     abci-cli info
 
-The application should echo ``hello`` and give you some information
-about itself.
+You'll see something like:
+
+::
+
+    -> data: hello
+    -> data.hex: 68656C6C6F
+
+and:
+
+::
+
+    -> data: {"size":0}
+    -> data.hex: 7B2273697A65223A307D
 
 An ABCI application must provide two things:
 
@@ -86,7 +107,7 @@ The server may be generic for a particular language, and we provide a
 `reference implementation in
 Golang <https://github.com/tendermint/abci/tree/master/server>`__. See
 the `list of other ABCI
-implementations <https://tendermint.com/ecosystem>`__ for servers in
+implementations <./ecosystem.html>`__ for servers in
 other languages.
 
 The handler is specific to the application, and may be arbitrary, so
@@ -109,36 +130,50 @@ Try running these commands:
 ::
 
     > echo hello
+    -> code: OK
     -> data: hello
-
+    -> data.hex: 0x68656C6C6F
+    
     > info
+    -> code: OK
     -> data: {"size":0}
-
+    -> data.hex: 0x7B2273697A65223A307D
+    
     > commit
-    -> data: 0x
-
+    -> code: OK
+    
     > deliver_tx "abc"
     -> code: OK
-
+    
     > info
+    -> code: OK
     -> data: {"size":1}
-
+    -> data.hex: 0x7B2273697A65223A317D
+    
     > commit
-    -> data: 0x750502FC7E84BBD788ED589624F06CFA871845D1
-
+    -> code: OK
+    -> data.hex: 0x49DFD15CCDACDEAE9728CB01FBB5E8688CA58B91
+    
     > query "abc"
     -> code: OK
-    -> data: {"index":0,"value":"abc","exists":true}
-
+    -> log: exists
+    -> height: 0
+    -> value: abc
+    -> value.hex: 616263
+    
     > deliver_tx "def=xyz"
     -> code: OK
-
+    
     > commit
-    -> data: 0x76393B8A182E450286B0694C629ECB51B286EFD5
-
+    -> code: OK
+    -> data.hex: 0x70102DB32280373FBF3F9F89DA2A20CE2CD62B0B
+    
     > query "def"
     -> code: OK
-    -> data: {"index":1,"value":"xyz","exists":true}
+    -> log: exists
+    -> height: 0
+    -> value: xyz
+    -> value.hex: 78797A
 
 Note that if we do ``deliver_tx "abc"`` it will store ``(abc, abc)``,
 but if we do ``deliver_tx "abc=efg"`` it will store ``(abc, efg)``.
@@ -181,37 +216,39 @@ app:
 
 ::
 
-    counter
+    abci-cli counter
 
 In another window, start the ``abci-cli console``:
 
 ::
 
     > set_option serial on
-    -> data: serial=on
-
+    -> code: OK
+    
     > check_tx 0x00
     -> code: OK
-
+    
     > check_tx 0xff
     -> code: OK
-
+    
     > deliver_tx 0x00
     -> code: OK
-
+    
     > check_tx 0x00
     -> code: BadNonce
     -> log: Invalid nonce. Expected >= 1, got 0
-
+    
     > deliver_tx 0x01
     -> code: OK
-
+    
     > deliver_tx 0x04
     -> code: BadNonce
     -> log: Invalid nonce. Expected 2, got 4
-
+    
     > info
+    -> code: OK
     -> data: {"hashes":0,"txs":2}
+    -> data.hex: 0x7B22686173686573223A302C22747873223A327D
 
 This is a very simple application, but between ``counter`` and
 ``dummy``, its easy to see how you can build out arbitrary application

docs/app-development.rst

@@ -142,6 +142,13 @@ It is unlikely that you will need to implement a client. For details of
 our client, see
 `here <https://github.com/tendermint/abci/tree/master/client>`__.
 
+Most of the examples below are from `dummy application
+<https://github.com/tendermint/abci/blob/master/example/dummy/dummy.go>`__,
+which is a part of the abci repo. `persistent_dummy application
+<https://github.com/tendermint/abci/blob/master/example/dummy/persistent_dummy.go>`__
+is used to show ``BeginBlock``, ``EndBlock`` and ``InitChain``
+example implementations.
+
 Blockchain Protocol
 -------------------
 
@@ -187,6 +194,38 @@ through all transactions in the mempool, removing any that were included
 in the block, and re-run the rest using CheckTx against the post-Commit
 mempool state.
 
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        func (app *DummyApplication) CheckTx(tx []byte) types.Result {
+          return types.OK
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        ResponseCheckTx requestCheckTx(RequestCheckTx req) {
+            byte[] transaction = req.getTx().toByteArray();
+
+            // validate transaction
+
+            if (notValid) {
+                return ResponseCheckTx.newBuilder().setCode(CodeType.BadNonce).setLog("invalid tx").build();
+            } else {
+                return ResponseCheckTx.newBuilder().setCode(CodeType.OK).build();
+            }
+        }
+
 Consensus Connection
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -215,6 +254,49 @@ The block header will be updated (TODO) to include some commitment to
 the results of DeliverTx, be it a bitarray of non-OK transactions, or a
 merkle root of the data returned by the DeliverTx requests, or both.
 
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        // tx is either "key=value" or just arbitrary bytes
+        func (app *DummyApplication) DeliverTx(tx []byte) types.Result {
+          parts := strings.Split(string(tx), "=")
+          if len(parts) == 2 {
+            app.state.Set([]byte(parts[0]), []byte(parts[1]))
+          } else {
+            app.state.Set(tx, tx)
+          }
+          return types.OK
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        /**
+         * Using Protobuf types from the protoc compiler, we always start with a byte[]
+         */
+        ResponseDeliverTx deliverTx(RequestDeliverTx request) {
+            byte[] transaction  = request.getTx().toByteArray();
+
+            // validate your transaction
+
+            if (notValid) {
+                return ResponseDeliverTx.newBuilder().setCode(CodeType.BadNonce).setLog("transaction was invalid").build();
+            } else {
+                ResponseDeliverTx.newBuilder().setCode(CodeType.OK).build();
+            }
+
+        }
+
 Commit
 ^^^^^^
 
@@ -228,7 +310,7 @@ Commit, or there will be deadlock. Note also that all remaining
 transactions in the mempool are replayed on the mempool connection
 (CheckTx) following a commit.
 
-The Commit response includes a byte array, which is the deterministic
+The app should respond to the Commit request with a byte array, which is the deterministic
 state root of the application. It is included in the header of the next
 block. It can be used to provide easily verified Merkle-proofs of the
 state of the application.
@@ -237,6 +319,36 @@ It is expected that the app will persist state to disk on Commit. The
 option to have all transactions replayed from some previous block is the
 job of the `Handshake <#handshake>`__.
 
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        func (app *DummyApplication) Commit() types.Result {
+          hash := app.state.Hash()
+          return types.NewResultOK(hash, "")
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        ResponseCommit requestCommit(RequestCommit requestCommit) {
+
+            // update the internal app-state
+            byte[] newAppState = calculateAppState();
+
+            // and return it to the node
+            return ResponseCommit.newBuilder().setCode(CodeType.OK).setData(ByteString.copyFrom(newAppState)).build();
+        }
+
 BeginBlock
 ^^^^^^^^^^
 
@@ -248,6 +360,46 @@ The app should remember the latest height and header (ie. from which it
 has run a successful Commit) so that it can tell Tendermint where to
 pick up from when it restarts. See information on the Handshake, below.
 
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        // Track the block hash and header information
+        func (app *PersistentDummyApplication) BeginBlock(params types.RequestBeginBlock) {
+          // update latest block info
+          app.blockHeader = params.Header
+
+          // reset valset changes
+          app.changes = make([]*types.Validator, 0)
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        /*
+         * all types come from protobuf definition
+         */
+        ResponseBeginBlock requestBeginBlock(RequestBeginBlock req) {
+
+            Header header = req.getHeader();
+            byte[] prevAppHash = header.getAppHash().toByteArray();
+            long prevHeight = header.getHeight();
+            long numTxs = header.getNumTxs();
+
+            // run your pre-block logic. Maybe prepare a state snapshot, message components, etc
+
+            return ResponseBeginBlock.newBuilder().build();
+        }
+
 EndBlock
 ^^^^^^^^
 
@@ -260,6 +412,40 @@ EndBlock response. To remove one, include it in the list with a
 validator set. Note validator set changes are only available in v0.8.0
 and up.
 
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        // Update the validator set
+        func (app *PersistentDummyApplication) EndBlock(height uint64) (resEndBlock types.ResponseEndBlock) {
+          return types.ResponseEndBlock{Diffs: app.changes}
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        /*
+         * Assume that one validator changes. The new validator has a power of 10
+         */
+        ResponseEndBlock requestEndBlock(RequestEndBlock req) {
+            final long currentHeight = req.getHeight();
+            final byte[] validatorPubKey = getValPubKey();
+
+            ResponseEndBlock.Builder builder = ResponseEndBlock.newBuilder();
+            builder.addDiffs(1, Types.Validator.newBuilder().setPower(10L).setPubKey(ByteString.copyFrom(validatorPubKey)).build());
+
+            return builder.build();
+        }
+
 Query Connection
 ~~~~~~~~~~~~~~~~
 
@@ -281,6 +467,73 @@ cause Tendermint to not connect to the corresponding peer:
 
 Note: these query formats are subject to change!
 
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        func (app *DummyApplication) Query(reqQuery types.RequestQuery) (resQuery types.ResponseQuery) {
+          if reqQuery.Prove {
+            value, proof, exists := app.state.Proof(reqQuery.Data)
+            resQuery.Index = -1 // TODO make Proof return index
+            resQuery.Key = reqQuery.Data
+            resQuery.Value = value
+            resQuery.Proof = proof
+            if exists {
+              resQuery.Log = "exists"
+            } else {
+              resQuery.Log = "does not exist"
+            }
+            return
+          } else {
+            index, value, exists := app.state.Get(reqQuery.Data)
+            resQuery.Index = int64(index)
+            resQuery.Value = value
+            if exists {
+              resQuery.Log = "exists"
+            } else {
+              resQuery.Log = "does not exist"
+            }
+            return
+          }
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        ResponseQuery requestQuery(RequestQuery req) {
+            final boolean isProveQuery = req.getProve();
+            final ResponseQuery.Builder responseBuilder = ResponseQuery.newBuilder();
+
+            if (isProveQuery) {
+                com.app.example.ProofResult proofResult = generateProof(req.getData().toByteArray());
+                final byte[] proofAsByteArray = proofResult.getAsByteArray();
+
+                responseBuilder.setProof(ByteString.copyFrom(proofAsByteArray));
+                responseBuilder.setKey(req.getData());
+                responseBuilder.setValue(ByteString.copyFrom(proofResult.getData()));
+                responseBuilder.setLog(result.getLogValue());
+            } else {
+                byte[] queryData = req.getData().toByteArray();
+
+                final com.app.example.QueryResult result = generateQueryResult(queryData);
+
+                responseBuilder.setIndex(result.getIndex());
+                responseBuilder.setValue(ByteString.copyFrom(result.getValue()));
+                responseBuilder.setLog(result.getLogValue());
+            }
+
+            return responseBuilder.build();
+        }
+
 Handshake
 ~~~~~~~~~
 
@@ -288,7 +541,7 @@ When the app or tendermint restarts, they need to sync to a common
 height. When an ABCI connection is first established, Tendermint will
 call ``Info`` on the Query connection. The response should contain the
 LastBlockHeight and LastBlockAppHash - the former is the last block for
-the which the app ran Commit successfully, the latter is the response
+which the app ran Commit successfully, the latter is the response
 from that Commit.
 
 Using this information, Tendermint will determine what needs to be
@@ -297,3 +550,79 @@ the app are synced to the latest block height.
 
 If the app returns a LastBlockHeight of 0, Tendermint will just replay
 all blocks.
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        func (app *DummyApplication) Info(req types.RequestInfo) (resInfo types.ResponseInfo) {
+          return types.ResponseInfo{Data: cmn.Fmt("{\"size\":%v}", app.state.Size())}
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        ResponseInfo requestInfo(RequestInfo req) {
+            final byte[] lastAppHash = getLastAppHash();
+            final long lastHeight = getLastHeight();
+            return ResponseInfo.newBuilder().setLastBlockAppHash(ByteString.copyFrom(lastAppHash)).setLastBlockHeight(lastHeight).build();
+        }
+
+Genesis
+~~~~~~~
+
+``InitChain`` will be called once upon the genesis. ``params`` includes the
+initial validator set. Later on, it may be extended to take parts of the
+consensus params.
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Go Example**
+
+    .. code-block:: go
+
+        // Save the validators in the merkle tree
+        func (app *PersistentDummyApplication) InitChain(params types.RequestInitChain) {
+          for _, v := range params.Validators {
+            r := app.updateValidator(v)
+            if r.IsErr() {
+              app.logger.Error("Error updating validators", "r", r)
+            }
+          }
+        }
+
+.. container:: toggle
+
+    .. container:: header
+
+        **Show/Hide Java Example**
+
+    .. code-block:: java
+
+        /*
+         * all types come from protobuf definition
+         */
+        ResponseInitChain requestInitChain(RequestInitChain req) {
+            final int validatorsCount = req.getValidatorsCount();
+            final List<Types.Validator> validatorsList = req.getValidatorsList();
+
+            validatorsList.forEach((validator) -> {
+                long power = validator.getPower();
+                byte[] validatorPubKey = validator.getPubKey().toByteArray();
+
+                // do somehing for validator setup in app
+            });
+
+            return ResponseInitChain.newBuilder().build();
+        }

docs/architecture/adr-006-trust-metric.md

@@ -0,0 +1,238 @@
+# ADR 006: Trust Metric Design
+
+## Context
+
+The proposed trust metric will allow Tendermint to maintain local trust rankings for peers it has directly interacted with, which can then be used to implement soft security controls. The calculations were obtained from the [TrustGuard](https://dl.acm.org/citation.cfm?id=1060808) project.
+
+### Background
+
+The Tendermint Core project developers would like to improve Tendermint security and reliability by keeping track of the level of trustworthiness peers have demonstrated within the peer-to-peer network. This way, undesirable outcomes from peers will not immediately result in them being dropped from the network (potentially causing drastic changes to take place). Instead, peers behavior can be monitored with appropriate metrics and be removed from the network once Tendermint Core is certain the peer is a threat. For example, when the PEXReactor makes a request for peers network addresses from a already known peer, and the returned network addresses are unreachable, this untrustworthy behavior should be tracked. Returning a few bad network addresses probably shouldn’t cause a peer to be dropped, while excessive amounts of this behavior does qualify the peer being dropped.
+
+Trust metrics can be circumvented by malicious nodes through the use of strategic oscillation techniques, which adapts the malicious node’s behavior pattern in order to maximize its goals. For instance, if the malicious node learns that the time interval of the Tendermint trust metric is *X* hours, then it could wait *X* hours in-between malicious activities. We could try to combat this issue by increasing the interval length, yet this will make the system less adaptive to recent events.
+
+Instead, having shorter intervals, but keeping a history of interval values, will give our metric the flexibility needed in order to keep the network stable, while also making it resilient against a strategic malicious node in the Tendermint peer-to-peer network. Also, the metric can access trust data over a rather long period of time while not greatly increasing its history size by aggregating older history values over a larger number of intervals, and at the same time, maintain great precision for the recent intervals. This approach is referred to as fading memories, and closely resembles the way human beings remember their experiences. The trade-off to using history data is that the interval values should be preserved in-between executions of the node.
+
+### References
+
+S. Mudhakar, L. Xiong, and L. Liu, “TrustGuard: Countering Vulnerabilities in Reputation Management for Decentralized Overlay Networks,” in *Proceedings of the 14th international conference on World Wide Web, pp. 422-431*, May 2005.
+
+## Decision
+
+The proposed trust metric will allow a developer to inform the trust metric store of all good and bad events relevant to a peer's behavior, and at any time, the metric can be queried for a peer's current trust ranking.
+
+The three subsections below will cover the process being considered for calculating the trust ranking, the concept of the trust metric store, and the interface for the trust metric.
+
+### Proposed Process
+
+The proposed trust metric will count good and bad events relevant to the object, and calculate the percent of counters that are good over an interval with a predefined duration. This is the procedure that will continue for the life of the trust metric. When the trust metric is queried for the current **trust value**, a resilient equation will be utilized to perform the calculation.
+
+The equation being proposed resembles a Proportional-Integral-Derivative (PID) controller used in control systems. The proportional component allows us to be sensitive to the value of the most recent interval, while the integral component allows us to incorporate trust values stored in the history data, and the derivative component allows us to give weight to sudden changes in the behavior of a peer. We compute the trust value of a peer in interval i based on its current trust ranking, its trust rating history prior to interval *i* (over the past *maxH* number of intervals) and its trust ranking fluctuation. We will break up the equation into the three components.
+
+```math
+(1) Proportional Value = a * R[i]
+```
+
+where *R*[*i*] denotes the raw trust value at time interval *i* (where *i* == 0 being current time) and *a* is the weight applied to the contribution of the current reports. The next component of our equation uses a weighted sum over the last *maxH* intervals to calculate the history value for time *i*:
+ 
+
+`H[i] = ` ![formula1](img/formula1.png "Weighted Sum Formula")
+
+
+The weights can be chosen either optimistically or pessimistically. An optimistic weight creates larger weights for newer history data values, while the the pessimistic weight creates larger weights for time intervals with lower scores. The default weights used during the calculation of the history value are optimistic and calculated as *Wk* = 0.8^*k*, for time interval *k*. With the history value available, we can now finish calculating the integral value:
+
+```math
+(2) Integral Value = b * H[i]
+```
+
+Where *H*[*i*] denotes the history value at time interval *i* and *b* is the weight applied to the contribution of past performance for the object being measured. The derivative component will be calculated as follows:
+
+```math
+D[i] = R[i] – H[i]
+
+(3) Derivative Value = c(D[i]) * D[i]
+```
+
+Where the value of *c* is selected based on the *D*[*i*] value relative to zero. The default selection process makes *c* equal to 0 unless *D*[*i*] is a negative value, in which case c is equal to 1. The result is that the maximum penalty is applied when current behavior is lower than previously experienced behavior. If the current behavior is better than the previously experienced behavior, then the Derivative Value has no impact on the trust value. With the three components brought together, our trust value equation is calculated as follows:
+
+```math
+TrustValue[i] = a * R[i] + b * H[i] + c(D[i]) * D[i]
+```
+
+As a performance optimization that will keep the amount of raw interval data being saved to a reasonable size of *m*, while allowing us to represent 2^*m* - 1 history intervals, we can employ the fading memories technique that will trade space and time complexity for the precision of the history data values by summarizing larger quantities of less recent values. While our equation above attempts to access up to *maxH* (which can be 2^*m* - 1), we will map those requests down to *m* values using equation 4 below:
+
+```math
+(4) j = index, where index > 0
+```
+
+Where *j* is one of *(0, 1, 2, … , m – 1)* indices used to access history interval data. Now we can access the raw intervals using the following calculations:
+
+```math
+R[0] = raw data for current time interval
+```
+
+`R[j] = ` ![formula2](img/formula2.png "Fading Memories Formula")
+
+### Trust Metric Store
+
+Similar to the P2P subsystem AddrBook, the trust metric store will maintain information relevant to Tendermint peers. Additionally, the trust metric store will ensure that trust metrics will only be active for peers that a node is currently and directly engaged with.
+
+Reactors will provide a peer key to the trust metric store in order to retrieve the associated trust metric. The trust metric can then record new positive and negative events experienced by the reactor, as well as provided the current trust score calculated by the metric.
+
+When the node is shutting down, the trust metric store will save history data for trust metrics associated with all known peers. This saved information allows experiences with a peer to be preserved across node executions, which can span a tracking windows of days or weeks. The trust history data is loaded automatically during OnStart.
+
+### Interface Detailed Design
+
+Each trust metric allows for the recording of positive/negative events, querying the current trust value/score, and the stopping/pausing of tracking over time intervals. This can be seen below:
+
+
+```go
+
+// TrustMetric - keeps track of peer reliability
+type TrustMetric struct {
+    // Private elements.
+}
+
+// Pause tells the metric to pause recording data over time intervals.
+// All method calls that indicate events will unpause the metric
+func (tm *TrustMetric) Pause() {}
+
+// Stop tells the metric to stop recording data over time intervals
+func (tm *TrustMetric) Stop() {}
+
+// BadEvents indicates that an undesirable event(s) took place
+func (tm *TrustMetric) BadEvents(num int) {}
+
+// GoodEvents indicates that a desirable event(s) took place
+func (tm *TrustMetric) GoodEvents(num int) {}
+
+// TrustValue gets the dependable trust value; always between 0 and 1
+func (tm *TrustMetric) TrustValue() float64 {}
+
+// TrustScore gets a score based on the trust value always between 0 and 100
+func (tm *TrustMetric) TrustScore() int {}
+
+// NewMetric returns a trust metric with the default configuration
+func NewMetric() *TrustMetric {}
+
+//------------------------------------------------------------------------------------------------
+// For example
+
+tm := NewMetric()
+
+tm.BadEvents(1)
+score := tm.TrustScore()
+
+tm.Stop()
+
+```
+
+Some of the trust metric parameters can be configured. The weight values should probably be left alone in more cases, yet the time durations for the tracking window and individual time interval should be considered.
+
+```go
+
+// TrustMetricConfig - Configures the weight functions and time intervals for the metric
+type TrustMetricConfig struct {
+    // Determines the percentage given to current behavior
+    ProportionalWeight float64
+
+    // Determines the percentage given to prior behavior
+    IntegralWeight float64
+
+    // The window of time that the trust metric will track events across.
+    // This can be set to cover many days without issue
+    TrackingWindow time.Duration
+
+    // Each interval should be short for adapability.
+    // Less than 30 seconds is too sensitive,
+    // and greater than 5 minutes will make the metric numb
+    IntervalLength time.Duration
+}
+
+// DefaultConfig returns a config with values that have been tested and produce desirable results
+func DefaultConfig() TrustMetricConfig {}
+
+// NewMetricWithConfig returns a trust metric with a custom configuration
+func NewMetricWithConfig(tmc TrustMetricConfig) *TrustMetric {}
+
+//------------------------------------------------------------------------------------------------
+// For example
+
+config := TrustMetricConfig{
+    TrackingWindow: time.Minute * 60 * 24, // one day
+    IntervalLength:    time.Minute * 2,
+}
+
+tm := NewMetricWithConfig(config)
+
+tm.BadEvents(10)
+tm.Pause() 
+tm.GoodEvents(1) // becomes active again
+
+```
+
+A trust metric store should be created with a DB that has persistent storage so it can save history data across node executions. All trust metrics instantiated by the store will be created with the provided TrustMetricConfig configuration. 
+
+When you attempt to fetch the trust metric for a peer, and an entry does not exist in the trust metric store, a new metric is automatically created and the entry made within the store.
+
+In additional to the fetching method, GetPeerTrustMetric, the trust metric store provides a method to call when a peer has disconnected from the node. This is so the metric can be paused (history data will not be saved) for periods of time when the node is not having direct experiences with the peer.
+
+```go
+
+// TrustMetricStore - Manages all trust metrics for peers
+type TrustMetricStore struct {
+    cmn.BaseService
+    
+    // Private elements
+}
+
+// OnStart implements Service
+func (tms *TrustMetricStore) OnStart() error {}
+
+// OnStop implements Service
+func (tms *TrustMetricStore) OnStop() {}
+
+// NewTrustMetricStore returns a store that saves data to the DB
+// and uses the config when creating new trust metrics
+func NewTrustMetricStore(db dbm.DB, tmc TrustMetricConfig) *TrustMetricStore {}
+
+// Size returns the number of entries in the trust metric store
+func (tms *TrustMetricStore) Size() int {}
+
+// GetPeerTrustMetric returns a trust metric by peer key
+func (tms *TrustMetricStore) GetPeerTrustMetric(key string) *TrustMetric {}
+
+// PeerDisconnected pauses the trust metric associated with the peer identified by the key
+func (tms *TrustMetricStore) PeerDisconnected(key string) {}
+
+//------------------------------------------------------------------------------------------------
+// For example
+
+db := dbm.NewDB("trusthistory", "goleveldb", dirPathStr)
+tms := NewTrustMetricStore(db, DefaultConfig())
+
+tm := tms.GetPeerTrustMetric(key)
+tm.BadEvents(1)
+
+tms.PeerDisconnected(key)
+
+```
+
+## Status
+
+Approved.
+
+## Consequences
+
+### Positive
+
+- The trust metric will allow Tendermint to make non-binary security and reliability decisions
+- Will help Tendermint implement deterrents that provide soft security controls, yet avoids disruption on the network
+- Will provide useful profiling information when analyzing performance over time related to peer interaction
+
+### Negative
+
+- Requires saving the trust metric history data across node executions
+
+### Neutral
+
+- Keep in mind that, good events need to be recorded just as bad events do using this implementation

docs/architecture/adr-007-trust-metric-usage.md

@@ -0,0 +1,103 @@
+# ADR 007: Trust Metric Usage Guide
+
+## Context
+
+Tendermint is required to monitor peer quality in order to inform its peer dialing and peer exchange strategies.
+
+When a node first connects to the network, it is important that it can quickly find good peers.
+Thus, while a node has fewer connections, it should prioritize connecting to higher quality peers.
+As the node becomes well connected to the rest of the network, it can dial lesser known or lesser
+quality peers and help assess their quality. Similarly, when queried for peers, a node should make
+sure they dont return low quality peers.
+
+Peer quality can be tracked using a trust metric that flags certain behaviours as good or bad. When enough
+bad behaviour accumulates, we can mark the peer as bad and disconnect.
+For example, when the PEXReactor makes a request for peers network addresses from an already known peer, and the returned network addresses are unreachable, this undesirable behavior should be tracked. Returning a few bad network addresses probably shouldn’t cause a peer to be dropped, while excessive amounts of this behavior does qualify the peer for removal. The originally proposed approach and design document for the trust metric can be found in the [ADR 006](adr-006-trust-metric.md) document.
+
+The trust metric implementation allows a developer to obtain a peer's trust metric from a trust metric store, and track good and bad events relevant to a peer's behavior, and at any time, the peer's metric can be queried for a current trust value. The current trust value is calculated with a formula that utilizes current behavior, previous behavior, and change between the two. Current behavior is calculated as the percentage of good behavior within a time interval. The time interval is short; probably set between 30 seconds and 5 minutes. On the other hand, the historic data can estimate a peer's behavior over days worth of tracking. At the end of a time interval, the current behavior becomes part of the historic data, and a new time interval begins with the good and bad counters reset to zero.
+
+These are some important things to keep in mind regarding how the trust metrics handle time intervals and scoring:
+- Each new time interval begins with a perfect score
+- Bad events quickly bring the score down and good events cause the score to slowly rise
+- When the time interval is over, the percentage of good events becomes historic data.
+
+Some useful information about the inner workings of the trust metric:
+- When a trust metric is first instantiated, a timer (ticker) periodically fires in order to handle transitions between trust metric time intervals
+- If a peer is disconnected from a node, the timer should be paused, since the node is no longer connected to that peer
+- The ability to pause the metric is supported with the store **PeerDisconnected** method and the metric **Pause** method
+- After a pause, if a good or bad event method is called on a metric, it automatically becomes unpaused and begins a new time interval.
+
+## Decision
+
+The trust metric capability is now available, yet, it still leaves the question of how should it be applied throughout Tendermint in order to properly track the quality of peers?
+
+### Proposed Process
+
+Peers are managed using an address book and a trust metric:
+
+- The address book keeps a record of peers and provides selection methods
+- The trust metric tracks the quality of the peers
+
+#### Presence in Address Book
+
+Outbound peers are added to the address book before they are dialed,
+and inbound peers are added once the peer connection is set up.
+Peers are also added to the address book when they are received in response to
+a pexRequestMessage.
+
+While a node has less than `needAddressThreshold`, it will periodically request more,
+via pexRequestMessage, from randomly selected peers and from newly dialed outbound peers.
+
+When a new address is added to an address book that has more than `0.5*needAddressThreshold` addresses,
+then with some low probability, a randomly chosen low quality peer is removed.
+
+#### Outbound Peers
+
+Peers attempt to maintain a minimum number of outbound connections by
+repeatedly querying the address book for peers to connect to.
+While a node has few to no outbound connections, the address book is biased to return
+higher quality peers. As the node increases the number of outbound connections,
+the address book is biased to return less-vetted or lower-quality peers.
+
+#### Inbound Peers
+
+Peers also maintain a maximum number of total connections, MaxNumPeers.
+If a peer has MaxNumPeers, new incoming connections will be accepted with low probability.
+When such a new connection is accepted, the peer disconnects from a probabilistically chosen low ranking peer
+so it does not exceed MaxNumPeers.
+
+#### Peer Exchange
+
+When a peer receives a pexRequestMessage, it returns a random sample of high quality peers from the address book. Peers with no score or low score should not be inclided in a response to pexRequestMessage.
+
+#### Peer Quality
+
+Peer quality is tracked in the connection and across the reactors by storing the TrustMetric in the peer's
+thread safe Data store.
+
+Peer behaviour is then defined as one of the following:
+- Fatal - something outright malicious that causes us to disconnect the peer and ban it from the address book for some amount of time
+- Bad - Any kind of timeout, messages that don't unmarshal, fail other validity checks, or messages we didn't ask for or aren't expecting (usually worth one bad event)
+- Neutral - Unknown channels/message types/version upgrades (no good or bad events recorded)
+- Correct - Normal correct behavior (worth one good event)
+- Good - some random majority of peers per reactor sending us useful messages (worth more than one good event).
+
+Note that Fatal behaviour causes us to remove the peer, and neutral behaviour does not affect the score.
+
+## Status
+
+Proposed.
+
+## Consequences
+
+### Positive
+
+- Bringing the address book and trust metric store together will cause the network to be built in a way that encourages greater security and reliability.
+
+### Negative
+
+- TBD
+
+### Neutral
+
+- Keep in mind that, good events need to be recorded just as bad events do using this implementation.

docs/deploy-testnets.rst

@@ -13,7 +13,7 @@ It's relatively easy to setup a Tendermint cluster manually. The only
 requirements for a particular Tendermint node are a private key for the
 validator, stored as ``priv_validator.json``, and a list of the public
 keys of all validators, stored as ``genesis.json``. These files should
-be stored in ``~/.tendermint``, or wherever the ``$TMROOT`` variable
+be stored in ``~/.tendermint``, or wherever the ``$TMHOME`` variable
 might be set to.
 
 Here are the steps to setting up a testnet manually:

docs/ecosystem.rst

@@ -36,55 +36,78 @@ Ethermint
 
 The go-ethereum state machine run as a ABCI app, written in Go, `authored by Tendermint <https://github.com/tendermint/ethermint>`__.
 
+IAVL
+^^^^
 
-Merkle AVL Tree
-^^^^^^^^^^^^^^^
+Immutable AVL+ tree with Merkle proofs, Written in Go, `authored by Tendermint <https://github.com/tendermint/iavl>`__.
 
-The following are implementations of the Tendermint IAVL tree as an ABCI application
-
-Merkleeyes
-~~~~~~~~~~
-
-Written in Go, `authored by Tendermint <https://github.com/tendermint/merkleeyes>`__.
-
-MerkleTree
-~~~~~~~~~~
-
-Written in Java, `authored by jTendermint <https://github.com/jTendermint/MerkleTree>`__.
-
-
-TMChat
+Lotion
 ^^^^^^
 
-P2P chat using Tendermint, written in Java, `authored by woldposd <https://github.com/wolfposd/TMChat>`__.
-      
+A Javascript microframework for building blockchain applications with Tendermint, written in Javascript, `authored by Judd Keppel of Tendermint <https://github.com/keppel/lotion>`__. See also `lotion-chat <https://github.com/keppel/lotion-chat>`__ and `lotion-coin <https://github.com/keppel/lotion-coin>`__ apps written using Lotion.
+
+MerkleTree
+^^^^^^^^^^
+
+Immutable AVL+ tree with Merkle proofs, Written in Java, `authored by jTendermint <https://github.com/jTendermint/MerkleTree>`__.
+
+Passchain
+^^^^^^^^^
+
+Passchain is a tool to securely store and share passwords, tokens and other short secrets, `authored by trusch <https://github.com/trusch/passchain>`__.
+
 Passwerk
 ^^^^^^^^
 
 Encrypted storage web-utility backed by Tendermint, written in Go, `authored by Rigel Rozanski <https://github.com/rigelrozanski/passwerk>`__.
 
+Py-Tendermint
+^^^^^^^^^^^^^
+
+A Python microframework for building blockchain applications with Tendermint, written in Python, `authored by Dave Bryson <https://github.com/davebryson/py-tendermint>`__.
+
+Stratumn
+^^^^^^^^
+
+SDK for "Proof-of-Process" networks, written in Go, `authored by the Stratumn team <https://github.com/stratumn/sdk>`__.
+
+TMChat
+^^^^^^
+
+P2P chat using Tendermint, written in Java, `authored by wolfposd <https://github.com/wolfposd/TMChat>`__.
+      
+
 ABCI Servers
 ------------
 
-
-+-------------------------------------------------------------+--------------------+--------------+
-| **Name**                                                    | **Author**         | **Language** |       
-|                                                             |                    |              |      
-+-------------------------------------------------------------+--------------------+--------------+
-| `abci <https://github.com/tendermint/abci>`__               | Tendermint         | Go           |
-+-------------------------------------------------------------+--------------------+--------------+
-| `js abci <https://github.com/tendermint/js-abci>`__         | Tendermint         | Javascript   |                       
-+-------------------------------------------------------------+--------------------+--------------+
-| `cpp-tmsp <https://github.com/mdyring/cpp-tmsp>`__          | Martin Dyring      | C++          |
-+-------------------------------------------------------------+--------------------+--------------+
-| `jabci <https://github.com/jTendermint/jabci>`__            | jTendermint        | Java         |
-+-------------------------------------------------------------+--------------------+--------------+
-| `Spearmint <https://github.com/dennismckinnon/spearmint>`__ | Dennis Mckinnon    | Javascript   |
-+-------------------------------------------------------------+--------------------+--------------+
-| `ocaml-tmsp <https://github.com/zbo14/ocaml-tmsp>`__        | Zach Balder        | Ocaml        |
-+-------------------------------------------------------------+--------------------+--------------+
-| `abci_server <https://github.com/KrzysiekJ/abci_server>`__  | Krzysztof Jurewicz | Erlang       |
-+-------------------------------------------------------------+--------------------+--------------+
++------------------------------------------------------------------+--------------------+--------------+
+| **Name**                                                         | **Author**         | **Language** |
+|                                                                  |                    |              |
++------------------------------------------------------------------+--------------------+--------------+
+| `abci <https://github.com/tendermint/abci>`__                    | Tendermint         | Go           |
++------------------------------------------------------------------+--------------------+--------------+
+| `js abci <https://github.com/tendermint/js-abci>`__              | Tendermint         | Javascript   |
++------------------------------------------------------------------+--------------------+--------------+
+| `cpp-tmsp <https://github.com/block-finance/cpp-abci>`__         | Martin Dyring      | C++          |
++------------------------------------------------------------------+--------------------+--------------+
+| `c-abci <https://github.com/chainx-org/c-abci>`__                | ChainX             | C            |
++------------------------------------------------------------------+--------------------+--------------+
+| `jabci <https://github.com/jTendermint/jabci>`__                 | jTendermint        | Java         |
++------------------------------------------------------------------+--------------------+--------------+
+| `ocaml-tmsp <https://github.com/zbo14/ocaml-tmsp>`__             | Zach Balder        | Ocaml        |
++------------------------------------------------------------------+--------------------+--------------+
+| `abci_server <https://github.com/KrzysiekJ/abci_server>`__       | Krzysztof Jurewicz | Erlang       |
++------------------------------------------------------------------+--------------------+--------------+
+| `rust-tsp <https://github.com/tendermint/rust-tsp>`__            | Adrian Brink       | Rust         |
++------------------------------------------------------------------+--------------------+--------------+
+| `hs-abci <https://github.com/albertov/hs-abci>`__                | Alberto Gonzalez   | Haskell      |
++------------------------------------------------------------------+--------------------+--------------+
+| `haskell-abci <https://github.com/cwgoes/haskell-abci>`__        | Christoper Goes    | Haskell      |
++------------------------------------------------------------------+--------------------+--------------+
+| `Spearmint <https://github.com/dennismckinnon/spearmint>`__      | Dennis Mckinnon    | Javascript   |
++------------------------------------------------------------------+--------------------+--------------+
+| `py-abci <https://github.com/davebryson/py-abci>`__		   | Dave Bryson        | Python       |
++------------------------------------------------------------------+--------------------+--------------+
 
 Deployment Tools
 ----------------
@@ -92,3 +115,8 @@ Deployment Tools
 See `deploy testnets <./deploy-testnets.html>`__ for information about all the tools built by Tendermint. We have Kubernetes, Ansible, and Terraform integrations.
 
 Cloudsoft built `brooklyn-tendermint <https://github.com/cloudsoft/brooklyn-tendermint>`__ for deploying a tendermint testnet in docker continers. It uses Clocker for Apache Brooklyn.
+
+Dev Tools
+---------
+
+For upgrading from older to newer versions of tendermint and to migrate your chain data, see `tm-migrator <https://github.com/hxzqlh/tm-tools>`__ written by @hxzqlh.

docs/getting-started.rst

@@ -5,7 +5,7 @@ As a general purpose blockchain engine, Tendermint is agnostic to the
 application you want to run. So, to run a complete blockchain that does
 something useful, you must start two programs: one is Tendermint Core,
 the other is your application, which can be written in any programming
-language. Recall from `the intro to ABCI <introduction.rst#ABCI-Overview>`__ that
+language. Recall from `the intro to ABCI <introduction.html#ABCI-Overview>`__ that
 Tendermint Core handles all the p2p and consensus stuff, and just
 forwards transactions to the application when they need to be validated,
 or when they're ready to be committed to a block.
@@ -25,7 +25,7 @@ Then run
 
 ::
 
-    go get -u github.com/tendermint/abci/cmd/...
+    go get -u github.com/tendermint/abci/cmd/abci-cli
 
 If there is an error, install and run the ``glide`` tool to pin the
 dependencies:
@@ -35,20 +35,12 @@ dependencies:
     go get github.com/Masterminds/glide
     cd $GOPATH/src/github.com/tendermint/abci
     glide install
-    go install ./cmd/...
+    go install ./cmd/abci-cli
 
-Now you should have the ``abci-cli`` plus two apps installed:
-
-::
-
-    dummy --help
-    counter --help
-
-These binaries are installed on ``$GOPATH/bin`` and all come from within
-the ``./cmd/...`` directory of the abci repository.
-
-Both of these example applications are in Go. See below for an
-application written in Javascript.
+Now you should have the ``abci-cli`` installed; you'll see
+a couple of commands (``counter`` and ``dummy``) that are
+example applications written in Go. See below for an application
+written in Javascript.
 
 Now, let's run some apps!
 
@@ -66,14 +58,14 @@ Let's start a dummy application.
 
 ::
 
-    dummy
+    abci-cli dummy
 
 In another terminal, we can start Tendermint. If you have never run
 Tendermint before, use:
 
 ::
 
-    tendermint init 
+    tendermint init
     tendermint node
 
 If you have used Tendermint, you may want to reset the data for a new
@@ -107,31 +99,56 @@ like:
 
 ::
 
-    {"jsonrpc":"2.0","id":"","result":[98,{"check_tx":{},"deliver_tx":{}}],"error":""}
-
-The ``98`` is a type-byte, and can be ignored (it's useful for
-serializing and deserializing arbitrary json). Otherwise, this result is
-empty - there's nothing to report on and everything is OK.
+    {
+      "jsonrpc": "2.0",
+      "id": "",
+      "result": {
+        "check_tx": {
+          "code": 0,
+          "data": "",
+          "log": ""
+        },
+        "deliver_tx": {
+          "code": 0,
+          "data": "",
+          "log": ""
+        },
+        "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF",
+        "height": 154
+      }
+    }
 
 We can confirm that our transaction worked and the value got stored by
 querying the app:
 
 ::
 
-    curl -s 'localhost:46657/abci_query?data="abcd"&path=""&prove=false'
+    curl -s 'localhost:46657/abci_query?data="abcd"'
 
-The ``path`` and ``prove`` arguments can be ignored for now, and in a
-future release can be left out. The result should look like:
+The result should look like:
 
 ::
 
-    {"jsonrpc":"2.0","id":"","result":[112,{"response":{"value":"61626364","log":"exists"}}],"error":""}
+    {
+      "jsonrpc": "2.0",
+      "id": "",
+      "result": {
+        "response": {
+          "code": 0,
+          "index": 0,
+          "key": "",
+          "value": "61626364",
+          "proof": "",
+          "height": 0,
+          "log": "exists"
+        }
+      }
+    }
 
-Again, the ``112`` is the type-byte. Note the ``value`` in the result
-(``61626364``); this is the hex-encoding of the ASCII of ``abcd``. You
-can verify this in a python shell by running
-``"61626364".decode('hex')``. Stay tuned for a future release that makes
-this output more human-readable ;).
+Note the ``value`` in the result (``61626364``); this is the
+hex-encoding of the ASCII of ``abcd``. You can verify this in
+a python shell by running ``"61626364".decode('hex')``. Stay
+tuned for a future release that `makes this output more human-readable <https://github.com/tendermint/abci/issues/32>`__.
 
 Now let's try setting a different key and value:
 
@@ -144,7 +161,7 @@ Now if we query for ``name``, we should get ``satoshi``, or
 
 ::
 
-    curl -s 'localhost:46657/abci_query?data="name"&path=""&prove=false'
+    curl -s 'localhost:46657/abci_query?data="name"'
 
 Try some other transactions and queries to make sure everything is
 working!
@@ -153,7 +170,7 @@ Counter - Another Example
 -------------------------
 
 Now that we've got the hang of it, let's try another application, the
-"counter" app.
+**counter** app.
 
 The counter app doesn't use a Merkle tree, it just counts how many times
 we've sent a transaction, or committed the state.
@@ -181,7 +198,7 @@ a flag:
 
 ::
 
-    counter --serial
+    abci-cli counter --serial
 
 In another window, reset then start Tendermint:
 
@@ -204,14 +221,48 @@ the number ``1``. If instead, we try to send a ``5``, we get an error:
 ::
 
     > curl localhost:46657/broadcast_tx_commit?tx=0x05
-    {"jsonrpc":"2.0","id":"","result":[98,{"check_tx":{},"deliver_tx":{"code":3,"log":"Invalid nonce. Expected 1, got 5"}}],"error":""}
+    {
+      "jsonrpc": "2.0",
+      "id": "",
+      "result": {
+        "check_tx": {
+          "code": 0,
+          "data": "",
+          "log": ""
+        },
+        "deliver_tx": {
+          "code": 3,
+          "data": "",
+          "log": "Invalid nonce. Expected 1, got 5"
+        },
+        "hash": "33B93DFF98749B0D6996A70F64071347060DC19C",
+        "height": 38
+      }
+    }
 
 But if we send a ``1``, it works again:
 
 ::
 
     > curl localhost:46657/broadcast_tx_commit?tx=0x01
-    {"jsonrpc":"2.0","id":"","result":[98,{"check_tx":{},"deliver_tx":{}}],"error":""}
+    {
+      "jsonrpc": "2.0",
+      "id": "",
+      "result": {
+        "check_tx": {
+          "code": 0,
+          "data": "",
+          "log": ""
+        },
+        "deliver_tx": {
+          "code": 0,
+          "data": "",
+          "log": ""
+        },
+        "hash": "F17854A977F6FA7EEA1BD758E296710B86F72F3D",
+        "height": 87
+      }
+    }
 
 For more details on the ``broadcast_tx`` API, see `the guide on using
 Tendermint <./using-tendermint.html>`__.
@@ -231,6 +282,7 @@ keep all our code under the ``$GOPATH``, so run:
     go get github.com/tendermint/js-abci &> /dev/null
     cd $GOPATH/src/github.com/tendermint/js-abci/example
     npm install
+    cd ..
 
 Kill the previous ``counter`` and ``tendermint`` processes. Now run the
 app:
@@ -261,12 +313,12 @@ Neat, eh?
 Basecoin - A More Interesting Example
 -------------------------------------
 
-We saved the best for last; the `Cosmos SDK <https://github.com/cosmos/cosmos-sdk>`__ is a general purpose framework for building cryptocurrencies. Unlike the``dummy`` and ``counter``, which are strictly for example purposes. The reference implementation of Cosmos SDK is ``basecoin``, which demonstrates how to use the building blocks of the Cosmos SDK.
+We saved the best for last; the `Cosmos SDK <https://github.com/cosmos/cosmos-sdk>`__ is a general purpose framework for building cryptocurrencies. Unlike the ``dummy`` and ``counter``, which are strictly for example purposes. The reference implementation of Cosmos SDK is ``basecoin``, which demonstrates how to use the building blocks of the Cosmos SDK.
 
 The default ``basecoin`` application is a multi-asset cryptocurrency
-that supports inter-blockchain communication. For more details on how
+that supports inter-blockchain communication (IBC). For more details on how
 basecoin works and how to use it, see our `basecoin
-guide <https://github.com/cosmos/cosmos-sdk/blob/develop/docs/guide/basecoin-basics.md>`__
+guide <http://cosmos-sdk.readthedocs.io/en/latest/basecoin-basics.html>`__
 
 In this tutorial you learned how to run applications using Tendermint
 on a single node. You saw how applications could be written in different

docs/how-to-read-logs.rst

@@ -0,0 +1,165 @@
+How to read logs
+================
+
+Walk through example
+--------------------
+
+We first create three connections (mempool, consensus and query) to the
+application (locally running dummy in this case).
+
+::
+
+    I[10-04|13:54:27.364] Starting multiAppConn                        module=proxy impl=multiAppConn
+    I[10-04|13:54:27.366] Starting localClient                         module=abci-client connection=query impl=localClient
+    I[10-04|13:54:27.366] Starting localClient                         module=abci-client connection=mempool impl=localClient
+    I[10-04|13:54:27.367] Starting localClient                         module=abci-client connection=consensus impl=localClient
+
+Then Tendermint Core and the application perform a handshake.
+
+::
+
+    I[10-04|13:54:27.367] ABCI Handshake                               module=consensus appHeight=90 appHash=E0FBAFBF6FCED8B9786DDFEB1A0D4FA2501BADAD
+    I[10-04|13:54:27.368] ABCI Replay Blocks                           module=consensus appHeight=90 storeHeight=90 stateHeight=90
+    I[10-04|13:54:27.368] Completed ABCI Handshake - Tendermint and App are synced module=consensus appHeight=90 appHash=E0FBAFBF6FCED8B9786DDFEB1A0D4FA2501BADAD
+
+After that, we start a few more things like the event switch, reactors, and
+perform UPNP discover in order to detect the IP address.
+
+::
+
+    I[10-04|13:54:27.374] Starting EventSwitch                         module=types impl=EventSwitch
+    I[10-04|13:54:27.375] This node is a validator                     module=consensus
+    I[10-04|13:54:27.379] Starting Node                                module=main impl=Node
+    I[10-04|13:54:27.381] Local listener                               module=p2p ip=:: port=46656
+    I[10-04|13:54:27.382] Getting UPNP external address                module=p2p
+    I[10-04|13:54:30.386] Could not perform UPNP discover              module=p2p err="write udp4 0.0.0.0:38238->239.255.255.250:1900: i/o timeout"
+    I[10-04|13:54:30.386] Starting DefaultListener                     module=p2p impl=Listener(@10.0.2.15:46656)
+    I[10-04|13:54:30.387] Starting P2P Switch                          module=p2p impl="P2P Switch"
+    I[10-04|13:54:30.387] Starting MempoolReactor                      module=mempool impl=MempoolReactor
+    I[10-04|13:54:30.387] Starting BlockchainReactor                   module=blockchain impl=BlockchainReactor
+    I[10-04|13:54:30.387] Starting ConsensusReactor                    module=consensus impl=ConsensusReactor
+    I[10-04|13:54:30.387] ConsensusReactor                             module=consensus fastSync=false
+    I[10-04|13:54:30.387] Starting ConsensusState                      module=consensus impl=ConsensusState
+    I[10-04|13:54:30.387] Starting WAL                                 module=consensus wal=/home/vagrant/.tendermint/data/cs.wal/wal impl=WAL
+    I[10-04|13:54:30.388] Starting TimeoutTicker                       module=consensus impl=TimeoutTicker
+
+Notice the second row where Tendermint Core reports that "This node is a
+validator". It also could be just an observer (regular node).
+
+Next we replay all the messages from the WAL.
+
+::
+
+    I[10-04|13:54:30.390] Catchup by replaying consensus messages      module=consensus height=91
+    I[10-04|13:54:30.390] Replay: New Step                             module=consensus height=91 round=0 step=RoundStepNewHeight
+    I[10-04|13:54:30.390] Replay: Done                                 module=consensus
+
+"Started node" message signals that everything is ready for work.
+
+::
+
+    I[10-04|13:54:30.391] Starting RPC HTTP server on tcp socket 0.0.0.0:46657 module=rpc-server
+    I[10-04|13:54:30.392] Started node                                 module=main nodeInfo="NodeInfo{pk: PubKeyEd25519{DF22D7C92C91082324A1312F092AA1DA197FA598DBBFB6526E177003C4D6FD66}, moniker: anonymous, network: test-chain-3MNw2N [remote , listen 10.0.2.15:46656], version: 0.11.0-10f361fc ([wire_version=0.6.2 p2p_version=0.5.0 consensus_version=v1/0.2.2 rpc_version=0.7.0/3 tx_index=on rpc_addr=tcp://0.0.0.0:46657])}"
+
+Next follows a standard block creation cycle, where we enter a new round,
+propose a block, receive more than 2/3 of prevotes, then precommits and finally
+have a chance to commit a block. For details, please refer to `Consensus
+Overview
+<introduction.html#consensus-overview>`__
+or `Byzantine Consensus Algorithm
+<specification.html>`__.
+
+::
+
+    I[10-04|13:54:30.393] enterNewRound(91/0). Current: 91/0/RoundStepNewHeight module=consensus
+    I[10-04|13:54:30.393] enterPropose(91/0). Current: 91/0/RoundStepNewRound module=consensus
+    I[10-04|13:54:30.393] enterPropose: Our turn to propose            module=consensus proposer=125B0E3C5512F5C2B0E1109E31885C4511570C42 privValidator="PrivValidator{125B0E3C5512F5C2B0E1109E31885C4511570C42 LH:90, LR:0, LS:3}"
+    I[10-04|13:54:30.394] Signed proposal                              module=consensus height=91 round=0 proposal="Proposal{91/0 1:21B79872514F (-1,:0:000000000000) {/10EDEDD7C84E.../}}"
+    I[10-04|13:54:30.397] Received complete proposal block             module=consensus height=91 hash=F671D562C7B9242900A286E1882EE64E5556FE9E
+    I[10-04|13:54:30.397] enterPrevote(91/0). Current: 91/0/RoundStepPropose module=consensus
+    I[10-04|13:54:30.397] enterPrevote: ProposalBlock is valid         module=consensus height=91 round=0
+    I[10-04|13:54:30.398] Signed and pushed vote                       module=consensus height=91 round=0 vote="Vote{0:125B0E3C5512 91/00/1(Prevote) F671D562C7B9 {/89047FFC21D8.../}}" err=null
+    I[10-04|13:54:30.401] Added to prevote                             module=consensus vote="Vote{0:125B0E3C5512 91/00/1(Prevote) F671D562C7B9 {/89047FFC21D8.../}}" prevotes="VoteSet{H:91 R:0 T:1 +2/3:F671D562C7B9242900A286E1882EE64E5556FE9E:1:21B79872514F BA{1:X} map[]}"
+    I[10-04|13:54:30.401] enterPrecommit(91/0). Current: 91/0/RoundStepPrevote module=consensus
+    I[10-04|13:54:30.401] enterPrecommit: +2/3 prevoted proposal block. Locking module=consensus hash=F671D562C7B9242900A286E1882EE64E5556FE9E
+    I[10-04|13:54:30.402] Signed and pushed vote                       module=consensus height=91 round=0 vote="Vote{0:125B0E3C5512 91/00/2(Precommit) F671D562C7B9 {/80533478E41A.../}}" err=null
+    I[10-04|13:54:30.404] Added to precommit                           module=consensus vote="Vote{0:125B0E3C5512 91/00/2(Precommit) F671D562C7B9 {/80533478E41A.../}}" precommits="VoteSet{H:91 R:0 T:2 +2/3:F671D562C7B9242900A286E1882EE64E5556FE9E:1:21B79872514F BA{1:X} map[]}"
+    I[10-04|13:54:30.404] enterCommit(91/0). Current: 91/0/RoundStepPrecommit module=consensus
+    I[10-04|13:54:30.405] Finalizing commit of block with 0 txs        module=consensus height=91 hash=F671D562C7B9242900A286E1882EE64E5556FE9E root=E0FBAFBF6FCED8B9786DDFEB1A0D4FA2501BADAD
+    I[10-04|13:54:30.405] Block{
+      Header{
+        ChainID:        test-chain-3MNw2N
+        Height:         91
+        Time:           2017-10-04 13:54:30.393 +0000 UTC
+        NumTxs:         0
+        LastBlockID:    F15AB8BEF9A6AAB07E457A6E16BC410546AA4DC6:1:D505DA273544
+        LastCommit:     56FEF2EFDB8B37E9C6E6D635749DF3169D5F005D
+        Data:
+        Validators:     CE25FBFF2E10C0D51AA1A07C064A96931BC8B297
+        App:            E0FBAFBF6FCED8B9786DDFEB1A0D4FA2501BADAD
+      }#F671D562C7B9242900A286E1882EE64E5556FE9E
+      Data{
+
+      }#
+      Commit{
+        BlockID:    F15AB8BEF9A6AAB07E457A6E16BC410546AA4DC6:1:D505DA273544
+        Precommits: Vote{0:125B0E3C5512 90/00/2(Precommit) F15AB8BEF9A6 {/FE98E2B956F0.../}}
+      }#56FEF2EFDB8B37E9C6E6D635749DF3169D5F005D
+    }#F671D562C7B9242900A286E1882EE64E5556FE9E module=consensus
+    I[10-04|13:54:30.408] Executed block                               module=state height=91 validTxs=0 invalidTxs=0
+    I[10-04|13:54:30.410] Committed state                              module=state height=91 txs=0 hash=E0FBAFBF6FCED8B9786DDFEB1A0D4FA2501BADAD
+    I[10-04|13:54:30.410] Recheck txs                                  module=mempool numtxs=0 height=91
+
+List of modules
+---------------
+
+Here is the list of modules you may encounter in Tendermint's log and a little
+overview what they do.
+
+- ``abci-client`` As mentioned in `Application Development Guide
+  <app-development.html#abci-design>`__,
+  Tendermint acts as an ABCI client with respect to the application and
+  maintains 3 connections: mempool, consensus and query. The code used by
+  Tendermint Core can be found `here
+  <https://github.com/tendermint/abci/tree/master/client>`__.
+
+- ``blockchain``
+  Provides storage, pool (a group of peers), and reactor for both storing and
+  exchanging blocks between peers.
+
+- ``consensus``
+  The heart of Tendermint core, which is the implementation of the consensus
+  algorithm. Includes two "submodules": ``wal`` (write-ahead logging) for
+  ensuring data integrity and ``replay`` to replay blocks and messages on
+  recovery from a crash.
+
+- ``events``
+  Simple event notification system. The list of events can be found
+  `here
+  <https://github.com/tendermint/tendermint/blob/master/types/events.go>`__.
+  You can subscribe to them by calling ``subscribe`` RPC method.
+  Refer to `RPC docs
+  <specification/rpc.html>`__
+  for additional information.
+
+- ``mempool``
+  Mempool module handles all incoming transactions, whenever they are
+  coming from peers or the application.
+
+- ``p2p``
+  Provides an abstraction around peer-to-peer communication. For more details,
+  please check out the `README
+  <https://github.com/tendermint/tendermint/blob/56c60fba2381e4ac41d2ae38a1eb6569acfbc095/p2p/README.md>`__.
+
+- ``rpc``
+  `Tendermint's RPC <specification/rpc.html>`__.
+
+- ``rpc-server``
+  RPC server. For implementation details, please read the `README <https://github.com/tendermint/tendermint/blob/master/rpc/lib/README.md>`__.
+
+- ``state``
+  Represents the latest state and execution submodule, which executes
+  blocks against the application.
+
+- ``types``
+  A collection of the publicly exposed types and methods to work with them.

docs/index.rst

@@ -8,15 +8,13 @@ Welcome to Tendermint!
 
 
 .. image:: assets/tmint-logo-blue.png
-   :height: 500px
-   :width: 500px
+   :height: 200px
+   :width: 200px
    :align: center
 
 Tendermint 101
 --------------
 
-.. maxdepth set to 2 for sexinesss
-.. but use 4 to upgrade overall documentation
 .. toctree::
    :maxdepth: 2
 
@@ -25,9 +23,19 @@ Tendermint 101
    getting-started.rst
    using-tendermint.rst
 
+Tendermint Ecosystem
+--------------------
+
+.. toctree::
+   :maxdepth: 2
+
+   ecosystem.rst
+
 Tendermint Tools
 ----------------
 
+.. the tools/ files are pulled in from the tools repo
+.. see the bottom of conf.py
 .. toctree::
    :maxdepth: 2
 
@@ -38,15 +46,6 @@ Tendermint Tools
    tools/terraform-digitalocean.rst
    tools/benchmarking-and-monitoring.rst
 
-
-Tendermint Ecosystem
---------------------
-
-.. toctree::
-   :maxdepth: 2
-
-   ecosystem.rst
-
 Tendermint 102
 --------------
 
@@ -56,6 +55,7 @@ Tendermint 102
    abci-cli.rst
    app-architecture.rst
    app-development.rst
+   how-to-read-logs.rst
 
 Tendermint 201
 --------------

docs/install.rst

@@ -9,13 +9,13 @@ To download pre-built binaries, see the `Download page <https://tendermint.com/d
 From Source
 -----------
 
-You'll need `go`, maybe `glide` and the tendermint source code.
+You'll need ``go``, maybe ``glide``, and the tendermint source code.
 
 Install Go
 ^^^^^^^^^^
 
 Make sure you have `installed Go <https://golang.org/doc/install>`__ and
-set the ``GOPATH``.
+set the ``GOPATH``. You should also put ``GOPATH/bin`` on your ``PATH``.
 
 Get Source Code
 ^^^^^^^^^^^^^^^
@@ -29,7 +29,7 @@ You should be able to install the latest with a simple
 Run ``tendermint --help`` and ``tendermint version`` to ensure your
 installation worked.
 
-If the installation failed, a dependency may been updated and become
+If the installation failed, a dependency may have been updated and become
 incompatible with the latest Tendermint master branch. We solve this
 using the ``glide`` tool for dependency management.
 

docs/introduction.rst

@@ -146,11 +146,11 @@ The ABCI consists of 3 primary message types that get delivered from the core to
 
 The messages are specified here: `ABCI Message Types <https://github.com/tendermint/abci#message-types>`__.
 
-The `DeliverTx` message is the work horse of the application.  Each transaction in the blockchain is delivered with this message. The application needs to validate each transaction received with the `DeliverTx` message against the current state, application protocol, and the cryptographic credentials of the transaction. A validated transaction then needs to update the application state — by binding a value into a key values store, or by updating the UTXO database, for instance.
+The **DeliverTx** message is the work horse of the application. Each transaction in the blockchain is delivered with this message. The application needs to validate each transaction received with the **DeliverTx** message against the current state, application protocol, and the cryptographic credentials of the transaction. A validated transaction then needs to update the application state — by binding a value into a key values store, or by updating the UTXO database, for instance.
 
-The `CheckTx` message is similar to `DeliverTx`, but it's only for validating transactions.  Tendermint Core's mempool first checks the validity of a transaction with `CheckTx`, and only relays valid transactions to its peers.  For instance, an application may check an incrementing sequence number in the transaction and return an error upon `CheckTx` if the sequence number is old. Alternatively, they might use a capabilities based system that requires capabilities to be renewed with every transaction.
+The **CheckTx** message is similar to **DeliverTx**, but it's only for validating transactions. Tendermint Core's mempool first checks the validity of a transaction with **CheckTx**, and only relays valid transactions to its peers. For instance, an application may check an incrementing sequence number in the transaction and return an error upon **CheckTx** if the sequence number is old. Alternatively, they might use a capabilities based system that requires capabilities to be renewed with every transaction.
 
-The `Commit` message is used to compute a cryptographic commitment to the current application state, to be placed into the next block header. This has some handy properties. Inconsistencies in updating that state will now appear as blockchain forks which catches a whole class of programming errors. This also simplifies the development of secure lightweight clients, as Merkle-hash proofs can be verified by checking against the block hash, and that the block hash is signed by a quorum.
+The **Commit** message is used to compute a cryptographic commitment to the current application state, to be placed into the next block header. This has some handy properties. Inconsistencies in updating that state will now appear as blockchain forks which catches a whole class of programming errors. This also simplifies the development of secure lightweight clients, as Merkle-hash proofs can be verified by checking against the block hash, and that the block hash is signed by a quorum.
 
 There can be multiple ABCI socket connections to an application. Tendermint Core creates three ABCI connections to the application; one for the validation of transactions when broadcasting in the mempool, one for the consensus engine to run block proposals, and one more for querying the application state.
 
@@ -169,7 +169,7 @@ Solidity on Ethereum is a great language of choice for blockchain applications b
  * race conditions on threads (or avoiding threads altogether)
  * the system clock
  * uninitialized memory (in unsafe programming languages like C or C++)
- * `floating point arithmetic <http://gafferongames.com/networking-for-game-programmers/floating-point-determinism/>`__.
+ * `floating point arithmetic <http://gafferongames.com/networking-for-game-programmers/floating-point-determinism/>`__
  * language features that are random (e.g. map iteration in Go)
 
 While programmers can avoid non-determinism by being careful, it is also possible to create a special linter or static analyzer for each language to check for determinism.  In the future we may work with partners to create such tools.
@@ -182,17 +182,17 @@ The protocol follows a simple state machine that looks like this:
 
 .. figure:: assets/consensus_logic.png
 
-Participants in the protocol are called "validators";
+Participants in the protocol are called **validators**;
 they take turns proposing blocks of transactions and voting on them.
-Blocks are committed in a chain, with one block at each "height".
-A block may fail to be committed, in which case the protocol moves to the next "round",
+Blocks are committed in a chain, with one block at each **height**.
+A block may fail to be committed, in which case the protocol moves to the next **round**,
 and a new validator gets to propose a block for that height.
 Two stages of voting are required to successfully commit a block;
-we call them "pre-vote" and "pre-commit".
+we call them **pre-vote** and **pre-commit**.
 A block is committed when more than 2/3 of validators pre-commit for the same block in the same round.
 
 There is a picture of a couple doing the polka because validators are doing something like a polka dance.
-When more than two-thirds of the validators pre-vote for the same block, we call that a "polka".
+When more than two-thirds of the validators pre-vote for the same block, we call that a **polka**.
 Every pre-commit must be justified by a polka in the same round.
 
 Validators may fail to commit a block for a number of reasons; 
@@ -204,8 +204,8 @@ However, the rest of the protocol is asynchronous, and validators only make prog
 A simplifying element of Tendermint is that it uses the same mechanism to commit a block as it does to skip to the next round.
 
 Assuming less than one-third of the validators are Byzantine, Tendermint guarantees that safety will never be violated - that is, validators will never commit conflicting blocks at the same height.
-To do this it introduces a few "locking" rules which modulate which paths can be followed in the flow diagram.
-Once a validator precommits a block, it is "locked" on that block. 
+To do this it introduces a few **locking** rules which modulate which paths can be followed in the flow diagram.
+Once a validator precommits a block, it is locked on that block.
 Then, 
 
 1) it must prevote for the block it is locked on

docs/specification.rst

@@ -2,19 +2,19 @@
 Specification
 #############
 
-Here you'll find details of the Tendermint specification. See `the spec repo <https://github.com/tendermint/spec>`__ for upcoming material. Tendermint's types are produced by `godoc <https://godoc.org/github.com/tendermint/tendermint/types>`__
+Here you'll find details of the Tendermint specification. See `the spec repo <https://github.com/tendermint/spec>`__ for upcoming material. Tendermint's types are produced by `godoc <https://godoc.org/github.com/tendermint/tendermint/types>`__.
 
 .. toctree::
    :maxdepth: 2
 
-  specification/block-structure.rst
-  specification/byzantine-consensus-algorithm.rst
-  specification/configuration.rst
-  specification/fast-sync.rst
-  specification/genesis.rst
-  specification/light-client-protocol.rst
-  specification/merkle.rst
-  specification/rpc.rst
-  specification/secure-p2p.rst
-  specification/validators.rst
-  specification/wire-protocol.rst
+   specification/block-structure.rst
+   specification/byzantine-consensus-algorithm.rst
+   specification/configuration.rst
+   specification/fast-sync.rst
+   specification/genesis.rst
+   specification/light-client-protocol.rst
+   specification/merkle.rst
+   specification/rpc.rst
+   specification/secure-p2p.rst
+   specification/validators.rst
+   specification/wire-protocol.rst

docs/specification/block-structure.rst

@@ -98,7 +98,7 @@ This is to protect anyone from swapping votes between chains to fake (or
 frame) a validator. Also note that this ``chainID`` is in the
 ``genesis.json`` from *Tendermint*, not the ``genesis.json`` from the
 basecoin app (`that is a different
-chainID... <https://github.com/tendermint/basecoin/issues/32>`__).
+chainID... <https://github.com/cosmos/cosmos-sdk/issues/32>`__).
 
 Once we have those votes, and we calculated the proper `sign
 bytes <https://godoc.org/github.com/tendermint/tendermint/types#Vote.WriteSignBytes>`__
@@ -136,7 +136,7 @@ Block Hash
 
 The `block
 hash <https://godoc.org/github.com/tendermint/tendermint/types#Block.Hash>`__
-is the `Simple Tree hash <Merkle-Trees#simple-tree-with-dictionaries>`__
+is the `Simple Tree hash <./merkle.html#simple-tree-with-dictionaries>`__
 of the fields of the block ``Header`` encoded as a list of
 ``KVPair``\ s.
 

docs/specification/configuration.rst

@@ -21,7 +21,8 @@ The main config parameters are defined
 -  ``genesis_file``: The location of the genesis file. *Default*:
    ``"$TMHOME/genesis.json"``
 -  ``log_level``: *Default*: ``"state:info,*:error"``
--  ``moniker``: Name of this node. *Default*: ``"anonymous"``
+-  ``moniker``: Name of this node. *Default*: the host name or ``"anonymous"``
+   if runtime fails to get the host name
 -  ``priv_validator_file``: Validator private key file. *Default*:
    ``"$TMHOME/priv_validator.json"``
 -  ``prof_laddr``: Profile listen address. *Default*: ``""``
@@ -30,14 +31,16 @@ The main config parameters are defined
 
 -  ``consensus.max_block_size_txs``: Maximum number of block txs.
    *Default*: ``10000``
+-  ``consensus.create_empty_blocks``: Create empty blocks w/o txs.
+   *Default*: ``true``
+-  ``consensus.create_empty_blocks_interval``: Block creation interval, even if empty.
 -  ``consensus.timeout_*``: Various consensus timeout parameters
-   **TODO**
 -  ``consensus.wal_file``: Consensus state WAL. *Default*:
-   ``"$TMHOME/data/cswal"``
+   ``"$TMHOME/data/cs.wal/wal"``
 -  ``consensus.wal_light``: Whether to use light-mode for Consensus
    state WAL. *Default*: ``false``
 
--  ``mempool.*``: Various mempool parameters **TODO**
+-  ``mempool.*``: Various mempool parameters
 
 -  ``p2p.addr_book_file``: Peer address book. *Default*:
    ``"$TMHOME/addrbook.json"``. **NOT USED**

docs/specification/genesis.rst

@@ -1,7 +1,7 @@
 Genesis
 =======
 
-The genesis.json file in ``$TMROOT`` defines the initial TendermintCore
+The genesis.json file in ``$TMHOME`` defines the initial TendermintCore
 state upon genesis of the blockchain (`see
 definition <https://github.com/tendermint/tendermint/blob/master/types/genesis.go>`__).
 

docs/specification/merkle.rst

@@ -6,9 +6,9 @@ For an overview of Merkle trees, see
 
 There are two types of Merkle trees used in Tendermint.
 
--  ```IAVL+ Tree`` <#iavl-tree>`__: An immutable self-balancing binary
+-  **IAVL+ Tree**: An immutable self-balancing binary
    tree for persistent application state
--  ```Simple Tree`` <#simple-tree>`__: A simple compact binary tree for
+-  **Simple Tree**: A simple compact binary tree for
    a static list of items
 
 IAVL+ Tree

docs/specification/validators.rst

@@ -4,7 +4,7 @@ Validators
 Validators are responsible for committing new blocks in the blockchain.
 These validators participate in the consensus protocol by broadcasting
 *votes* which contain cryptographic signatures signed by each
-validator's public key.
+validator's private key.
 
 Some Proof-of-Stake consensus algorithms aim to create a "completely"
 decentralized system where all stakeholders (even those who are not

docs/using-tendermint.rst

@@ -12,7 +12,7 @@ Directory Root
 --------------
 
 The default directory for blockchain data is ``~/.tendermint``. Override
-this by setting the ``TMROOT`` environment variable.
+this by setting the ``TMHOME`` environment variable.
 
 Initialize
 ----------
@@ -39,7 +39,7 @@ To run a tendermint node, use
 
     tendermint node
 
-By default, Tendermint will try to connect to a abci appliction on
+By default, Tendermint will try to connect to an ABCI application on
 `127.0.0.1:46658 <127.0.0.1:46658>`__. If you have the ``dummy`` ABCI
 app installed, run it in another window. If you don't, kill tendermint
 and run an in-process version with
@@ -49,9 +49,7 @@ and run an in-process version with
     tendermint node --proxy_app=dummy
 
 After a few seconds you should see blocks start streaming in. Note that
-blocks are produced regularly, even if there are no transactions. This
-changes `with this pull
-request <https://github.com/tendermint/tendermint/pull/584>`__.
+blocks are produced regularly, even if there are no transactions. See *No Empty Blocks*, below, to modify this setting.
 
 Tendermint supports in-process versions of the dummy, counter, and nil
 apps that ship as examples in the `ABCI
@@ -59,7 +57,7 @@ repository <https://github.com/tendermint/abci>`__. It's easy to compile
 your own app in-process with tendermint if it's written in Go. If your
 app is not written in Go, simply run it in another process, and use the
 ``--proxy_app`` flag to specify the address of the socket it is
-listening on, for instance
+listening on, for instance:
 
 ::
 
@@ -116,8 +114,8 @@ new blockchain will not make any blocks.
 Configuration
 -------------
 
-Tendermint uses a ``config.toml`` for configutation. For details, see
-`the documentation <./specification/configuration.html>`__.
+Tendermint uses a ``config.toml`` for configuration. For details, see
+`the config specification <./specification/configuration.html>`__.
 
 Notable options include the socket address of the application
 (``proxy_app``), the listenting address of the tendermint peer
@@ -126,6 +124,38 @@ Notable options include the socket address of the application
 
 Some fields from the config file can be overwritten with flags.
 
+No Empty Blocks
+---------------
+
+This much requested feature was implemented in version 0.10.3. While the default behaviour of ``tendermint`` is still to create blocks approximately once per second, it is possible to disable empty blocks or set a block creation interval. In the former case, blocks will be created when there are new transactions or when the AppHash changes.
+
+To configure tendermint to not produce empty blocks unless there are txs or the app hash changes, 
+run tendermint with this additional flag:
+
+::
+
+    tendermint node --consensus.create_empty_blocks=false
+
+or set the configuration via the ``config.toml`` file:
+
+::
+
+    [consensus]
+    create_empty_blocks = false
+
+Remember: because the default is to *create empty blocks*, avoiding empty blocks requires the config option to be set to ``false``.
+
+The block interval setting allows for a delay (in seconds) between the creation of each new empty block. It is set via the ``config.toml``:
+
+::
+
+    [consensus]
+    create_empty_blocks_interval = 5
+
+With this setting, empty blocks will be produced every 5s if no block has been produced otherwise,
+regardless of the value of `create_empty_blocks`.
+
+
 Broadcast API
 -------------
 
@@ -251,7 +281,7 @@ specify peers for a running node to connect to:
 
 Additionally, the peer-exchange protocol can be enabled using the
 ``--pex`` flag, though this feature is `still under
-development <https://github.com/tendermint/tendermint/issues/598>`__ If
+development <https://github.com/tendermint/tendermint/issues/598>`__. If
 ``--pex`` is enabled, peers will gossip about known peers and form a
 more resilient network.
 
@@ -354,3 +384,11 @@ the listening addresses of the various sockets don't conflict.
 Additionally, you must set ``addrbook_strict=false`` in the
 ``config.toml``, otherwise Tendermint's p2p library will deny making
 connections to peers with the same IP address.
+
+Upgrading
+~~~~~~~~~
+
+The tendermint development cycle includes a lot of breaking changes. Upgrading from
+an old version to a new version usually means throwing away the chain data. Try out
+the `tm-migrate <https://github.com/hxzqlh/tm-tools>`__ tool written by @hxqlh if
+you are keen to preserve the state of your chain when upgrading to newer versions.

glide.lock

@@ -1,8 +1,8 @@
-hash: e3649cac7b1b9a23c024a9d1bbebd5a147861d55da2bca77c95129b6021850b4
-updated: 2017-09-22T13:24:29.443800586-04:00
+hash: f420f1f858100218dad50997d939eaaf129ff654a0648a47ddc60d626ab0b8e9
+updated: 2017-12-10T05:37:46.41123196Z
 imports:
 - name: github.com/btcsuite/btcd
-  version: 4803a8291c92a1d2d41041b942a9a9e37deab065
+  version: 2e60448ffcc6bf78332d1fe590260095f554dd78
   subpackages:
   - btcec
 - name: github.com/ebuchman/fail-test
@@ -10,7 +10,7 @@ imports:
 - name: github.com/fsnotify/fsnotify
   version: 4da3e2cfbabc9f751898f250b49f2439785783a1
 - name: github.com/go-kit/kit
-  version: 0d313fb5fb3a94d87d61e6434785264e87a5d740
+  version: e3b2152e0063c5f05efea89ecbe297852af2a92d
   subpackages:
   - log
   - log/level
@@ -18,19 +18,24 @@ imports:
 - name: github.com/go-logfmt/logfmt
   version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
 - name: github.com/go-playground/locales
-  version: 1e5f1161c6416a5ff48840eb8724a394e48cc534
+  version: e4cbcb5d0652150d40ad0646651076b6bd2be4f6
   subpackages:
   - currency
 - name: github.com/go-playground/universal-translator
   version: 71201497bace774495daed26a3874fd339e0b538
 - name: github.com/go-stack/stack
-  version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf
+  version: 259ab82a6cad3992b4e21ff5cac294ccb06474bc
 - name: github.com/gogo/protobuf
-  version: 2adc21fd136931e0388e278825291678e1d98309
+  version: 342cbe0a04158f6dcb03ca0079991a51a4248c02
   subpackages:
+  - gogoproto
+  - jsonpb
   - proto
+  - protoc-gen-gogo/descriptor
+  - sortkeys
+  - types
 - name: github.com/golang/protobuf
-  version: 130e6b02ab059e7b717a096f397c5b60111cae74
+  version: 1e59b77b52bf8e4b449a57e6f79f21226d571845
   subpackages:
   - proto
   - ptypes
@@ -40,9 +45,9 @@ imports:
 - name: github.com/golang/snappy
   version: 553a641470496b2327abcac10b36396bd98e45c9
 - name: github.com/gorilla/websocket
-  version: 6f34763140ed8887aed6a044912009832b4733d7
+  version: ea4d1f681babbce9545c9c5f3d5194a789c89f5b
 - name: github.com/hashicorp/hcl
-  version: 68e816d1c783414e79bc65b3994d9ab6b0a722ab
+  version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8
   subpackages:
   - hcl/ast
   - hcl/parser
@@ -59,31 +64,31 @@ imports:
 - name: github.com/kr/logfmt
   version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0
 - name: github.com/magiconair/properties
-  version: 8d7837e64d3c1ee4e54a880c5a920ab4316fc90a
+  version: 49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934
 - name: github.com/mitchellh/mapstructure
-  version: d0303fe809921458f417bcf828397a65db30a7e4
+  version: 06020f85339e21b2478f756a78e295255ffa4d6a
 - name: github.com/pelletier/go-toml
-  version: 1d6b12b7cb290426e27e6b4e38b89fcda3aeef03
+  version: 4e9e0ee19b60b13eb79915933f44d8ed5f268bdd
 - name: github.com/pkg/errors
   version: 645ef00459ed84a119197bfb8d8205042c6df63d
 - name: github.com/rcrowley/go-metrics
-  version: 1f30fe9094a513ce4c700b9a54458bbb0c96996c
+  version: e181e095bae94582363434144c61a9653aff6e50
 - name: github.com/spf13/afero
-  version: ee1bd8ee15a1306d1f9201acc41ef39cd9f99a1b
+  version: 8d919cbe7e2627e417f3e45c3c0e489a5b7e2536
   subpackages:
   - mem
 - name: github.com/spf13/cast
   version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4
 - name: github.com/spf13/cobra
-  version: b78744579491c1ceeaaa3b40205e56b0591b93a3
+  version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b
 - name: github.com/spf13/jwalterweatherman
   version: 12bd96e66386c1960ab0f74ced1362f66f552f7b
 - name: github.com/spf13/pflag
-  version: 7aff26db30c1be810f9de5038ec5ef96ac41fd7c
+  version: 4c012f6dcd9546820e378d0bdda4d8fc772cdfea
 - name: github.com/spf13/viper
   version: 25b30aa063fc18e48662b86996252eabdcf2f0c7
 - name: github.com/syndtr/goleveldb
-  version: b89cc31ef7977104127d34c1bd31ebd1a9db2199
+  version: adf24ef3f94bd13ec4163060b21a5678f22b429b
   subpackages:
   - leveldb
   - leveldb/cache
@@ -98,31 +103,33 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 191c4b6d176169ffc7f9972d490fa362a3b7d940
+  version: fca2b508c185b855af1446ec4afc19bdfc7b315d
   subpackages:
   - client
+  - example/code
   - example/counter
   - example/dummy
   - server
   - types
 - name: github.com/tendermint/ed25519
-  version: 1f52c6f8b8a5c7908aff4497c186af344b428925
+  version: d8387025d2b9d158cf4efb07e7ebf814bcce2057
   subpackages:
   - edwards25519
   - extra25519
 - name: github.com/tendermint/go-crypto
-  version: 311e8c1bf00fa5868daad4f8ea56dcad539182c0
+  version: dd20358a264c772b4a83e477b0cfce4c88a7001d
 - name: github.com/tendermint/go-wire
-  version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb
+  version: b6fc872b42d41158a60307db4da051dd6f179415
   subpackages:
   - data
   - data/base58
-- name: github.com/tendermint/merkleeyes
-  version: 2a93256d2c6fbcc3b55673c0d2b96a7e32c6238b
+  - nowriter/tmlegacy
+- name: github.com/tendermint/iavl
+  version: 594cc0c062a7174475f0ab654384038d77067917
   subpackages:
   - iavl
 - name: github.com/tendermint/tmlibs
-  version: 9997e3a3b46db1d2f88aa9816ed0e7915dad6ac1
+  version: e4ef2835f0081c2ece83b9c1f777cf071f956e81
   subpackages:
   - autofile
   - cli
@@ -130,13 +137,14 @@ imports:
   - clist
   - common
   - db
-  - events
   - flowrate
   - log
   - merkle
+  - pubsub
+  - pubsub/query
   - test
 - name: golang.org/x/crypto
-  version: 7d9177d70076375b9a59c8fde23d52d9c4a7ecd5
+  version: 94eea52f7b742c7cbe0b03b22f0c4c8631ece122
   subpackages:
   - curve25519
   - nacl/box
@@ -147,7 +155,7 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: 0744d001aa8470aaa53df28d32e5ceeb8af9bd70
+  version: a8b9294777976932365dabb6640cf1468d95c70f
   subpackages:
   - context
   - http2
@@ -157,22 +165,22 @@ imports:
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: 429f518978ab01db8bb6f44b66785088e7fba58b
+  version: 8b4580aae2a0dd0c231a45d3ccb8434ff533b840
   subpackages:
   - unix
 - name: golang.org/x/text
-  version: 1cbadb444a806fd9430d14ad08967ed91da4fa0a
+  version: 75cc3cad82b5f47d3fb229ddda8c5167da14f294
   subpackages:
   - secure/bidirule
   - transform
   - unicode/bidi
   - unicode/norm
 - name: google.golang.org/genproto
-  version: 1e559d0a00eef8a9a43151db4665280bd8dd5886
+  version: 7f0da29060c682909f650ad8ed4e515bd74fa12a
   subpackages:
   - googleapis/rpc/status
 - name: google.golang.org/grpc
-  version: d4b75ebd4f9f8c4a2b1cdadbdbe0d7920431ccca
+  version: 401e0e00e4bb830a10496d64cd95e068c5bf50de
   subpackages:
   - balancer
   - codes
@@ -191,9 +199,9 @@ imports:
   - tap
   - transport
 - name: gopkg.in/go-playground/validator.v9
-  version: a021b2ec9a8a8bb970f3f15bc42617cb520e8a64
+  version: 61caf9d3038e1af346dbf5c2e16f6678e1548364
 - name: gopkg.in/yaml.v2
-  version: eb3733d160e74a9c7e442f435eb3bea458e1d19f
+  version: 287cf08546ab5e7e37d55a84f7ed3fd1db036de5
 testImports:
 - name: github.com/davecgh/go-spew
   version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9
@@ -204,7 +212,7 @@ testImports:
   subpackages:
   - difflib
 - name: github.com/stretchr/testify
-  version: 890a5c3458b43e6104ff5da8dfa139d013d77544
+  version: 2aa2c176b9dab406a6970f6a55f513e8a8c8b18f
   subpackages:
   - assert
   - require

glide.yaml

@@ -2,35 +2,39 @@ package: github.com/tendermint/tendermint
 import:
 - package: github.com/ebuchman/fail-test
 - package: github.com/gogo/protobuf
+  version: v0.5
   subpackages:
   - proto
 - package: github.com/golang/protobuf
   subpackages:
   - proto
 - package: github.com/gorilla/websocket
+  version: v1.2.0
 - package: github.com/pkg/errors
   version: ~0.8.0
 - package: github.com/rcrowley/go-metrics
 - package: github.com/spf13/cobra
+  version: v0.0.1
 - package: github.com/spf13/viper
+  version: v1.0.0
 - package: github.com/tendermint/abci
-  version: ~0.6.0
+  version: ~v0.8.0
   subpackages:
   - client
   - example/dummy
   - types
 - package: github.com/tendermint/go-crypto
-  version: ~0.3.0
+  version: ~0.4.1
 - package: github.com/tendermint/go-wire
-  version: ~0.6.2
+  version: ~0.7.2
   subpackages:
   - data
-- package: github.com/tendermint/merkleeyes
-  version: master
+- package: github.com/tendermint/iavl
+  version: ~0.2.0
   subpackages:
   - iavl
 - package: github.com/tendermint/tmlibs
-  version: ~0.3.1
+  version: e4ef2835f0081c2ece83b9c1f777cf071f956e81
   subpackages:
   - autofile
   - cli
@@ -41,6 +45,7 @@ import:
   - flowrate
   - log
   - merkle
+  - pubsub
 - package: golang.org/x/crypto
   subpackages:
   - nacl/box
@@ -50,6 +55,7 @@ import:
   subpackages:
   - context
 - package: google.golang.org/grpc
+  version: v1.7.3
 testImport:
 - package: github.com/go-kit/kit
   subpackages:

lite/client/main_test.go

@@ -0,0 +1,25 @@
+package client_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/tendermint/abci/example/dummy"
+
+	nm "github.com/tendermint/tendermint/node"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+var node *nm.Node
+
+func TestMain(m *testing.M) {
+	// start a tendermint node (and merkleeyes) in the background to test against
+	app := dummy.NewDummyApplication()
+	node = rpctest.StartTendermint(app)
+	code := m.Run()
+
+	// and shut down proper at the end
+	node.Stop()
+	node.Wait()
+	os.Exit(code)
+}

lite/client/provider.go

@@ -0,0 +1,141 @@
+/*
+Package client defines a provider that uses a rpcclient
+to get information, which is used to get new headers
+and validators directly from a node.
+*/
+package client
+
+import (
+	"bytes"
+
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// SignStatusClient combines a SignClient and StatusClient.
+type SignStatusClient interface {
+	rpcclient.SignClient
+	rpcclient.StatusClient
+}
+
+type provider struct {
+	node       SignStatusClient
+	lastHeight int64
+}
+
+// NewProvider can wrap any rpcclient to expose it as
+// a read-only provider.
+func NewProvider(node SignStatusClient) lite.Provider {
+	return &provider{node: node}
+}
+
+// NewHTTPProvider can connect to a tendermint json-rpc endpoint
+// at the given url, and uses that as a read-only provider.
+func NewHTTPProvider(remote string) lite.Provider {
+	return &provider{
+		node: rpcclient.NewHTTP(remote, "/websocket"),
+	}
+}
+
+// StatusClient returns the internal node as a StatusClient
+func (p *provider) StatusClient() rpcclient.StatusClient {
+	return p.node
+}
+
+// StoreCommit is a noop, as clients can only read from the chain...
+func (p *provider) StoreCommit(_ lite.FullCommit) error { return nil }
+
+// GetHash gets the most recent validator and sees if it matches
+//
+// TODO: improve when the rpc interface supports more functionality
+func (p *provider) GetByHash(hash []byte) (lite.FullCommit, error) {
+	var fc lite.FullCommit
+	vals, err := p.node.Validators(nil)
+	// if we get no validators, or a different height, return an error
+	if err != nil {
+		return fc, err
+	}
+	p.updateHeight(vals.BlockHeight)
+	vhash := types.NewValidatorSet(vals.Validators).Hash()
+	if !bytes.Equal(hash, vhash) {
+		return fc, liteErr.ErrCommitNotFound()
+	}
+	return p.seedFromVals(vals)
+}
+
+// GetByHeight gets the validator set by height
+func (p *provider) GetByHeight(h int64) (fc lite.FullCommit, err error) {
+	commit, err := p.node.Commit(&h)
+	if err != nil {
+		return fc, err
+	}
+	return p.seedFromCommit(commit)
+}
+
+// LatestCommit returns the newest commit stored.
+func (p *provider) LatestCommit() (fc lite.FullCommit, err error) {
+	commit, err := p.GetLatestCommit()
+	if err != nil {
+		return fc, err
+	}
+	return p.seedFromCommit(commit)
+}
+
+// GetLatestCommit should return the most recent commit there is,
+// which handles queries for future heights as per the semantics
+// of GetByHeight.
+func (p *provider) GetLatestCommit() (*ctypes.ResultCommit, error) {
+	status, err := p.node.Status()
+	if err != nil {
+		return nil, err
+	}
+	return p.node.Commit(&status.LatestBlockHeight)
+}
+
+// CommitFromResult ...
+func CommitFromResult(result *ctypes.ResultCommit) lite.Commit {
+	return (lite.Commit)(result.SignedHeader)
+}
+
+func (p *provider) seedFromVals(vals *ctypes.ResultValidators) (lite.FullCommit, error) {
+	// now get the commits and build a full commit
+	commit, err := p.node.Commit(&vals.BlockHeight)
+	if err != nil {
+		return lite.FullCommit{}, err
+	}
+	fc := lite.NewFullCommit(
+		CommitFromResult(commit),
+		types.NewValidatorSet(vals.Validators),
+	)
+	return fc, nil
+}
+
+func (p *provider) seedFromCommit(commit *ctypes.ResultCommit) (fc lite.FullCommit, err error) {
+	fc.Commit = CommitFromResult(commit)
+
+	// now get the proper validators
+	vals, err := p.node.Validators(&commit.Header.Height)
+	if err != nil {
+		return fc, err
+	}
+
+	// make sure they match the commit (as we cannot enforce height)
+	vset := types.NewValidatorSet(vals.Validators)
+	if !bytes.Equal(vset.Hash(), commit.Header.ValidatorsHash) {
+		return fc, liteErr.ErrValidatorsChanged()
+	}
+
+	p.updateHeight(commit.Header.Height)
+	fc.Validators = vset
+	return fc, nil
+}
+
+func (p *provider) updateHeight(h int64) {
+	if h > p.lastHeight {
+		p.lastHeight = h
+	}
+}

lite/client/provider_test.go

@@ -0,0 +1,61 @@
+package client
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+func TestProvider(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	cfg := rpctest.GetConfig()
+	rpcAddr := cfg.RPC.ListenAddress
+	chainID := cfg.ChainID
+	p := NewHTTPProvider(rpcAddr)
+	require.NotNil(t, p)
+
+	// let it produce some blocks
+	err := rpcclient.WaitForHeight(p.(*provider).node, 6, nil)
+	require.Nil(err)
+
+	// let's get the highest block
+	seed, err := p.LatestCommit()
+
+	require.Nil(err, "%+v", err)
+	sh := seed.Height()
+	vhash := seed.Header.ValidatorsHash
+	assert.True(sh < 5000)
+
+	// let's check this is valid somehow
+	assert.Nil(seed.ValidateBasic(chainID))
+	cert := lite.NewStatic(chainID, seed.Validators)
+
+	// historical queries now work :)
+	lower := sh - 5
+	seed, err = p.GetByHeight(lower)
+	assert.Nil(err, "%+v", err)
+	assert.Equal(lower, seed.Height())
+
+	// also get by hash (given the match)
+	seed, err = p.GetByHash(vhash)
+	require.Nil(err, "%+v", err)
+	require.Equal(vhash, seed.Header.ValidatorsHash)
+	err = cert.Certify(seed.Commit)
+	assert.Nil(err, "%+v", err)
+
+	// get by hash fails without match
+	seed, err = p.GetByHash([]byte("foobar"))
+	assert.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+
+	// storing the seed silently ignored
+	err = p.StoreCommit(seed)
+	assert.Nil(err, "%+v", err)
+}

lite/commit.go

@@ -0,0 +1,99 @@
+package lite
+
+import (
+	"bytes"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/types"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// Certifier checks the votes to make sure the block really is signed properly.
+// Certifier must know the current set of validitors by some other means.
+type Certifier interface {
+	Certify(check Commit) error
+	ChainID() string
+}
+
+// Commit is basically the rpc /commit response, but extended
+//
+// This is the basepoint for proving anything on the blockchain. It contains
+// a signed header.  If the signatures are valid and > 2/3 of the known set,
+// we can store this checkpoint and use it to prove any number of aspects of
+// the system: such as txs, abci state, validator sets, etc...
+type Commit types.SignedHeader
+
+// FullCommit is a commit and the actual validator set,
+// the base info you need to update to a given point,
+// assuming knowledge of some previous validator set
+type FullCommit struct {
+	Commit     `json:"commit"`
+	Validators *types.ValidatorSet `json:"validator_set"`
+}
+
+// NewFullCommit returns a new FullCommit.
+func NewFullCommit(commit Commit, vals *types.ValidatorSet) FullCommit {
+	return FullCommit{
+		Commit:     commit,
+		Validators: vals,
+	}
+}
+
+// Height returns the height of the header.
+func (c Commit) Height() int64 {
+	if c.Header == nil {
+		return 0
+	}
+	return c.Header.Height
+}
+
+// ValidatorsHash returns the hash of the validator set.
+func (c Commit) ValidatorsHash() []byte {
+	if c.Header == nil {
+		return nil
+	}
+	return c.Header.ValidatorsHash
+}
+
+// ValidateBasic does basic consistency checks and makes sure the headers
+// and commits are all consistent and refer to our chain.
+//
+// Make sure to use a Verifier to validate the signatures actually provide
+// a significantly strong proof for this header's validity.
+func (c Commit) ValidateBasic(chainID string) error {
+	// make sure the header is reasonable
+	if c.Header == nil {
+		return errors.New("Commit missing header")
+	}
+	if c.Header.ChainID != chainID {
+		return errors.Errorf("Header belongs to another chain '%s' not '%s'",
+			c.Header.ChainID, chainID)
+	}
+
+	if c.Commit == nil {
+		return errors.New("Commit missing signatures")
+	}
+
+	// make sure the header and commit match (height and hash)
+	if c.Commit.Height() != c.Header.Height {
+		return liteErr.ErrHeightMismatch(c.Commit.Height(), c.Header.Height)
+	}
+	hhash := c.Header.Hash()
+	chash := c.Commit.BlockID.Hash
+	if !bytes.Equal(hhash, chash) {
+		return errors.Errorf("Commits sign block %X header is block %X",
+			chash, hhash)
+	}
+
+	// make sure the commit is reasonable
+	err := c.Commit.ValidateBasic()
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	// looks good, we just need to make sure the signatures are really from
+	// empowered validators
+	return nil
+}

lite/doc.go

@@ -0,0 +1,133 @@
+/*
+Package lite allows you to securely validate headers
+without a full node.
+
+This library pulls together all the crypto and algorithms,
+so given a relatively recent (< unbonding period) known
+validator set, one can get indisputable proof that data is in
+the chain (current state) or detect if the node is lying to
+the client.
+
+Tendermint RPC exposes a lot of info, but a malicious node
+could return any data it wants to queries, or even to block
+headers, even making up fake signatures from non-existent
+validators to justify it. This is a lot of logic to get
+right, to be contained in a small, easy to use library,
+that does this for you, so you can just build nice UI.
+
+We design for clients who have no strong trust relationship
+with any tendermint node, just the validator set as a whole.
+Beyond building nice mobile or desktop applications, the
+cosmos hub is another important example of a client,
+that needs undeniable proof without syncing the full chain,
+in order to efficiently implement IBC.
+
+Commits
+
+There are two main data structures that we pass around - Commit
+and FullCommit. Both of them mirror what information is
+exposed in tendermint rpc.
+
+Commit is a block header along with enough validator signatures
+to prove its validity (> 2/3 of the voting power). A FullCommit
+is a Commit along with the full validator set. When the
+validator set doesn't change, the Commit is enough, but since
+the block header only has a hash, we need the FullCommit to
+follow any changes to the validator set.
+
+Certifiers
+
+A Certifier validates a new Commit given the currently known
+state. There are three different types of Certifiers exposed,
+each one building on the last one, with additional complexity.
+
+Static - given the validator set upon initialization. Verifies
+all signatures against that set and if the validator set
+changes, it will reject all headers.
+
+Dynamic - This wraps Static and has the same Certify
+method. However, it adds an Update method, which can be called
+with a FullCommit when the validator set changes. If it can
+prove this is a valid transition, it will update the validator
+set.
+
+Inquiring - this wraps Dynamic and implements an auto-update
+strategy on top of the Dynamic update. If a call to
+Certify fails as the validator set has changed, then it
+attempts to find a FullCommit and Update to that header.
+To get these FullCommits, it makes use of a Provider.
+
+Providers
+
+A Provider allows us to store and retrieve the FullCommits,
+to provide memory to the Inquiring Certifier.
+
+NewMemStoreProvider - in-memory cache.
+
+files.NewProvider - disk backed storage.
+
+client.NewHTTPProvider - query tendermint rpc.
+
+NewCacheProvider - combine multiple providers.
+
+The suggested use for local light clients is
+client.NewHTTPProvider for getting new data (Source),
+and NewCacheProvider(NewMemStoreProvider(),
+files.NewProvider()) to store confirmed headers (Trusted)
+
+How We Track Validators
+
+Unless you want to blindly trust the node you talk with, you
+need to trace every response back to a hash in a block header
+and validate the commit signatures of that block header match
+the proper validator set.  If there is a contant validator
+set, you store it locally upon initialization of the client,
+and check against that every time.
+
+Once there is a dynamic validator set, the issue of
+verifying a block becomes a bit more tricky. There is
+background information in a
+github issue (https://github.com/tendermint/tendermint/issues/377).
+
+In short, if there is a block at height H with a known
+(trusted) validator set V, and another block at height H'
+(H' > H) with validator set V' != V, then we want a way to
+safely update it.
+
+First, get the new (unconfirmed) validator set V' and
+verify H' is internally consistent and properly signed by
+this V'. Assuming it is a valid block, we check that at
+least 2/3 of the validators in V also signed it, meaning
+it would also be valid under our old assumptions.
+That should be enough, but we can also check that the
+V counts for at least 2/3 of the total votes in H'
+for extra safety (we can have a discussion if this is
+strictly required). If we can verify all this,
+then we can accept H' and V' as valid and use that to
+validate all blocks X > H'.
+
+If we cannot update directly from H -> H' because there was
+too much change to the validator set, then we can look for
+some Hm (H < Hm < H') with a validator set Vm.  Then we try
+to update H -> Hm and Hm -> H' in two separate steps.
+If one of these steps doesn't work, then we continue
+bisecting, until we eventually have to externally
+validate the valdiator set changes at every block.
+
+Since we never trust any server in this protocol, only the
+signatures themselves, it doesn't matter if the seed comes
+from a (possibly malicious) node or a (possibly malicious) user.
+We can accept it or reject it based only on our trusted
+validator set and cryptographic proofs. This makes it
+extremely important to verify that you have the proper
+validator set when initializing the client, as that is the
+root of all trust.
+
+Or course, this assumes that the known block is within the
+unbonding period to avoid the "nothing at stake" problem.
+If you haven't seen the state in a few months, you will need
+to manually verify the new validator set hash using off-chain
+means (the same as getting the initial hash).
+
+*/
+package lite

lite/dynamic.go

@@ -0,0 +1,95 @@
+package lite
+
+import (
+	"github.com/tendermint/tendermint/types"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+var _ Certifier = &Dynamic{}
+
+// Dynamic uses a Static for Certify, but adds an
+// Update method to allow for a change of validators.
+//
+// You can pass in a FullCommit with another validator set,
+// and if this is a provably secure transition (< 1/3 change,
+// sufficient signatures), then it will update the
+// validator set for the next Certify call.
+// For security, it will only follow validator set changes
+// going forward.
+type Dynamic struct {
+	cert       *Static
+	lastHeight int64
+}
+
+// NewDynamic returns a new dynamic certifier.
+func NewDynamic(chainID string, vals *types.ValidatorSet, height int64) *Dynamic {
+	return &Dynamic{
+		cert:       NewStatic(chainID, vals),
+		lastHeight: height,
+	}
+}
+
+// ChainID returns the chain id of this certifier.
+func (c *Dynamic) ChainID() string {
+	return c.cert.ChainID()
+}
+
+// Validators returns the validators of this certifier.
+func (c *Dynamic) Validators() *types.ValidatorSet {
+	return c.cert.vSet
+}
+
+// Hash returns the hash of this certifier.
+func (c *Dynamic) Hash() []byte {
+	return c.cert.Hash()
+}
+
+// LastHeight returns the last height of this certifier.
+func (c *Dynamic) LastHeight() int64 {
+	return c.lastHeight
+}
+
+// Certify will verify whether the commit is valid and will update the height if it is or return an
+// error if it is not.
+func (c *Dynamic) Certify(check Commit) error {
+	err := c.cert.Certify(check)
+	if err == nil {
+		// update last seen height if input is valid
+		c.lastHeight = check.Height()
+	}
+	return err
+}
+
+// Update will verify if this is a valid change and update
+// the certifying validator set if safe to do so.
+//
+// Returns an error if update is impossible (invalid proof or IsTooMuchChangeErr)
+func (c *Dynamic) Update(fc FullCommit) error {
+	// ignore all checkpoints in the past -> only to the future
+	h := fc.Height()
+	if h <= c.lastHeight {
+		return liteErr.ErrPastTime()
+	}
+
+	// first, verify if the input is self-consistent....
+	err := fc.ValidateBasic(c.ChainID())
+	if err != nil {
+		return err
+	}
+
+	// now, make sure not too much change... meaning this commit
+	// would be approved by the currently known validator set
+	// as well as the new set
+	commit := fc.Commit.Commit
+	err = c.Validators().VerifyCommitAny(fc.Validators, c.ChainID(),
+		commit.BlockID, h, commit)
+	if err != nil {
+		return liteErr.ErrTooMuchChange()
+	}
+
+	// looks good, we can update
+	c.cert = NewStatic(c.ChainID(), fc.Validators)
+	c.lastHeight = h
+	return nil
+}

lite/dynamic_test.go

@@ -0,0 +1,130 @@
+package lite_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/types"
+
+	"github.com/tendermint/tendermint/lite"
+	"github.com/tendermint/tendermint/lite/errors"
+)
+
+// TestDynamicCert just makes sure it still works like StaticCert
+func TestDynamicCert(t *testing.T) {
+	// assert, require := assert.New(t), require.New(t)
+	assert := assert.New(t)
+	// require := require.New(t)
+
+	keys := lite.GenValKeys(4)
+	// 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do!
+	vals := keys.ToValidators(20, 10)
+	// and a certifier based on our known set
+	chainID := "test-dyno"
+	cert := lite.NewDynamic(chainID, vals, 0)
+
+	cases := []struct {
+		keys        lite.ValKeys
+		vals        *types.ValidatorSet
+		height      int64
+		first, last int  // who actually signs
+		proper      bool // true -> expect no error
+		changed     bool // true -> expect validator change error
+	}{
+		// perfect, signed by everyone
+		{keys, vals, 1, 0, len(keys), true, false},
+		// skip little guy is okay
+		{keys, vals, 2, 1, len(keys), true, false},
+		// but not the big guy
+		{keys, vals, 3, 0, len(keys) - 1, false, false},
+		// even changing the power a little bit breaks the static validator
+		// the sigs are enough, but the validator hash is unknown
+		{keys, keys.ToValidators(20, 11), 4, 0, len(keys), false, true},
+	}
+
+	for _, tc := range cases {
+		check := tc.keys.GenCommit(chainID, tc.height, nil, tc.vals,
+			[]byte("bar"), tc.first, tc.last)
+		err := cert.Certify(check)
+		if tc.proper {
+			assert.Nil(err, "%+v", err)
+			assert.Equal(cert.LastHeight(), tc.height)
+		} else {
+			assert.NotNil(err)
+			if tc.changed {
+				assert.True(errors.IsValidatorsChangedErr(err), "%+v", err)
+			}
+		}
+	}
+}
+
+// TestDynamicUpdate makes sure we update safely and sanely
+func TestDynamicUpdate(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	chainID := "test-dyno-up"
+	keys := lite.GenValKeys(5)
+	vals := keys.ToValidators(20, 0)
+	cert := lite.NewDynamic(chainID, vals, 40)
+
+	// one valid block to give us a sense of time
+	h := int64(100)
+	good := keys.GenCommit(chainID, h, nil, vals, []byte("foo"), 0, len(keys))
+	err := cert.Certify(good)
+	require.Nil(err, "%+v", err)
+
+	// some new sets to try later
+	keys2 := keys.Extend(2)
+	keys3 := keys2.Extend(4)
+
+	// we try to update with some blocks
+	cases := []struct {
+		keys        lite.ValKeys
+		vals        *types.ValidatorSet
+		height      int64
+		first, last int  // who actually signs
+		proper      bool // true -> expect no error
+		changed     bool // true -> expect too much change error
+	}{
+		// same validator set, well signed, of course it is okay
+		{keys, vals, h + 10, 0, len(keys), true, false},
+		// same validator set, poorly signed, fails
+		{keys, vals, h + 20, 2, len(keys), false, false},
+
+		// shift the power a little, works if properly signed
+		{keys, keys.ToValidators(10, 0), h + 30, 1, len(keys), true, false},
+		// but not on a poor signature
+		{keys, keys.ToValidators(10, 0), h + 40, 2, len(keys), false, false},
+		// and not if it was in the past
+		{keys, keys.ToValidators(10, 0), h + 25, 0, len(keys), false, false},
+
+		// let's try to adjust to a whole new validator set (we have 5/7 of the votes)
+		{keys2, keys2.ToValidators(10, 0), h + 33, 0, len(keys2), true, false},
+
+		// properly signed but too much change, not allowed (only 7/11 validators known)
+		{keys3, keys3.ToValidators(10, 0), h + 50, 0, len(keys3), false, true},
+	}
+
+	for _, tc := range cases {
+		fc := tc.keys.GenFullCommit(chainID, tc.height, nil, tc.vals,
+			[]byte("bar"), tc.first, tc.last)
+		err := cert.Update(fc)
+		if tc.proper {
+			assert.Nil(err, "%d: %+v", tc.height, err)
+			// we update last seen height
+			assert.Equal(cert.LastHeight(), tc.height)
+			// and we update the proper validators
+			assert.EqualValues(fc.Header.ValidatorsHash, cert.Hash())
+		} else {
+			assert.NotNil(err, "%d", tc.height)
+			// we don't update the height
+			assert.NotEqual(cert.LastHeight(), tc.height)
+			if tc.changed {
+				assert.True(errors.IsTooMuchChangeErr(err),
+					"%d: %+v", tc.height, err)
+			}
+		}
+	}
+}

lite/errors/errors.go

@@ -0,0 +1,92 @@
+package errors
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	errValidatorsChanged = fmt.Errorf("Validators differ between header and certifier")
+	errCommitNotFound    = fmt.Errorf("Commit not found by provider")
+	errTooMuchChange     = fmt.Errorf("Validators change too much to safely update")
+	errPastTime          = fmt.Errorf("Update older than certifier height")
+	errNoPathFound       = fmt.Errorf("Cannot find a path of validators")
+)
+
+// IsCommitNotFoundErr checks whether an error is due to missing data
+func IsCommitNotFoundErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errCommitNotFound)
+}
+
+// ErrCommitNotFound indicates that a the requested commit was not found.
+func ErrCommitNotFound() error {
+	return errors.WithStack(errCommitNotFound)
+}
+
+// IsValidatorsChangedErr checks whether an error is due
+// to a differing validator set.
+func IsValidatorsChangedErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errValidatorsChanged)
+}
+
+// ErrValidatorsChanged indicates that the validator set was changed between two commits.
+func ErrValidatorsChanged() error {
+	return errors.WithStack(errValidatorsChanged)
+}
+
+// IsTooMuchChangeErr checks whether an error is due to too much change
+// between these validators sets.
+func IsTooMuchChangeErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errTooMuchChange)
+}
+
+// ErrTooMuchChange indicates that the underlying validator set was changed by >1/3.
+func ErrTooMuchChange() error {
+	return errors.WithStack(errTooMuchChange)
+}
+
+// IsPastTimeErr ...
+func IsPastTimeErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errPastTime)
+}
+
+// ErrPastTime ...
+func ErrPastTime() error {
+	return errors.WithStack(errPastTime)
+}
+
+// IsNoPathFoundErr checks whether an error is due to no path of
+// validators in provider from where we are to where we want to be
+func IsNoPathFoundErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errNoPathFound)
+}
+
+// ErrNoPathFound ...
+func ErrNoPathFound() error {
+	return errors.WithStack(errNoPathFound)
+}
+
+//--------------------------------------------
+
+type errHeightMismatch struct {
+	h1, h2 int64
+}
+
+func (e errHeightMismatch) Error() string {
+	return fmt.Sprintf("Blocks don't match - %d vs %d", e.h1, e.h2)
+}
+
+// IsHeightMismatchErr checks whether an error is due to data from different blocks
+func IsHeightMismatchErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, ok := errors.Cause(err).(errHeightMismatch)
+	return ok
+}
+
+// ErrHeightMismatch returns an mismatch error with stack-trace
+func ErrHeightMismatch(h1, h2 int64) error {
+	return errors.WithStack(errHeightMismatch{h1, h2})
+}

lite/errors/errors_test.go

@@ -0,0 +1,18 @@
+package errors
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestErrorHeight(t *testing.T) {
+	e1 := ErrHeightMismatch(2, 3)
+	e1.Error()
+	assert.True(t, IsHeightMismatchErr(e1))
+
+	e2 := errors.New("foobar")
+	assert.False(t, IsHeightMismatchErr(e2))
+	assert.False(t, IsHeightMismatchErr(nil))
+}

lite/files/commit.go

@@ -0,0 +1,79 @@
+package files
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/pkg/errors"
+
+	wire "github.com/tendermint/go-wire"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+const (
+	// MaxFullCommitSize is the maximum number of bytes we will
+	// read in for a full commit to avoid excessive allocations
+	// in the deserializer
+	MaxFullCommitSize = 1024 * 1024
+)
+
+// SaveFullCommit exports the seed in binary / go-wire style
+func SaveFullCommit(fc lite.FullCommit, path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer f.Close()
+
+	var n int
+	wire.WriteBinary(fc, f, &n, &err)
+	return errors.WithStack(err)
+}
+
+// SaveFullCommitJSON exports the seed in a json format
+func SaveFullCommitJSON(fc lite.FullCommit, path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer f.Close()
+	stream := json.NewEncoder(f)
+	err = stream.Encode(fc)
+	return errors.WithStack(err)
+}
+
+// LoadFullCommit loads the full commit from the file system.
+func LoadFullCommit(path string) (lite.FullCommit, error) {
+	var fc lite.FullCommit
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fc, liteErr.ErrCommitNotFound()
+		}
+		return fc, errors.WithStack(err)
+	}
+	defer f.Close()
+
+	var n int
+	wire.ReadBinaryPtr(&fc, f, MaxFullCommitSize, &n, &err)
+	return fc, errors.WithStack(err)
+}
+
+// LoadFullCommitJSON loads the commit from the file system in JSON format.
+func LoadFullCommitJSON(path string) (lite.FullCommit, error) {
+	var fc lite.FullCommit
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fc, liteErr.ErrCommitNotFound()
+		}
+		return fc, errors.WithStack(err)
+	}
+	defer f.Close()
+
+	stream := json.NewDecoder(f)
+	err = stream.Decode(&fc)
+	return fc, errors.WithStack(err)
+}

lite/files/commit_test.go

@@ -0,0 +1,66 @@
+package files
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/lite"
+)
+
+func tmpFile() string {
+	suffix := cmn.RandStr(16)
+	return filepath.Join(os.TempDir(), "fc-test-"+suffix)
+}
+
+func TestSerializeFullCommits(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// some constants
+	appHash := []byte("some crazy thing")
+	chainID := "ser-ial"
+	h := int64(25)
+
+	// build a fc
+	keys := lite.GenValKeys(5)
+	vals := keys.ToValidators(10, 0)
+	fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, 0, 5)
+
+	require.Equal(h, fc.Height())
+	require.Equal(vals.Hash(), fc.ValidatorsHash())
+
+	// try read/write with json
+	jfile := tmpFile()
+	defer os.Remove(jfile)
+	jseed, err := LoadFullCommitJSON(jfile)
+	assert.NotNil(err)
+	err = SaveFullCommitJSON(fc, jfile)
+	require.Nil(err)
+	jseed, err = LoadFullCommitJSON(jfile)
+	assert.Nil(err, "%+v", err)
+	assert.Equal(h, jseed.Height())
+	assert.Equal(vals.Hash(), jseed.ValidatorsHash())
+
+	// try read/write with binary
+	bfile := tmpFile()
+	defer os.Remove(bfile)
+	bseed, err := LoadFullCommit(bfile)
+	assert.NotNil(err)
+	err = SaveFullCommit(fc, bfile)
+	require.Nil(err)
+	bseed, err = LoadFullCommit(bfile)
+	assert.Nil(err, "%+v", err)
+	assert.Equal(h, bseed.Height())
+	assert.Equal(vals.Hash(), bseed.ValidatorsHash())
+
+	// make sure they don't read the other format (different)
+	_, err = LoadFullCommit(jfile)
+	assert.NotNil(err)
+	_, err = LoadFullCommitJSON(bfile)
+	assert.NotNil(err)
+}

lite/files/provider.go

@@ -0,0 +1,139 @@
+/*
+Package files defines a Provider that stores all data in the filesystem
+
+We assume the same validator hash may be reused by many different
+headers/Commits, and thus store it separately. This leaves us
+with three issues:
+
+  1. Given a validator hash, retrieve the validator set if previously stored
+  2. Given a block height, find the Commit with the highest height <= h
+  3. Given a FullCommit, store it quickly to satisfy 1 and 2
+
+Note that we do not worry about caching, as that can be achieved by
+pairing this with a MemStoreProvider and CacheProvider from certifiers
+*/
+package files
+
+import (
+	"encoding/hex"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// nolint
+const (
+	Ext      = ".tsd"
+	ValDir   = "validators"
+	CheckDir = "checkpoints"
+	dirPerm  = os.FileMode(0755)
+	//filePerm = os.FileMode(0644)
+)
+
+type provider struct {
+	valDir   string
+	checkDir string
+}
+
+// NewProvider creates the parent dir and subdirs
+// for validators and checkpoints as needed
+func NewProvider(dir string) lite.Provider {
+	valDir := filepath.Join(dir, ValDir)
+	checkDir := filepath.Join(dir, CheckDir)
+	for _, d := range []string{valDir, checkDir} {
+		err := os.MkdirAll(d, dirPerm)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return &provider{valDir: valDir, checkDir: checkDir}
+}
+
+func (p *provider) encodeHash(hash []byte) string {
+	return hex.EncodeToString(hash) + Ext
+}
+
+func (p *provider) encodeHeight(h int64) string {
+	// pad up to 10^12 for height...
+	return fmt.Sprintf("%012d%s", h, Ext)
+}
+
+// StoreCommit saves a full commit after it has been verified.
+func (p *provider) StoreCommit(fc lite.FullCommit) error {
+	// make sure the fc is self-consistent before saving
+	err := fc.ValidateBasic(fc.Commit.Header.ChainID)
+	if err != nil {
+		return err
+	}
+
+	paths := []string{
+		filepath.Join(p.checkDir, p.encodeHeight(fc.Height())),
+		filepath.Join(p.valDir, p.encodeHash(fc.Header.ValidatorsHash)),
+	}
+	for _, path := range paths {
+		err := SaveFullCommit(fc, path)
+		// unknown error in creating or writing immediately breaks
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// GetByHeight returns the closest commit with height <= h.
+func (p *provider) GetByHeight(h int64) (lite.FullCommit, error) {
+	// first we look for exact match, then search...
+	path := filepath.Join(p.checkDir, p.encodeHeight(h))
+	fc, err := LoadFullCommit(path)
+	if liteErr.IsCommitNotFoundErr(err) {
+		path, err = p.searchForHeight(h)
+		if err == nil {
+			fc, err = LoadFullCommit(path)
+		}
+	}
+	return fc, err
+}
+
+// LatestCommit returns the newest commit stored.
+func (p *provider) LatestCommit() (fc lite.FullCommit, err error) {
+	// Note to future: please update by 2077 to avoid rollover
+	return p.GetByHeight(math.MaxInt32 - 1)
+}
+
+// search for height, looks for a file with highest height < h
+// return certifiers.ErrCommitNotFound() if not there...
+func (p *provider) searchForHeight(h int64) (string, error) {
+	d, err := os.Open(p.checkDir)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	files, err := d.Readdirnames(0)
+
+	d.Close()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	desired := p.encodeHeight(h)
+	sort.Strings(files)
+	i := sort.SearchStrings(files, desired)
+	if i == 0 {
+		return "", liteErr.ErrCommitNotFound()
+	}
+	found := files[i-1]
+	path := filepath.Join(p.checkDir, found)
+	return path, errors.WithStack(err)
+}
+
+// GetByHash returns a commit exactly matching this validator hash.
+func (p *provider) GetByHash(hash []byte) (lite.FullCommit, error) {
+	path := filepath.Join(p.valDir, p.encodeHash(hash))
+	return LoadFullCommit(path)
+}