fixes

Otherwise, we'll be trying to include them in each consecutive block.

The downside is that evil proposers will be able to drop valid
transactions (see #3322).

Reverts https://github.com/tendermint/tendermint/pull/3625

Fixes #3699

.circleci/config.yml

@@ -3,10 +3,20 @@ version: 2
 defaults: &defaults
   working_directory: /go/src/github.com/tendermint/tendermint
   docker:
-    - image: circleci/golang:1.10.3
+    - image: circleci/golang
   environment:
     GOBIN: /tmp/workspace/bin
 
+docs_update_config: &docs_update_config
+  working_directory: ~/repo
+  docker:
+    - image: tendermintdev/jq_curl
+  environment:
+    AWS_REGION: us-east-1
+
+release_management_docker: &release_management_docker
+  machine: true
+
 jobs:
   setup_dependencies:
     <<: *defaults
@@ -41,10 +51,10 @@ jobs:
           key: v3-pkg-cache
           paths:
             - /go/pkg
-      # - save_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      #     paths:
-      #       - /go/src/github.com/tendermint/tendermint
+      - save_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
+          paths:
+            - /go/src/github.com/tendermint/tendermint
 
   build_slate:
     <<: *defaults
@@ -53,23 +63,8 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # https://discuss.circleci.com/t/saving-cache-stopped-working-warning-skipping-this-step-disabled-in-configuration/24423/2
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: slate docs
           command: |
@@ -84,28 +79,14 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: metalinter
           command: |
             set -ex
             export PATH="$GOBIN:$PATH"
-            make metalinter
+            make lint
       - run:
           name: check_dep
           command: |
@@ -120,22 +101,8 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run abci apps tests
           command: |
@@ -151,22 +118,8 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run abci-cli tests
           command: |
@@ -180,22 +133,8 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils
       - run:
           name: Run tests
@@ -209,22 +148,8 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run: mkdir -p /tmp/logs
       - run:
           name: Run tests
@@ -232,7 +157,7 @@ jobs:
             for pkg in $(go list github.com/tendermint/tendermint/... | circleci tests split --split-by=timings); do
               id=$(basename "$pkg")
 
-              GOCACHE=off go test -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
+              go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
             done
       - persist_to_workspace:
           root: /tmp/workspace
@@ -248,22 +173,8 @@ jobs:
           at: /tmp/workspace
       - restore_cache:
           key: v3-pkg-cache
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run tests
           command: bash test/persist/test_failure_indices.sh
@@ -284,9 +195,7 @@ jobs:
             name: run localnet and exit on failure
             command: |
               set -x
-              make get_tools
-              make get_vendor_deps
-              make build-linux
+              docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang make build-linux
               make localnet-start &
               ./scripts/localnet-blocks-test.sh 40 5 10 localhost
 
@@ -309,22 +218,10 @@ jobs:
     steps:
       - attach_workspace:
           at: /tmp/workspace
-      # - restore_cache:
-      #     key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
-      - checkout
-      - run:
-          name: tools
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_tools
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-
+      - restore_cache:
+          key: v3-pkg-cache
+      - restore_cache:
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: gather
           command: |
@@ -338,10 +235,139 @@ jobs:
           name: upload
           command: bash .circleci/codecov.sh -f coverage.txt
 
+  deploy_docs:
+    <<: *docs_update_config
+    steps:
+      - checkout
+      - run:
+          name: Trigger website build
+          command: |
+            curl --silent \
+                 --show-error \
+                 -X POST \
+                 --header "Content-Type: application/json" \
+                 -d "{\"branch\": \"$CIRCLE_BRANCH\"}" \
+                 "https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$WEBSITE_REPO_NAME/build?circle-token=$TENDERBOT_API_TOKEN" > response.json
+
+            RESULT=`jq -r '.status' response.json`
+            MESSAGE=`jq -r '.message' response.json`
+
+            if [[ ${RESULT} == "null" ]] || [[ ${RESULT} -ne "200" ]]; then
+                echo "CircleCI API call failed: $MESSAGE"
+                exit 1
+            else
+                echo "Website build started"
+            fi
+
+  prepare_build:
+    <<: *defaults
+    steps:
+      - checkout
+      - run:
+          name: Get next release number
+          command: |
+            export LAST_TAG="`git describe --tags --abbrev=0 --match "${CIRCLE_BRANCH}.*"`"
+            echo "Last tag: ${LAST_TAG}"
+            if [ -z "${LAST_TAG}" ]; then
+              export LAST_TAG="${CIRCLE_BRANCH}"
+              echo "Last tag not found. Possibly fresh branch or feature branch. Setting ${LAST_TAG} as tag."
+            fi
+            export NEXT_TAG="`python -u scripts/release_management/bump-semver.py --version "${LAST_TAG}"`"
+            echo "Next tag: ${NEXT_TAG}"
+            echo "export CIRCLE_TAG=\"${NEXT_TAG}\"" > release-version.source
+      - run:
+          name: Build dependencies
+          command: |
+            make get_tools get_vendor_deps
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "release-version.source"
+      - save_cache:
+          key: v1-release-deps-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - "vendor"
+
+  build_artifacts:
+    <<: *defaults
+    parallelism: 4
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-release-deps-{{ .Branch }}-{{ .Revision }}
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Build artifact
+          command: |
+            # Setting CIRCLE_TAG because we do not tag the release ourselves.
+            source /tmp/workspace/release-version.source
+            if test ${CIRCLE_NODE_INDEX:-0} == 0 ;then export GOOS=linux GOARCH=amd64   && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
+            if test ${CIRCLE_NODE_INDEX:-0} == 1 ;then export GOOS=darwin GOARCH=amd64  && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
+            if test ${CIRCLE_NODE_INDEX:-0} == 2 ;then export GOOS=windows GOARCH=amd64 && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
+            if test ${CIRCLE_NODE_INDEX:-0} == 3 ;then export GOOS=linux GOARCH=arm     && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
+      - persist_to_workspace:
+          root: build
+          paths:
+            - "*.zip"
+            - "tendermint_linux_amd64"
+
+  release_artifacts:
+    <<: *defaults
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Deploy to GitHub
+          command: |
+            # Setting CIRCLE_TAG because we do not tag the release ourselves.
+            source /tmp/workspace/release-version.source
+            echo "---"
+            ls -la /tmp/workspace/*.zip
+            echo "---"
+            python -u scripts/release_management/sha-files.py
+            echo "---"
+            cat /tmp/workspace/SHA256SUMS
+            echo "---"
+            export RELEASE_ID="`python -u scripts/release_management/github-draft.py`"
+            echo "Release ID: ${RELEASE_ID}"
+            #Todo: Parallelize uploads
+            export GOOS=linux GOARCH=amd64   && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
+            export GOOS=darwin GOARCH=amd64  && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
+            export GOOS=windows GOARCH=amd64 && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
+            export GOOS=linux GOARCH=arm     && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
+            python -u scripts/release_management/github-upload.py --file "/tmp/workspace/SHA256SUMS" --id "${RELEASE_ID}"
+            python -u scripts/release_management/github-publish.py --id "${RELEASE_ID}"
+
+  release_docker:
+    <<: *release_management_docker
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Deploy to Docker Hub
+          command: |
+            # Setting CIRCLE_TAG because we do not tag the release ourselves.
+            source /tmp/workspace/release-version.source
+            cp /tmp/workspace/tendermint_linux_amd64 DOCKER/tendermint
+            docker build --label="tendermint" --tag="tendermint/tendermint:${CIRCLE_TAG}" --tag="tendermint/tendermint:latest" "DOCKER"
+            docker login -u "${DOCKERHUB_USER}" --password-stdin <<< "${DOCKERHUB_PASS}"
+            docker push "tendermint/tendermint"
+            docker logout
+
 workflows:
   version: 2
   test-suite:
     jobs:
+      - deploy_docs:
+          filters:
+            branches:
+              only:
+                - master
+                - develop
       - setup_dependencies
       - lint:
           requires:
@@ -368,3 +394,25 @@ workflows:
       - upload_coverage:
           requires:
             - test_cover
+  release:
+    jobs:
+      - prepare_build
+      - build_artifacts:
+          requires:
+            - prepare_build
+      - release_artifacts:
+          requires:
+            - prepare_build
+            - build_artifacts
+          filters:
+            branches:
+              only:
+                - /v[0-9]+\.[0-9]+/
+      - release_docker:
+          requires:
+            - prepare_build
+            - build_artifacts
+          filters:
+            branches:
+              only:
+                - /v[0-9]+\.[0-9]+/

.github/CODEOWNERS

@@ -4,4 +4,6 @@
 *       @ebuchman @melekes @xla
 
 # Precious documentation
-/docs/  @zramsay
+/docs/README.md  @zramsay
+/docs/DOCS_README.md  @zramsay
+/docs/.vuepress/  @zramsay

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,12 @@
-<!-- Thanks for filing a PR! Before hitting the button, please check the following items.-->
+<!--
 
+Thanks for filing a PR! Before hitting the button, please check the following items.
+Please note that every non-trivial PR must reference an issue that explains the
+changes in the PR.
+
+-->
+
+* [ ] Referenced an issue explaining the need for the change
 * [ ] Updated all relevant documentation in docs
 * [ ] Updated all code comments where relevant
 * [ ] Wrote tests

.gitignore

@@ -35,6 +35,7 @@ shunit2
 addrbook.json
 
 */vendor
+.vendor-new/
 */.glide
 .terraform
 terraform.tfstate

.golangci.yml

@@ -0,0 +1,57 @@
+run:
+  deadline: 1m
+
+linters:
+  enable-all: true
+  disable:
+    - gocyclo
+    - golint
+    - maligned
+    - errcheck
+    - staticcheck
+    - interfacer
+    - unconvert
+    - goconst
+    - unparam
+    - nakedret
+    - lll
+    - gochecknoglobals
+    - gocritic
+    - gochecknoinits
+    - scopelint
+    - stylecheck
+
+# linters-settings:
+#   govet:
+#     check-shadowing: true
+#   golint:
+#     min-confidence: 0
+#   gocyclo:
+#     min-complexity: 10
+#   maligned:
+#     suggest-new: true
+#   dupl:
+#     threshold: 100
+#   goconst:
+#     min-len: 2
+#     min-occurrences: 2
+#   depguard:
+#     list-type: blacklist
+#     packages:
+#       # logging is allowed only by logutils.Log, logrus
+#       # is allowed to use only in logutils package
+#       - github.com/sirupsen/logrus
+#   misspell:
+#     locale: US
+#   lll:
+#     line-length: 140
+#   goimports:
+#     local-prefixes: github.com/golangci/golangci-lint
+#   gocritic:
+#     enabled-tags:
+#       - performance
+#       - style
+#       - experimental
+#     disabled-checks:
+#       - wrapperFunc
+#       - commentFormatting # https://github.com/go-critic/go-critic/issues/755

CHANGELOG_PENDING.md

@@ -1,124 +1,23 @@
-# Pending
+## v0.31.7
 
-## v0.26.0
-
-*October 29, 2018*
-
-Special thanks to external contributors on this release:
-@bradyjoestar, @connorwstein, @goolAdapter, @HaoyangLiu,
-@james-ray, @overbool, @phymbert, @Slamper, @Uzair1995
-
-This release is primarily about adding Version fields to various data structures,
-optimizing consensus messages for signing and verification in
-restricted environments (like HSMs and the Ethereum Virtual Machine), and
-aligning the consensus code with the [specification](https://arxiv.org/abs/1807.04938).
-It also includes our first take at a generalized merkle proof system.
-
-See the [UPGRADING.md](UPGRADING.md#v0.26.0) for details on upgrading to the new
-version.
-
-Please note that we are still making breaking changes to the protocols.
-While the new Version fields should help us to keep the software backwards compatible
-even while upgrading the protocols, we cannot guarantee that new releases will
-be compatible with old chains just yet. Thanks for bearing with us!
-
-Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint).
+**
 
 ### BREAKING CHANGES:
 
 * CLI/RPC/Config
-  * [config] [\#2232](https://github.com/tendermint/tendermint/issues/2232) timeouts as time.Duration, not ints
-  * [config] [\#2505](https://github.com/tendermint/tendermint/issues/2505) Remove Mempool.RecheckEmpty (it was effectively useless anyways)
-  * [config] [\#2490](https://github.com/tendermint/tendermint/issues/2490) `mempool.wal` is disabled by default
-  * [privval] [\#2459](https://github.com/tendermint/tendermint/issues/2459) Split `SocketPVMsg`s implementations into Request and Response, where the Response may contain a error message (returned by the remote signer)
-  * [state] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Add Version field to State, breaking the format of State as
-    encoded on disk.
-  * [rpc] [\#2298](https://github.com/tendermint/tendermint/issues/2298) `/abci_query` takes `prove` argument instead of `trusted` and switches the default
-    behaviour to `prove=false`
-  * [rpc] [\#2654](https://github.com/tendermint/tendermint/issues/2654) Remove all `node_info.other.*_version` fields in `/status` and
-    `/net_info`
 
 * Apps
-  * [abci] [\#2298](https://github.com/tendermint/tendermint/issues/2298) ResponseQuery.Proof is now a structured merkle.Proof, not just
-    arbitrary bytes
-  * [abci] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Add Version to Header and shift all fields by one
-  * [abci] [\#2662](https://github.com/tendermint/tendermint/issues/2662) Bump the field numbers for some `ResponseInfo` fields to make room for
-      `AppVersion`
 
 * Go API
-  * [config] [\#2232](https://github.com/tendermint/tendermint/issues/2232) timeouts as time.Duration, not ints
-  * [crypto/merkle & lite] [\#2298](https://github.com/tendermint/tendermint/issues/2298) Various changes to accomodate General Merkle trees
-  * [crypto/merkle] [\#2595](https://github.com/tendermint/tendermint/issues/2595) Remove all Hasher objects in favor of byte slices
-  * [crypto/merkle] [\#2635](https://github.com/tendermint/tendermint/issues/2635) merkle.SimpleHashFromTwoHashes is no longer exported
-  * [node] [\#2479](https://github.com/tendermint/tendermint/issues/2479) Remove node.RunForever
-  * [rpc/client] [\#2298](https://github.com/tendermint/tendermint/issues/2298) `ABCIQueryOptions.Trusted` -> `ABCIQueryOptions.Prove`
-  * [types] [\#2298](https://github.com/tendermint/tendermint/issues/2298) Remove `Index` and `Total` fields from `TxProof`.
-  * [types] [\#2598](https://github.com/tendermint/tendermint/issues/2598) `VoteTypeXxx` are now of type `SignedMsgType byte` and named `XxxType`, eg. `PrevoteType`,
-    `PrecommitType`.
 
 * Blockchain Protocol
-  * [abci] [\#2636](https://github.com/tendermint/tendermint/issues/2636) Add ValidatorParams field to ConsensusParams.
-  (Used to control which pubkey types validators can use, by abci type)
-  * [types] Update SignBytes for `Vote`/`Proposal`/`Heartbeat`:
-    * [\#2459](https://github.com/tendermint/tendermint/issues/2459) Use amino encoding instead of JSON in `SignBytes`.
-    * [\#2598](https://github.com/tendermint/tendermint/issues/2598) Reorder fields and use fixed sized encoding.
-    * [\#2598](https://github.com/tendermint/tendermint/issues/2598) Change `Type` field fromt `string` to `byte` and use new
-      `SignedMsgType` to enumerate.
-  * [types] [\#2512](https://github.com/tendermint/tendermint/issues/2512) Remove the pubkey field from the validator hash
-  * [types] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Add Version struct to Header
-  * [types] [\#2609](https://github.com/tendermint/tendermint/issues/2609) ConsensusParams.Hash() is the hash of the amino encoded
-    struct instead of the Merkle tree of the fields
-  * [state] [\#2587](https://github.com/tendermint/tendermint/issues/2587) Require block.Time of the fist block to be genesis time
-  * [state] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Require block.Version to match state.Version
-  * [types] [\#2670](https://github.com/tendermint/tendermint/issues/2670) Header.Hash() builds Merkle tree out of fields in the same
-    order they appear in the header, instead of sorting by field name
-  * [types] [\#2682](https://github.com/tendermint/tendermint/issues/2682) Use proto3 `varint` encoding for ints that are usually unsigned (instead of zigzag encoding).
 
 * P2P Protocol
-  * [p2p] [\#2654](https://github.com/tendermint/tendermint/issues/2654) Add `ProtocolVersion` struct with protocol versions to top of
-    DefaultNodeInfo and require `ProtocolVersion.Block` to match during peer handshake
 
 ### FEATURES:
-- [abci] [\#2557](https://github.com/tendermint/tendermint/issues/2557) Add `Codespace` field to `Response{CheckTx, DeliverTx, Query}`
-- [abci] [\#2662](https://github.com/tendermint/tendermint/issues/2662) Add `BlockVersion` and `P2PVersion` to `RequestInfo`
-- [crypto/merkle] [\#2298](https://github.com/tendermint/tendermint/issues/2298) General Merkle Proof scheme for chaining various types of Merkle trees together
 
 ### IMPROVEMENTS:
-- Additional Metrics
-    - [consensus] [\#2169](https://github.com/cosmos/cosmos-sdk/issues/2169)
-    - [p2p] [\#2169](https://github.com/cosmos/cosmos-sdk/issues/2169)
-- [config] [\#2232](https://github.com/tendermint/tendermint/issues/2232) Added ValidateBasic method, which performs basic checks
-- [crypto/ed25519] [\#2558](https://github.com/tendermint/tendermint/issues/2558) Switch to use latest `golang.org/x/crypto` through our fork at
-  github.com/tendermint/crypto
-- [tools] [\#2238](https://github.com/tendermint/tendermint/issues/2238) Binary dependencies are now locked to a specific git commit
-- [libs/log] [\#2706](https://github.com/tendermint/tendermint/issues/2706) Add year to log format
-- [consensus] [\#2683] validate all incoming messages
-- [evidence] [\#2683] validate all incoming messages
-- [blockchain] [\#2683] validate all incoming messages
-- [p2p/pex] [\#2683] validate pexAddrsMessage addresses
 
 ### BUG FIXES:
-- [autofile] [\#2428](https://github.com/tendermint/tendermint/issues/2428) Group.RotateFile need call Flush() before rename (@goolAdapter)
-- [common] [\#2533](https://github.com/tendermint/tendermint/issues/2533) Fixed a bug in the `BitArray.Or` method
-- [common] [\#2506](https://github.com/tendermint/tendermint/issues/2506) Fixed a bug in the `BitArray.Sub` method (@james-ray)
-- [common] [\#2534](https://github.com/tendermint/tendermint/issues/2534) Fix `BitArray.PickRandom` to choose uniformly from true bits
-- [consensus] [\#1690](https://github.com/tendermint/tendermint/issues/1690) Wait for
-  timeoutPrecommit before starting next round
-- [consensus] [\#1745](https://github.com/tendermint/tendermint/issues/1745) Wait for
-  Proposal or timeoutProposal before entering prevote
-- [consensus] [\#2583](https://github.com/tendermint/tendermint/issues/2583) ensure valid 
-  block property with faulty proposer
-- [consensus] [\#2642](https://github.com/tendermint/tendermint/issues/2642) Only propose ValidBlock, not LockedBlock
-- [consensus] [\#2642](https://github.com/tendermint/tendermint/issues/2642) Initialized ValidRound and LockedRound to -1
-- [consensus] [\#1637](https://github.com/tendermint/tendermint/issues/1637) Limit the amount of evidence that can be included in a
-  block
-- [consensus] [\#2646](https://github.com/tendermint/tendermint/issues/2646) Simplify Proposal message (align with spec)
-- [crypto] [\#2733](https://github.com/tendermint/tendermint/pull/2733) Fix general merkle keypath to start w/ last op's key
-- [evidence] [\#2515](https://github.com/tendermint/tendermint/issues/2515) Fix db iter leak (@goolAdapter)
-- [libs/event] [\#2518](https://github.com/tendermint/tendermint/issues/2518) Fix event concurrency flaw (@goolAdapter)
-- [node] [\#2434](https://github.com/tendermint/tendermint/issues/2434) Make node respond to signal interrupts while sleeping for genesis time
-- [state] [\#2616](https://github.com/tendermint/tendermint/issues/2616) Pass nil to NewValidatorSet() when genesis file's Validators field is nil
-- [p2p] [\#2555](https://github.com/tendermint/tendermint/issues/2555) Fix p2p switch FlushThrottle value (@goolAdapter)
-- [p2p] [\#2668](https://github.com/tendermint/tendermint/issues/2668) Reconnect to originally dialed address (not self-reported
-  address) for persistent peers
-
+- [mempool] \#3699 Revert the change where we only remove valid transactions
+  from the mempool once a block is committed.

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ This code of conduct applies to all projects run by the Tendermint/COSMOS team a
 
 
 # Conduct
-## Contact: adrian@tendermint.com
+## Contact: conduct@tendermint.com
 
 * We are committed to providing a friendly, safe and welcoming environment for all, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, nationality, or other similar characteristic.
 

CONTRIBUTING.md

@@ -4,6 +4,14 @@ Thank you for considering making contributions to Tendermint and related reposit
 
 Please follow standard github best practices: fork the repo, branch from the tip of develop, make some commits, and submit a pull request to develop. See the [open issues](https://github.com/tendermint/tendermint/issues) for things we need help with!
 
+Before making a pull request, please open an issue describing the
+change you would like to make. If an issue for your change already exists,
+please comment on it that you will submit a pull request. Be sure to reference the issue in the opening
+comment of your pull request. If your change is substantial, you will be asked
+to write a more detailed design document in the form of an
+Architectural Decision Record (ie. see [here](./docs/architecture/)) before submitting code
+changes.
+
 Please make sure to use `gofmt` before every commit - the easiest way to do this is have your editor run it for you upon saving a file.
 
 ## Forking
@@ -27,8 +35,8 @@ Of course, replace `ebuchman` with your git handle.
 
 To pull in updates from the origin repo, run
 
-    * `git fetch upstream`
-    * `git rebase upstream/master` (or whatever branch you want)
+  * `git fetch upstream`
+  * `git rebase upstream/master` (or whatever branch you want)
 
 Please don't make Pull Requests to `master`.
 
@@ -50,6 +58,11 @@ as apps, tools, and the core, should use dep.
 Run `dep status` to get a list of vendor dependencies that may not be
 up-to-date.
 
+When updating dependencies, please only update the particular dependencies you
+need. Instead of running `dep ensure -update`, which will update anything,
+specify exactly the dependency you want to update, eg.
+`dep ensure -update github.com/tendermint/go-amino`.
+
 ## Vagrant
 
 If you are a [Vagrant](https://www.vagrantup.com/) user, you can get started
@@ -64,6 +77,86 @@ vagrant ssh
 make test
 ```
 
+## Changelog
+
+Every fix, improvement, feature, or breaking change should be made in a
+pull-request that includes an update to the `CHANGELOG_PENDING.md` file.
+
+Changelog entries should be formatted as follows:
+
+```
+- [module] \#xxx Some description about the change (@contributor)
+```
+
+Here, `module` is the part of the code that changed (typically a
+top-level Go package), `xxx` is the pull-request number, and `contributor`
+is the author/s of the change.
+
+It's also acceptable for `xxx` to refer to the relevent issue number, but pull-request
+numbers are preferred.
+Note this means pull-requests should be opened first so the changelog can then
+be updated with the pull-request's number.
+There is no need to include the full link, as this will be added
+automatically during release. But please include the backslash and pound, eg. `\#2313`.
+
+Changelog entries should be ordered alphabetically according to the
+`module`, and numerically according to the pull-request number.
+
+Changes with multiple classifications should be doubly included (eg. a bug fix
+that is also a breaking change should be recorded under both).
+
+Breaking changes are further subdivided according to the APIs/users they impact.
+Any change that effects multiple APIs/users should be recorded multiply - for
+instance, a change to the `Blockchain Protocol` that removes a field from the
+header should also be recorded under `CLI/RPC/Config` since the field will be
+removed from the header in rpc responses as well.
+
+## Branching Model and Release
+
+We follow a variant of [git flow](http://nvie.com/posts/a-successful-git-branching-model/).
+This means that all pull-requests should be made against develop. Any merge to
+master constitutes a tagged release.
+
+Note all pull requests should be squash merged except for merging to master and
+merging master back to develop. This keeps the commit history clean and makes it
+easy to reference the pull request where a change was introduced.
+
+### Development Procedure:
+- the latest state of development is on `develop`
+- `develop` must never fail `make test`
+- never --force onto `develop` (except when reverting a broken commit, which should seldom happen)
+- create a development branch either on github.com/tendermint/tendermint, or your fork (using `git remote add origin`)
+- make changes and update the `CHANGELOG_PENDING.md` to record your change
+- before submitting a pull request, run `git rebase` on top of the latest `develop`
+
+### Pull Merge Procedure:
+- ensure pull branch is based on a recent develop
+- run `make test` to ensure that all tests pass
+- squash merge pull request
+- the `unstable` branch may be used to aggregate pull merges before fixing tests
+
+### Release Procedure:
+- start on `develop`
+- run integration tests (see `test_integrations` in Makefile)
+- prepare release in a pull request against develop (to be squash merged):
+    - copy `CHANGELOG_PENDING.md` to top of `CHANGELOG.md`
+    - run `python ./scripts/linkify_changelog.py CHANGELOG.md` to add links for
+      all issues
+    - run `bash ./scripts/authors.sh` to get a list of authors since the latest
+      release, and add the github aliases of external contributors to the top of
+      the changelog. To lookup an alias from an email, try `bash
+      ./scripts/authors.sh <email>`
+    - reset the `CHANGELOG_PENDING.md`
+    - bump versions
+- push latest develop with prepared release details to release/vX.X.X to run the extended integration tests on the CI
+- if necessary, make pull requests against release/vX.X.X and squash merge them
+- merge to master (don't squash merge!)
+- merge master back to develop (don't squash merge!)
+
+### Hotfix Procedure:
+
+- follow the normal development and release procedure without any differences
+
 ## Testing
 
 All repos should be hooked up to [CircleCI](https://circleci.com/).
@@ -72,46 +165,3 @@ If they have `.go` files in the root directory, they will be automatically
 tested by circle using `go test -v -race ./...`. If not, they will need a
 `circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and
 includes its continuous integration status using a badge in the `README.md`.
-
-## Branching Model and Release
-
-User-facing repos should adhere to the branching model: http://nvie.com/posts/a-successful-git-branching-model/.
-That is, these repos should be well versioned, and any merge to master requires a version bump and tagged release.
-
-Libraries need not follow the model strictly, but would be wise to,
-especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint core.
-
-### Development Procedure:
-- the latest state of development is on `develop`
-- `develop` must never fail `make test`
-- no --force onto `develop` (except when reverting a broken commit, which should seldom happen)
-- create a development branch either on github.com/tendermint/tendermint, or your fork (using `git remote add origin`)
-- before submitting a pull request, begin `git rebase` on top of `develop`
-
-### Pull Merge Procedure:
-- ensure pull branch is rebased on develop
-- run `make test` to ensure that all tests pass
-- merge pull request
-- the `unstable` branch may be used to aggregate pull merges before testing once
-- push master may request that pull requests be rebased on top of `unstable`
-
-### Release Procedure:
-- start on `develop`
-- run integration tests (see `test_integrations` in Makefile)
-- prepare changelog/release issue
-- bump versions
-- push to release-vX.X.X to run the extended integration tests on the CI
-- merge to master
-- merge master back to develop
-
-### Hotfix Procedure:
-- start on `master`
-- checkout a new branch named hotfix-vX.X.X
-- make the required changes
-  - these changes should be small and an absolute necessity
-  - add a note to CHANGELOG.md
-- bump versions
-- push to hotfix-vX.X.X to run the extended integration tests on the CI
-- merge hotfix-vX.X.X to master
-- merge hotfix-vX.X.X to develop
-- delete the hotfix-vX.X.X branch

DOCKER/Dockerfile

@@ -1,5 +1,5 @@
-FROM alpine:3.7
-MAINTAINER Greg Szabo <greg@tendermint.com>
+FROM alpine:3.9
+LABEL maintainer="hello@tendermint.com"
 
 # Tendermint will be looking for the genesis file in /tendermint/config/genesis.json
 # (unless you change `genesis_file` in config.toml). You can put your config.toml and

DOCKER/build.sh

@@ -3,7 +3,7 @@ set -e
 
 # Get the tag from the version, or try to figure it out.
 if [ -z "$TAG" ]; then
-	TAG=$(awk -F\" '/Version =/ { print $2; exit }' < ../version/version.go)
+	TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go)
 fi
 if [ -z "$TAG" ]; then
 		echo "Please specify a tag."

DOCKER/push.sh

@@ -3,7 +3,7 @@ set -e
 
 # Get the tag from the version, or try to figure it out.
 if [ -z "$TAG" ]; then
-	TAG=$(awk -F\" '/Version =/ { print $2; exit }' < ../version/version.go)
+	TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go)
 fi
 if [ -z "$TAG" ]; then
 		echo "Please specify a tag."

Gopkg.lock

@@ -10,12 +10,11 @@
   revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
 
 [[projects]]
-  branch = "master"
-  digest = "1:c0decf632843204d2b8781de7b26e7038584e2dcccc7e2f401e88ae85b1df2b7"
+  digest = "1:093bf93a65962e8191e3e8cd8fc6c363f83d43caca9739c906531ba7210a9904"
   name = "github.com/btcsuite/btcd"
   packages = ["btcec"]
   pruneopts = "UT"
-  revision = "67e573d211ace594f1366b4ce9d39726c4b19bd0"
+  revision = "ed77733ec07dfc8a513741138419b8d9d3de9d2d"
 
 [[projects]]
   digest = "1:1d8e1cb71c33a9470bbbae09bfec09db43c6bf358dfcae13cd8807c4e2a9a2bf"
@@ -35,6 +34,14 @@
   revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
   version = "v1.1.1"
 
+[[projects]]
+  digest = "1:5f7414cf41466d4b4dd7ec52b2cd3e481e08cfd11e7e24fef730c0e483e88bb1"
+  name = "github.com/etcd-io/bbolt"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "63597a96ec0ad9e6d43c3fc81e809909e0237461"
+  version = "v1.3.2"
+
 [[projects]]
   digest = "1:544229a3ca0fb2dd5ebc2896d3d2ff7ce096d9751635301e44e37e761349ee70"
   name = "github.com/fortytw2/leaktest"
@@ -84,7 +91,7 @@
   version = "v1.8.0"
 
 [[projects]]
-  digest = "1:35621fe20f140f05a0c4ef662c26c0ab4ee50bca78aa30fe87d33120bd28165e"
+  digest = "1:95e1006e41c641abd2f365dfa0f1213c04da294e7cd5f0bf983af234b775db64"
   name = "github.com/gogo/protobuf"
   packages = [
     "gogoproto",
@@ -95,11 +102,11 @@
     "types",
   ]
   pruneopts = "UT"
-  revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
-  version = "v1.1.1"
+  revision = "ba06b47c162d49f2af050fb4c75bcbc86a159d5c"
+  version = "v1.2.1"
 
 [[projects]]
-  digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260"
+  digest = "1:239c4c7fd2159585454003d9be7207167970194216193a8a210b8d29576f19c9"
   name = "github.com/golang/protobuf"
   packages = [
     "proto",
@@ -109,8 +116,8 @@
     "ptypes/timestamp",
   ]
   pruneopts = "UT"
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
-  version = "v1.1.0"
+  revision = "c823c79ea1570fb5ff454033735a8e68575d1d0f"
+  version = "v1.3.0"
 
 [[projects]]
   branch = "master"
@@ -155,11 +162,12 @@
   version = "v1.0"
 
 [[projects]]
-  digest = "1:39b27d1381a30421f9813967a5866fba35dc1d4df43a6eefe3b7a5444cb07214"
+  digest = "1:a74b5a8e34ee5843cd6e65f698f3e75614f812ff170c2243425d75bc091e9af2"
   name = "github.com/jmhodges/levigo"
   packages = ["."]
   pruneopts = "UT"
-  revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
+  revision = "853d788c5c416eaaee5b044570784a96c7a26975"
+  version = "v1.0.0"
 
 [[projects]]
   branch = "master"
@@ -170,9 +178,12 @@
   revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
 
 [[projects]]
-  digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7"
+  digest = "1:53e8c5c79716437e601696140e8b1801aae4204f4ec54a504333702a49572c4f"
   name = "github.com/magiconair/properties"
-  packages = ["."]
+  packages = [
+    ".",
+    "assert",
+  ]
   pruneopts = "UT"
   revision = "c2353362d570a7bfa228149c62842019201cfb71"
   version = "v1.8.0"
@@ -218,14 +229,16 @@
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:c1a04665f9613e082e1209cf288bf64f4068dcd6c87a64bf1c4ff006ad422ba0"
+  digest = "1:26663fafdea73a38075b07e8e9d82fc0056379d2be8bb4e13899e8fda7c7dd23"
   name = "github.com/prometheus/client_golang"
   packages = [
     "prometheus",
+    "prometheus/internal",
     "prometheus/promhttp",
   ]
   pruneopts = "UT"
-  revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632"
+  revision = "abad2d1bd44235a26707c172eab6bca5bf2dbad3"
+  version = "v0.9.1"
 
 [[projects]]
   branch = "master"
@@ -267,6 +280,14 @@
   pruneopts = "UT"
   revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
 
+[[projects]]
+  digest = "1:b0c25f00bad20d783d259af2af8666969e2fc343fa0dc9efe52936bbd67fb758"
+  name = "github.com/rs/cors"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "9a47f48565a795472d43519dd49aac781f3034fb"
+  version = "v1.6.0"
+
 [[projects]]
   digest = "1:6a4a11ba764a56d2758899ec6f3848d24698d48442ebce85ee7a3f63284526cd"
   name = "github.com/spf13/afero"
@@ -351,22 +372,16 @@
   revision = "6b91fda63f2e36186f1c9d0e48578defb69c5d43"
 
 [[projects]]
-  digest = "1:605b6546f3f43745695298ec2d342d3e952b6d91cdf9f349bea9315f677d759f"
-  name = "github.com/tendermint/btcd"
-  packages = ["btcec"]
-  pruneopts = "UT"
-  revision = "e5840949ff4fff0c56f9b6a541e22b63581ea9df"
-
-[[projects]]
-  digest = "1:10b3a599325740c84a7c81f3f3cb2e1fdb70b3ea01b7fa28495567a2519df431"
+  digest = "1:ad9c4c1a4e7875330b1f62906f2830f043a23edb5db997e3a5ac5d3e6eadf80a"
   name = "github.com/tendermint/go-amino"
   packages = ["."]
   pruneopts = "UT"
-  revision = "6dcc6ddc143e116455c94b25c1004c99e0d0ca12"
-  version = "v0.14.0"
+  revision = "dc14acf9ef15f85828bfbc561ed9dd9d2a284885"
+  version = "v0.14.1"
 
 [[projects]]
-  digest = "1:72b71e3a29775e5752ed7a8012052a3dee165e27ec18cedddae5288058f09acf"
+  branch = "master"
+  digest = "1:f4edb30d5ff238e2abba10457010f74cd55ae20bbda8c54db1a07155fa020490"
   name = "golang.org/x/crypto"
   packages = [
     "bcrypt",
@@ -387,8 +402,7 @@
     "salsa20/salsa",
   ]
   pruneopts = "UT"
-  revision = "3764759f34a542a3aef74d6b02e35be7ab893bba"
-  source = "github.com/tendermint/crypto"
+  revision = "8dd112bcdc25174059e45e07517d9fc663123347"
 
 [[projects]]
   digest = "1:d36f55a999540d29b6ea3c2ea29d71c76b1d9853fdcd3e5c5cb4836f2ba118f1"
@@ -494,8 +508,10 @@
   analyzer-name = "dep"
   analyzer-version = 1
   input-imports = [
+    "github.com/btcsuite/btcd/btcec",
     "github.com/btcsuite/btcutil/base58",
     "github.com/btcsuite/btcutil/bech32",
+    "github.com/etcd-io/bbolt",
     "github.com/fortytw2/leaktest",
     "github.com/go-kit/kit/log",
     "github.com/go-kit/kit/log/level",
@@ -512,10 +528,12 @@
     "github.com/golang/protobuf/ptypes/timestamp",
     "github.com/gorilla/websocket",
     "github.com/jmhodges/levigo",
+    "github.com/magiconair/properties/assert",
     "github.com/pkg/errors",
     "github.com/prometheus/client_golang/prometheus",
     "github.com/prometheus/client_golang/prometheus/promhttp",
     "github.com/rcrowley/go-metrics",
+    "github.com/rs/cors",
     "github.com/spf13/cobra",
     "github.com/spf13/viper",
     "github.com/stretchr/testify/assert",
@@ -524,7 +542,6 @@
     "github.com/syndtr/goleveldb/leveldb/errors",
     "github.com/syndtr/goleveldb/leveldb/iterator",
     "github.com/syndtr/goleveldb/leveldb/opt",
-    "github.com/tendermint/btcd/btcec",
     "github.com/tendermint/go-amino",
     "golang.org/x/crypto/bcrypt",
     "golang.org/x/crypto/chacha20poly1305",

Gopkg.toml

@@ -20,88 +20,77 @@
 #   unused-packages = true
 #
 ###########################################################
-# NOTE: All packages should be pinned to specific versions.
-# Packages without releases must pin to a commit.
-
 
+# Allow only patch releases for serialization libraries
 [[constraint]]
-  name = "github.com/go-kit/kit"
-  version = "=0.6.0"
-
-[[constraint]]
-  name = "github.com/gogo/protobuf"
-  version = "=1.1.1"
-
-[[constraint]]
-  name = "github.com/golang/protobuf"
-  version = "=1.1.0"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "=1.2.0"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "=0.8.0"
-
-[[constraint]]
-  name = "github.com/spf13/cobra"
-  version = "=0.0.1"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "=1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "=1.2.1"
+  name = "github.com/etcd-io/bbolt"
+  version = "v1.3.2"
 
 [[constraint]]
   name = "github.com/tendermint/go-amino"
-  version = "v0.14.0"
+  version = "~0.14.1"
+
+[[constraint]]
+  name = "github.com/gogo/protobuf"
+  version = "~1.2.1"
+
+[[constraint]]
+  name = "github.com/golang/protobuf"
+  version = "~1.3.0"
+
+# Allow only minor releases for other libraries
+[[constraint]]
+  name = "github.com/go-kit/kit"
+  version = "^0.6.0"
+
+[[constraint]]
+  name = "github.com/gorilla/websocket"
+  version = "^1.2.0"
+
+[[constraint]]
+  name = "github.com/rs/cors"
+  version = "^1.6.0"
+
+[[constraint]]
+  name = "github.com/pkg/errors"
+  version = "^0.8.0"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "^0.0.1"
+
+[[constraint]]
+  name = "github.com/spf13/viper"
+  version = "^1.0.0"
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "^1.2.1"
 
 [[constraint]]
   name = "google.golang.org/grpc"
-  version = "=1.13.0"
+  version = "^1.13.0"
 
 [[constraint]]
   name = "github.com/fortytw2/leaktest"
-  version = "=1.2.0"
+  version = "^1.2.0"
 
-###################################
-## Some repos dont have releases.
-## Pin to revision
-
-[[constraint]]
-  name = "golang.org/x/crypto"
-  source = "github.com/tendermint/crypto"
-  revision = "3764759f34a542a3aef74d6b02e35be7ab893bba"
-
-[[override]]
-  name = "github.com/jmhodges/levigo"
-  revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
-
-# last revision used by go-crypto
-[[constraint]]
-  name = "github.com/btcsuite/btcutil"
-  revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4"
-
-[[constraint]]
-  name = "github.com/tendermint/btcd"
-  revision = "e5840949ff4fff0c56f9b6a541e22b63581ea9df"
-
-# Haven't made a release since 2016.
 [[constraint]]
   name = "github.com/prometheus/client_golang"
-  revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632"
+  version = "^0.9.1"
 
 [[constraint]]
-  name = "github.com/rcrowley/go-metrics"
-  revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
+  name = "github.com/jmhodges/levigo"
+  version = "^1.0.0"
 
-[[constraint]]
-  name = "golang.org/x/net"
-  revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f"
+###################################
+## Repos which don't have releases.
+
+## - github.com/btcsuite/btcd
+## - golang.org/x/crypto
+## - github.com/btcsuite/btcutil
+## - github.com/rcrowley/go-metrics
+## - golang.org/x/net
 
 [prune]
   go-tests = true

Makefile

@@ -1,38 +1,39 @@
 GOTOOLS = \
 	github.com/mitchellh/gox \
 	github.com/golang/dep/cmd/dep \
-	github.com/alecthomas/gometalinter \
+	github.com/golangci/golangci-lint/cmd/golangci-lint \
 	github.com/gogo/protobuf/protoc-gen-gogo \
 	github.com/square/certstrap
 GOBIN?=${GOPATH}/bin
 PACKAGES=$(shell go list ./...)
+OUTPUT?=build/tendermint
 
 INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf
 BUILD_TAGS?='tendermint'
 BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
 
-LINT_FLAGS = --exclude '.*\.pb\.go' --exclude 'vendor/*' --vendor --deadline=600s
-
 all: check build test install
 
 check: check_tools get_vendor_deps
 
-
 ########################################
 ### Build Tendermint
 
 build:
-	CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint/
+	CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint/
 
 build_c:
-	CGO_ENABLED=1 go build $(BUILD_FLAGS) -tags "$(BUILD_TAGS) gcc" -o build/tendermint ./cmd/tendermint/
+	CGO_ENABLED=1 go build $(BUILD_FLAGS) -tags "$(BUILD_TAGS) cleveldb" -o $(OUTPUT) ./cmd/tendermint/
 
 build_race:
-	CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint
+	CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint
 
 install:
 	CGO_ENABLED=0 go install  $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint
 
+install_c:
+	CGO_ENABLED=1 go install  $(BUILD_FLAGS) -tags "$(BUILD_TAGS) cleveldb" ./cmd/tendermint
+
 ########################################
 ### Protobuf
 
@@ -79,8 +80,6 @@ check_tools:
 get_tools:
 	@echo "--> Installing tools"
 	./scripts/get_tools.sh
-	@echo "--> Downloading linters (this may take awhile)"
-	$(GOPATH)/src/github.com/alecthomas/gometalinter/scripts/install.sh -b $(GOBIN)
 
 update_tools:
 	@echo "--> Updating tools"
@@ -111,7 +110,7 @@ draw_deps:
 
 get_deps_bin_size:
 	@# Copy of build recipe with additional flags to perform binary size analysis
-	$(eval $(shell go build -work -a $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint/ 2>&1))
+	$(eval $(shell go build -work -a $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint/ 2>&1))
 	@find $(WORK) -type f -name "*.a" | xargs -I{} du -hxs "{}" | sort -rh | sed -e s:${WORK}/::g > deps_bin_size.log
 	@echo "Results can be found here: $(CURDIR)/deps_bin_size.log"
 
@@ -133,7 +132,7 @@ clean_certs:
 	rm -f db/remotedb/::.crt db/remotedb/::.key
 
 test_libs: gen_certs
-	GOCACHE=off go test -tags gcc $(PACKAGES)
+	go test -tags clevedb boltdb $(PACKAGES)
 	make clean_certs
 
 grpc_dbserver:
@@ -216,12 +215,28 @@ vagrant_test:
 ### go tests
 test:
 	@echo "--> Running go test"
-	@GOCACHE=off go test -p 1 $(PACKAGES)
+	@go test -p 1 $(PACKAGES)
 
 test_race:
 	@echo "--> Running go test --race"
-	@GOCACHE=off go test -p 1 -v -race $(PACKAGES)
+	@go test -p 1 -v -race $(PACKAGES)
 
+# uses https://github.com/sasha-s/go-deadlock/ to detect potential deadlocks
+test_with_deadlock:
+	make set_with_deadlock
+	make test
+	make cleanup_after_test_with_deadlock
+
+set_with_deadlock:
+	find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.RWMutex/deadlock.RWMutex/'
+	find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.Mutex/deadlock.Mutex/'
+	find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w
+
+# cleanes up after you ran test_with_deadlock
+cleanup_after_test_with_deadlock:
+	find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.RWMutex/sync.RWMutex/'
+	find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.Mutex/sync.Mutex/'
+	find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w
 
 ########################################
 ### Formatting, linting, and vetting
@@ -229,38 +244,9 @@ test_race:
 fmt:
 	@go fmt ./...
 
-metalinter:
+lint:
 	@echo "--> Running linter"
-	@gometalinter $(LINT_FLAGS) --disable-all  \
-		--enable=deadcode \
-		--enable=gosimple \
-	 	--enable=misspell \
-		--enable=safesql \
-		./...
-		#--enable=gas \
-		#--enable=maligned \
-		#--enable=dupl \
-		#--enable=errcheck \
-		#--enable=goconst \
-		#--enable=gocyclo \
-		#--enable=goimports \
-		#--enable=golint \ <== comments on anything exported
-		#--enable=gotype \
-	 	#--enable=ineffassign \
-	   	#--enable=interfacer \
-	   	#--enable=megacheck \
-	   	#--enable=staticcheck \
-	   	#--enable=structcheck \
-	   	#--enable=unconvert \
-	   	#--enable=unparam \
-		#--enable=unused \
-	   	#--enable=varcheck \
-		#--enable=vet \
-		#--enable=vetshadow \
-
-metalinter_all:
-	@echo "--> Running linter (all)"
-	gometalinter $(LINT_FLAGS) --enable-all --disable=lll ./...
+	@golangci-lint run
 
 DESTINATION = ./index.html.md
 
@@ -276,7 +262,7 @@ check_dep:
 ### Docker image
 
 build-docker:
-	cp build/tendermint DOCKER/tendermint
+	cp $(OUTPUT) DOCKER/tendermint
 	docker build --label=tendermint --tag="tendermint/tendermint" DOCKER
 	rm -rf DOCKER/tendermint
 
@@ -284,12 +270,11 @@ build-docker:
 ### Local testnet using docker
 
 # Build linux binary on other platforms
-build-linux:
+build-linux: get_tools get_vendor_deps
 	GOOS=linux GOARCH=amd64 $(MAKE) build
 
 build-docker-localnode:
-	cd networks/local
-	make
+	@cd networks/local && make
 
 # Run a 4-node testnet locally
 localnet-start: localnet-stop
@@ -327,4 +312,4 @@ build-slate:
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: check build build_race build_abci dist install install_abci check_dep check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt rpc-docs build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all
+.PHONY: check build build_race build_abci dist install install_abci check_dep check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt rpc-docs build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all build_c install_c test_with_deadlock cleanup_after_test_with_deadlock lint

PHILOSOPHY.md

@@ -0,0 +1,158 @@
+## Design goals
+
+The design goals for Tendermint (and the SDK and related libraries) are:
+
+ * Simplicity and Legibility
+ * Parallel performance, namely ability to utilize multicore architecture
+ * Ability to evolve the codebase bug-free
+ * Debuggability
+ * Complete correctness that considers all edge cases, esp in concurrency
+ * Future-proof modular architecture, message protocol, APIs, and encapsulation
+
+
+### Justification
+
+Legibility is key to maintaining bug-free software as it evolves toward more
+optimizations, more ease of debugging, and additional features.
+
+It is too easy to introduce bugs over time by replacing lines of code with
+those that may panic, which means ideally locks are unlocked by defer
+statements.
+
+For example,
+
+```go
+func (obj *MyObj) something() {
+	mtx.Lock()
+	obj.something = other
+	mtx.Unlock()
+}
+```
+
+It is too easy to refactor the codebase in the future to replace `other` with
+`other.String()` for example, and this may introduce a bug that causes a
+deadlock.  So as much as reasonably possible, we need to be using defer
+statements, even though it introduces additional overhead.
+
+If it is necessary to optimize the unlocking of mutex locks, the solution is
+more modularity via smaller functions, so that defer'd unlocks are scoped
+within a smaller function.
+
+Similarly, idiomatic for-loops should always be preferred over those that use
+custom counters, because it is too easy to evolve the body of a for-loop to
+become more complicated over time, and it becomes more and more difficult to
+assess the correctness of such a for-loop by visual inspection.
+
+
+### On performance
+
+It doesn't matter whether there are alternative implementations that are 2x or
+3x more performant, when the software doesn't work, deadlocks, or if bugs
+cannot be debugged.  By taking advantage of multicore concurrency, the
+Tendermint implementation will at least be an order of magnitude within the
+range of what is theoretically possible.  The design philosophy of Tendermint,
+and the choice of Go as implementation language, is designed to make Tendermint
+implementation the standard specification for concurrent BFT software.
+
+By focusing on the message protocols (e.g. ABCI, p2p messages), and
+encapsulation e.g. IAVL module, (relatively) independent reactors, we are both
+implementing a standard implementation to be used as the specification for
+future implementations in more optimizable languages like Rust, Java, and C++;
+as well as creating sufficiently performant software. Tendermint Core will
+never be as fast as future implementations of the Tendermint Spec, because Go
+isn't designed to be as fast as possible.  The advantage of using Go is that we
+can develop the whole stack of modular components **faster** than in other
+languages.
+
+Furthermore, the real bottleneck is in the application layer, and it isn't
+necessary to support more than a sufficiently decentralized set of validators
+(e.g. 100 ~ 300 validators is sufficient, with delegated bonded PoS).
+
+Instead of optimizing Tendermint performance down to the metal, lets focus on
+optimizing on other matters, namely ability to push feature complete software
+that works well enough, can be debugged and maintained, and can serve as a spec
+for future implementations.
+
+
+### On encapsulation
+
+In order to create maintainable, forward-optimizable software, it is critical
+to develop well-encapsulated objects that have well understood properties, and
+to re-use these easy-to-use-correctly components as building blocks for further
+encapsulated meta-objects.
+
+For example, mutexes are cheap enough for Tendermint's design goals when there
+isn't goroutine contention, so it is encouraged to create concurrency safe
+structures with struct-level mutexes.  If they are used in the context of
+non-concurrent logic, then the performance is good enough.  If they are used in
+the context of concurrent logic, then it will still perform correctly.
+
+Examples of this design principle can be seen in the types.ValidatorSet struct,
+and the cmn.Rand struct.  It's one single struct declaration that can be used
+in both concurrent and non-concurrent logic, and due to its well encapsulation,
+it's easy to get the usage of the mutex right.
+
+#### example: cmn.Rand:
+
+`The default Source is safe for concurrent use by multiple goroutines, but
+Sources created by NewSource are not`.  The reason why the default
+package-level source is safe for concurrent use is because it is protected (see
+`lockedSource` in https://golang.org/src/math/rand/rand.go).
+
+But we shouldn't rely on the global source, we should be creating our own
+Rand/Source instances and using them, especially for determinism in testing.
+So it is reasonable to have cmn.Rand be protected by a mutex.  Whether we want
+our own implementation of Rand is another question, but the answer there is
+also in the affirmative.  Sometimes you want to know where Rand is being used
+in your code, so it becomes a simple matter of dropping in a log statement to
+inject inspectability into Rand usage.  Also, it is nice to be able to extend
+the functionality of Rand with custom methods.  For these reasons, and for the
+reasons which is outlined in this design philosophy document, we should
+continue to use the cmn.Rand object, with mutex protection.
+
+Another key aspect of good encapsulation is the choice of exposed vs unexposed
+methods.  It should be clear to the reader of the code, which methods are
+intended to be used in what context, and what safe usage is.  Part of this is
+solved by hiding methods via unexported methods.  Another part of this is
+naming conventions on the methods (e.g. underscores) with good documentation,
+and code organization.  If there are too many exposed methods and it isn't
+clear what methods have what side effects, then there is something wrong about
+the design of abstractions that should be revisited.
+
+
+### On concurrency
+
+In order for Tendermint to remain relevant in the years to come, it is vital
+for Tendermint to take advantage of multicore architectures.  Due to the nature
+of the problem, namely consensus across a concurrent p2p gossip network, and to
+handle RPC requests for a large number of consuming subscribers, it is
+unavoidable for Tendermint development to require expertise in concurrency
+design, especially when it comes to the reactor design, and also for RPC
+request handling.
+
+
+## Guidelines
+
+Here are some guidelines for designing for (sufficient) performance and concurrency:
+
+ * Mutex locks are cheap enough when there isn't contention.
+ * Do not optimize code without analytical or observed proof that it is in a hot path.
+ * Don't over-use channels when mutex locks w/ encapsulation are sufficient.
+ * The need to drain channels are often a hint of unconsidered edge cases.
+ * The creation of O(N) one-off goroutines is generally technical debt that
+   needs to get addressed sooner than later.  Avoid creating too many
+goroutines as a patch around incomplete concurrency design, or at least be
+aware of the debt and do not invest in the debt.  On the other hand, Tendermint
+is designed to have a limited number of peers (e.g. 10 or 20), so the creation
+of O(C) goroutines per O(P) peers is still O(C\*P=constant).
+  * Use defer statements to unlock as much as possible.  If you want to unlock sooner,
+    try to create more modular functions that do make use of defer statements.
+
+## Matras
+
+* Premature optimization kills
+* Readability is paramount
+* Beautiful is better than fast.
+* In the face of ambiguity, refuse the temptation to guess.
+* In the face of bugs, refuse the temptation to cover the bug.
+* There should be one-- and preferably only one --obvious way to do it.

README.md

@@ -1,14 +1,14 @@
 # Tendermint
 
 [Byzantine-Fault Tolerant](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance)
-[State Machine Replication](https://en.wikipedia.org/wiki/State_machine_replication).
-Or [Blockchain](https://en.wikipedia.org/wiki/Blockchain_(database)) for short.
+[State Machines](https://en.wikipedia.org/wiki/State_machine_replication).
+Or [Blockchain](https://en.wikipedia.org/wiki/Blockchain_(database)), for short.
 
 [![version](https://img.shields.io/github/tag/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/releases/latest)
 [![API Reference](
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/tendermint/tendermint)
-[![Go version](https://img.shields.io/badge/go-1.10.4-blue.svg)](https://github.com/moovweb/gvm)
+[![Go version](https://img.shields.io/badge/go-1.12.0-blue.svg)](https://github.com/moovweb/gvm)
 [![riot.im](https://img.shields.io/badge/riot.im-JOIN%20CHAT-green.svg)](https://riot.im/app/#/room/#tendermint:matrix.org)
 [![license](https://img.shields.io/github/license/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/blob/master/LICENSE)
 [![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint)
@@ -41,7 +41,7 @@ please [contact us](mailto:partners@tendermint.com) and [join the chat](https://
 ## Security
 
 To report a security vulnerability, see our [bug bounty
-program](https://tendermint.com/security).
+program](https://hackerone.com/tendermint)
 
 For examples of the kinds of bugs we're looking for, see [SECURITY.md](SECURITY.md)
 
@@ -49,61 +49,43 @@ For examples of the kinds of bugs we're looking for, see [SECURITY.md](SECURITY.
 
 Requirement|Notes
 ---|---
-Go version | Go1.10 or higher
+Go version | Go1.11.4 or higher
 
-## Install
+## Documentation
+
+Complete documentation can be found on the [website](https://tendermint.com/docs/).
+
+### Install
 
 See the [install instructions](/docs/introduction/install.md)
 
-## Quick Start
+### Quick Start
 
-- [Single node](/docs/tendermint-core/using-tendermint.md)
-- [Local cluster using docker-compose](/networks/local)
+- [Single node](/docs/introduction/quick-start.md)
+- [Local cluster using docker-compose](/docs/networks/docker-compose.md)
 - [Remote cluster using terraform and ansible](/docs/networks/terraform-and-ansible.md)
 - [Join the Cosmos testnet](https://cosmos.network/testnet)
 
-## Resources
-
-### Tendermint Core
-
-For details about the blockchain data structures and the p2p protocols, see the
-the [Tendermint specification](/docs/spec).
-
-For details on using the software, see the [documentation](/docs/) which is also
-hosted at: https://tendermint.com/docs/
-
-### Tools
-
-Benchmarking and monitoring is provided by `tm-bench` and `tm-monitor`, respectively.
-Their code is found [here](/tools) and these binaries need to be built seperately.
-Additional documentation is found [here](/docs/tools).
-
-### Sub-projects
-
-* [Amino](http://github.com/tendermint/go-amino), a reflection-based improvement on proto3
-* [IAVL](http://github.com/tendermint/iavl), Merkleized IAVL+ Tree implementation
-
-### Applications
-
-* [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework
-* [Ethermint](http://github.com/cosmos/ethermint); Ethereum on Tendermint
-* [Many more](https://tendermint.com/ecosystem)
-
-### Research
-
-* [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769)
-* [Original Whitepaper](https://tendermint.com/static/docs/tendermint.pdf)
-* [Blog](https://blog.cosmos.network/tendermint/home)
-
 ## Contributing
 
-Yay open source! Please see our [contributing guidelines](CONTRIBUTING.md).
+Please abide by the [Code of Conduct](CODE_OF_CONDUCT.md) in all interactions,
+and the [contributing guidelines](CONTRIBUTING.md) when submitting code.
+
+Join the larger community on the [forum](https://forum.cosmos.network/) and the [chat](https://riot.im/app/#/room/#tendermint:matrix.org).
+
+To learn more about the structure of the software, watch the [Developer
+Sessions](https://www.youtube.com/playlist?list=PLdQIb0qr3pnBbG5ZG-0gr3zM86_s8Rpqv)
+and read some [Architectural
+Decision Records](https://github.com/tendermint/tendermint/tree/master/docs/architecture).
+
+Learn more by reading the code and comparing it to the
+[specification](https://github.com/tendermint/tendermint/tree/develop/docs/spec).
 
 ## Versioning
 
-### SemVer
+### Semantic Versioning
 
-Tendermint uses [SemVer](http://semver.org/) to determine when and how the version changes.
+Tendermint uses [Semantic Versioning](http://semver.org/) to determine when and how the version changes.
 According to SemVer, anything in the public API can change at any time before version 1.0.0
 
 To provide some stability to Tendermint users in these 0.X.X days, the MINOR version is used
@@ -114,6 +96,7 @@ include the in-process Go APIs.
 That said, breaking changes in the following packages will be documented in the
 CHANGELOG even if they don't lead to MINOR version bumps:
 
+- crypto
 - types
 - rpc/client
 - config
@@ -140,8 +123,40 @@ data into the new chain.
 However, any bump in the PATCH version should be compatible with existing histories
 (if not please open an [issue](https://github.com/tendermint/tendermint/issues)).
 
-For more information on upgrading, see [here](./UPGRADING.md)
+For more information on upgrading, see [UPGRADING.md](./UPGRADING.md)
 
-## Code of Conduct
+## Resources
+
+### Tendermint Core
+
+For details about the blockchain data structures and the p2p protocols, see the
+[Tendermint specification](/docs/spec).
+
+For details on using the software, see the [documentation](/docs/) which is also
+hosted at: https://tendermint.com/docs/
+
+### Tools
+
+Benchmarking and monitoring is provided by `tm-bench` and `tm-monitor`, respectively.
+Their code is found [here](/tools) and these binaries need to be built seperately.
+Additional documentation is found [here](/docs/tools).
+
+### Sub-projects
+
+* [Amino](http://github.com/tendermint/go-amino), reflection-based proto3, with
+  interfaces
+* [IAVL](http://github.com/tendermint/iavl), Merkleized IAVL+ Tree implementation
+
+### Applications
+
+* [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework
+* [Ethermint](http://github.com/cosmos/ethermint); Ethereum on Tendermint
+* [Many more](https://tendermint.com/ecosystem)
+
+### Research
+
+* [The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938)
+* [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769)
+* [Original Whitepaper](https://tendermint.com/static/docs/tendermint.pdf)
+* [Blog](https://blog.cosmos.network/tendermint/home)
 
-Please read, understand and adhere to our [code of conduct](CODE_OF_CONDUCT.md).

SECURITY.md

@@ -1,7 +1,8 @@
 # Security
 
 As part of our [Coordinated Vulnerability Disclosure
-Policy](https://tendermint.com/security), we operate a bug bounty.
+Policy](https://tendermint.com/security), we operate a [bug
+bounty](https://hackerone.com/tendermint).
 See the policy for more details on submissions and rewards.
 
 Here is a list of examples of the kinds of bugs we're most interested in:

UPGRADING.md

@@ -3,9 +3,195 @@
 This guide provides steps to be followed when you upgrade your applications to
 a newer version of Tendermint Core.
 
+## v0.31.6
+
+There are no breaking changes in this release except Go API of p2p and
+mempool packages. Hovewer, if you're using cleveldb, you'll need to change
+the compilation tag:
+
+Use `cleveldb` tag instead of `gcc` to compile Tendermint with CLevelDB or
+use `make build_c` / `make install_c` (full instructions can be found at
+https://tendermint.com/docs/introduction/install.html#compile-with-cleveldb-support)
+
+## v0.31.0
+
+This release contains a breaking change to the behaviour of the pubsub system.
+It also contains some minor breaking changes in the Go API and ABCI.
+There are no changes to the block or p2p protocols, so v0.31.0 should work fine
+with blockchains created from the v0.30 series.
+
+### RPC
+
+The pubsub no longer blocks on publishing. This may cause some WebSocket (WS) clients to stop working as expected.
+If your WS client is not consuming events fast enough, Tendermint can terminate the subscription.
+In this case, the WS client will receive an error with description:
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": "{ID}#event",
+  "error": {
+    "code": -32000,
+    "msg": "Server error",
+    "data": "subscription was cancelled (reason: client is not pulling messages fast enough)" // or "subscription was cancelled (reason: Tendermint exited)"
+  }
+}
+
+Additionally, there are now limits on the number of subscribers and
+subscriptions that can be active at once. See the new
+`rpc.max_subscription_clients` and `rpc.max_subscriptions_per_client` values to
+configure this.
+```
+
+### Applications
+
+Simple rename of `ConsensusParams.BlockSize` to `ConsensusParams.Block`.
+
+The `ConsensusParams.Block.TimeIotaMS` field was also removed. It's configured
+in the ConsensusParsm in genesis.
+
+### Go API
+
+See the [CHANGELOG](CHANGELOG.md). These are relatively straight forward.
+
+## v0.30.0
+
+This release contains a breaking change to both the block and p2p protocols,
+however it may be compatible with blockchains created with v0.29.0 depending on
+the chain history. If your blockchain has not included any pieces of evidence,
+or no piece of evidence has been included in more than one block,
+and if your application has never returned multiple updates
+for the same validator in a single block, then v0.30.0 will work fine with
+blockchains created with v0.29.0.
+
+The p2p protocol change is to fix the proposer selection algorithm again.
+Note that proposer selection is purely a p2p concern right
+now since the algorithm is only relevant during real time consensus.
+This change is thus compatible with v0.29.0, but
+all nodes must be upgraded to avoid disagreements on the proposer.
+
+### Applications
+
+Applications must ensure they do not return duplicates in
+`ResponseEndBlock.ValidatorUpdates`. A pubkey must only appear once per set of
+updates. Duplicates will cause irrecoverable failure. If you have a very good
+reason why we shouldn't do this, please open an issue.
+
+## v0.29.0
+
+This release contains some breaking changes to the block and p2p protocols,
+and will not be compatible with any previous versions of the software, primarily
+due to changes in how various data structures are hashed.
+
+Any implementations of Tendermint blockchain verification, including lite clients,
+will need to be updated. For specific details:
+- [Merkle tree](./docs/spec/blockchain/encoding.md#merkle-trees)
+- [ConsensusParams](./docs/spec/blockchain/state.md#consensusparams)
+
+There was also a small change to field ordering in the vote struct. Any
+implementations of an out-of-process validator (like a Key-Management Server)
+will need to be updated. For specific details:
+- [Vote](https://github.com/tendermint/tendermint/blob/develop/docs/spec/consensus/signing.md#votes)
+
+Finally, the proposer selection algorithm continues to evolve. See the
+[work-in-progress
+specification](https://github.com/tendermint/tendermint/pull/3140).
+
+For everything else, please see the [CHANGELOG](./CHANGELOG.md#v0.29.0).
+
+## v0.28.0
+
+This release breaks the format for the `priv_validator.json` file
+and the protocol used for the external validator process.
+It is compatible with v0.27.0 blockchains (neither the BlockProtocol nor the
+P2PProtocol have changed).
+
+Please read carefully for details about upgrading.
+
+**Note:** Backup your `config/priv_validator.json`
+before proceeding.
+
+### `priv_validator.json`
+
+The `config/priv_validator.json` is now two files:
+`config/priv_validator_key.json` and `data/priv_validator_state.json`.
+The former contains the key material, the later contains the details on the last
+message signed.
+
+When running v0.28.0 for the first time, it will back up any pre-existing
+`priv_validator.json` file and proceed to split it into the two new files.
+Upgrading should happen automatically without problem.
+
+To upgrade manually, use the provided `privValUpgrade.go` script, with exact paths for the old
+`priv_validator.json` and the locations for the two new files. It's recomended
+to use the default paths, of `config/priv_validator_key.json` and
+`data/priv_validator_state.json`, respectively:
+
+```
+go run scripts/privValUpgrade.go <old-path> <new-key-path> <new-state-path>
+```
+
+### External validator signers
+
+The Unix and TCP implementations of the remote signing validator
+have been consolidated into a single implementation.
+Thus in both cases, the external process is expected to dial
+Tendermint. This is different from how Unix sockets used to work, where
+Tendermint dialed the external process.
+
+The `PubKeyMsg` was also split into separate `Request` and `Response` types
+for consistency with other messages.
+
+Note that the TCP sockets don't yet use a persistent key,
+so while they're encrypted, they can't yet be properly authenticated.
+See [#3105](https://github.com/tendermint/tendermint/issues/3105).
+Note the Unix socket has neither encryption nor authentication, but will
+add a shared-secret in [#3099](https://github.com/tendermint/tendermint/issues/3099).
+
+## v0.27.0
+
+This release contains some breaking changes to the block and p2p protocols,
+but does not change any core data structures, so it should be compatible with
+existing blockchains from the v0.26 series that only used Ed25519 validator keys.
+Blockchains using Secp256k1 for validators will not be compatible. This is due
+to the fact that we now enforce which key types validators can use as a
+consensus param. The default is Ed25519, and Secp256k1 must be activated
+explicitly.
+
+It is recommended to upgrade all nodes at once to avoid incompatibilities at the
+peer layer - namely, the heartbeat consensus message has been removed (only
+relevant if `create_empty_blocks=false` or `create_empty_blocks_interval > 0`),
+and the proposer selection algorithm has changed. Since proposer information is
+never included in the blockchain, this change only affects the peer layer.
+
+### Go API Changes
+
+#### libs/db
+
+The ReverseIterator API has changed the meaning of `start` and `end`.
+Before, iteration was from `start` to `end`, where
+`start > end`. Now, iteration is from `end` to `start`, where `start < end`.
+The iterator also excludes `end`. This change allows a simplified and more
+intuitive logic, aligning the semantic meaning of `start` and `end` in the
+`Iterator` and `ReverseIterator`.
+
+### Applications
+
+This release enforces a new consensus parameter, the
+ValidatorParams.PubKeyTypes. Applications must ensure that they only return
+validator updates with the allowed PubKeyTypes. If a validator update includes a
+pubkey type that is not included in the ConsensusParams.Validator.PubKeyTypes,
+block execution will fail and the consensus will halt.
+
+By default, only Ed25519 pubkeys may be used for validators. Enabling
+Secp256k1 requires explicit modification of the ConsensusParams.
+Please update your application accordingly (ie. restrict validators to only be
+able to use Ed25519 keys, or explicitly add additional key types to the genesis
+file).
+
 ## v0.26.0
 
-New 0.26.0 release contains a lot of changes to core data types. It is not
+This release contains a lot of changes to core data types and protocols. It is not
 compatible to the old versions and there is no straight forward way to update
 old data to be compatible with the new version.
 
@@ -33,7 +219,7 @@ to `prove`. To get proofs with your queries, ensure you set `prove=true`.
 Various version fields like `amino_version`, `p2p_version`, `consensus_version`,
 and `rpc_version` have been removed from the `node_info.other` and are
 consolidated under the tendermint semantic version (ie. `node_info.version`) and
-the new `block` and `p2p` protocol versions under `node_info.protocol_version`..
+the new `block` and `p2p` protocol versions under `node_info.protocol_version`.
 
 ### ABCI Changes
 
@@ -45,7 +231,7 @@ protobuf file for these changes.
 
 The `ResponseQuery.Proof` field is now structured as a `[]ProofOp` to support
 generalized Merkle tree constructions where the leaves of one Merkle tree are
-the root of another. If you don't need this functionaluty, and you used to
+the root of another. If you don't need this functionality, and you used to
 return `<proof bytes>` here, you should instead return a single `ProofOp` with
 just the `Data` field set:
 
@@ -67,7 +253,7 @@ For more information, see:
 
 ### Go API Changes
 
-#### crypto.merkle
+#### crypto/merkle
 
 The `merkle.Hasher` interface was removed. Functions which used to take `Hasher`
 now simply take `[]byte`. This means that any objects being Merklized should be
@@ -79,6 +265,10 @@ The `node.RunForever` function was removed. Signal handling and running forever
 should instead be explicitly configured by the caller. See how we do it
 [here](https://github.com/tendermint/tendermint/blob/30519e8361c19f4bf320ef4d26288ebc621ad725/cmd/tendermint/commands/run_node.go#L60).
 
+### Other
+
+All hashes, except for public key addresses, are now 32-bytes.
+
 ## v0.25.0
 
 This release has minimal impact.

Vagrantfile

@@ -29,10 +29,14 @@ Vagrant.configure("2") do |config|
     usermod -a -G docker vagrant
 
     # install go
-    wget -q https://dl.google.com/go/go1.11.linux-amd64.tar.gz
-    tar -xvf go1.11.linux-amd64.tar.gz
+    wget -q https://dl.google.com/go/go1.12.linux-amd64.tar.gz
+    tar -xvf go1.12.linux-amd64.tar.gz
     mv go /usr/local
-    rm -f go1.11.linux-amd64.tar.gz
+    rm -f go1.12.linux-amd64.tar.gz
+
+    # install nodejs (for docs)
+    curl -sL https://deb.nodesource.com/setup_11.x | bash -
+    apt-get install -y nodejs
 
     # cleanup
     apt-get autoremove -y

abci/README.md

@@ -1,7 +1,5 @@
 # Application BlockChain Interface (ABCI)
 
-[![CircleCI](https://circleci.com/gh/tendermint/abci.svg?style=svg)](https://circleci.com/gh/tendermint/abci)
-
 Blockchains are systems for multi-master state machine replication.
 **ABCI** is an interface that defines the boundary between the replication engine (the blockchain),
 and the state machine (the application).
@@ -12,160 +10,28 @@ Previously, the ABCI was referred to as TMSP.
 
 The community has provided a number of addtional implementations, see the [Tendermint Ecosystem](https://tendermint.com/ecosystem)
 
+
+## Installation & Usage
+
+To get up and running quickly, see the [getting started guide](../docs/app-dev/getting-started.md) along with the [abci-cli documentation](../docs/app-dev/abci-cli.md) which will go through the examples found in the [examples](./example/) directory.
+
 ## Specification
 
 A detailed description of the ABCI methods and message types is contained in:
 
-- [A prose specification](specification.md)
-- [A protobuf file](https://github.com/tendermint/tendermint/blob/master/abci/types/types.proto)
-- [A Go interface](https://github.com/tendermint/tendermint/blob/master/abci/types/application.go).
-
-For more background information on ABCI, motivations, and tendermint, please visit [the documentation](https://tendermint.com/docs/).
-The two guides to focus on are the `Application Development Guide` and `Using ABCI-CLI`.
-
+- [The main spec](../docs/spec/abci/abci.md)
+- [A protobuf file](./types/types.proto)
+- [A Go interface](./types/application.go)
 
 ## Protocol Buffers
 
-To compile the protobuf file, run:
+To compile the protobuf file, run (from the root of the repo):
 
 ```
-cd $GOPATH/src/github.com/tendermint/tendermint/; make protoc_abci
+make protoc_abci
 ```
 
 See `protoc --help` and [the Protocol Buffers site](https://developers.google.com/protocol-buffers)
-for details on compiling for other languages. Note we also include a [GRPC](http://www.grpc.io/docs)
+for details on compiling for other languages. Note we also include a [GRPC](https://www.grpc.io/docs)
 service definition.
 
-## Install ABCI-CLI
-
-The `abci-cli` is a simple tool for debugging ABCI servers and running some
-example apps. To install it:
-
-```
-mkdir -p $GOPATH/src/github.com/tendermint
-cd $GOPATH/src/github.com/tendermint
-git clone https://github.com/tendermint/tendermint.git
-cd tendermint
-make get_tools
-make get_vendor_deps
-make install_abci
-```
-
-## Implementation
-
-We provide three implementations of the ABCI in Go:
-
-- Golang in-process
-- ABCI-socket
-- GRPC
-
-Note the GRPC version is maintained primarily to simplify onboarding and prototyping and is not receiving the same
-attention to security and performance as the others
-
-### In Process
-
-The simplest implementation just uses function calls within Go.
-This means ABCI applications written in Golang can be compiled with TendermintCore and run as a single binary.
-
-See the [examples](#examples) below for more information.
-
-### Socket (TSP)
-
-ABCI is best implemented as a streaming protocol.
-The socket implementation provides for asynchronous, ordered message passing over unix or tcp.
-Messages are serialized using Protobuf3 and length-prefixed with a [signed Varint](https://developers.google.com/protocol-buffers/docs/encoding?csw=1#signed-integers)
-
-For example, if the Protobuf3 encoded ABCI message is `0xDEADBEEF` (4 bytes), the length-prefixed message is `0x08DEADBEEF`, since `0x08` is the signed varint
-encoding of `4`. If the Protobuf3 encoded ABCI message is 65535 bytes long, the length-prefixed message would be like `0xFEFF07...`.
-
-Note the benefit of using this `varint` encoding over the old version (where integers were encoded as `<len of len><big endian len>` is that
-it is the standard way to encode integers in Protobuf. It is also generally shorter.
-
-### GRPC
-
-GRPC is an rpc framework native to Protocol Buffers with support in many languages.
-Implementing the ABCI using GRPC can allow for faster prototyping, but is expected to be much slower than
-the ordered, asynchronous socket protocol. The implementation has also not received as much testing or review.
-
-Note the length-prefixing used in the socket implementation does not apply for GRPC.
-
-## Usage
-
-The `abci-cli` tool wraps an ABCI client and can be used for probing/testing an ABCI server.
-For instance, `abci-cli test` will run a test sequence against a listening server running the Counter application (see below).
-It can also be used to run some example applications.
-See [the documentation](https://tendermint.com/docs/) for more details.
-
-### Examples
-
-Check out the variety of example applications in the [example directory](example/).
-It also contains the code refered to by the `counter` and `kvstore` apps; these apps come
-built into the `abci-cli` binary.
-
-#### Counter
-
-The `abci-cli counter` application illustrates nonce checking in transactions. It's code looks like:
-
-```golang
-func cmdCounter(cmd *cobra.Command, args []string) error {
-
-	app := counter.NewCounterApplication(flagSerial)
-
-	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
-
-	// Start the listener
-	srv, err := server.NewServer(flagAddrC, flagAbci, app)
-	if err != nil {
-		return err
-	}
-	srv.SetLogger(logger.With("module", "abci-server"))
-	if err := srv.Start(); err != nil {
-		return err
-	}
-
-	// Wait forever
-	cmn.TrapSignal(func() {
-		// Cleanup
-		srv.Stop()
-	})
-	return nil
-}
-```
-
-and can be found in [this file](cmd/abci-cli/abci-cli.go).
-
-#### kvstore
-
-The `abci-cli kvstore` application, which illustrates a simple key-value Merkle tree
-
-```golang
-func cmdKVStore(cmd *cobra.Command, args []string) error {
-	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
-
-	// Create the application - in memory or persisted to disk
-	var app types.Application
-	if flagPersist == "" {
-		app = kvstore.NewKVStoreApplication()
-	} else {
-		app = kvstore.NewPersistentKVStoreApplication(flagPersist)
-		app.(*kvstore.PersistentKVStoreApplication).SetLogger(logger.With("module", "kvstore"))
-	}
-
-	// Start the listener
-	srv, err := server.NewServer(flagAddrD, flagAbci, app)
-	if err != nil {
-		return err
-	}
-	srv.SetLogger(logger.With("module", "abci-server"))
-	if err := srv.Start(); err != nil {
-		return err
-	}
-
-	// Wait forever
-	cmn.TrapSignal(func() {
-		// Cleanup
-		srv.Stop()
-	})
-	return nil
-}
-```

abci/client/client.go

@@ -105,8 +105,8 @@ func (reqRes *ReqRes) SetCallback(cb func(res *types.Response)) {
 		return
 	}
 
-	defer reqRes.mtx.Unlock()
 	reqRes.cb = cb
+	reqRes.mtx.Unlock()
 }
 
 func (reqRes *ReqRes) GetCallback() func(*types.Response) {

abci/client/grpc_client.go

@@ -54,7 +54,7 @@ RETRY_LOOP:
 			if cli.mustConnect {
 				return err
 			}
-			cli.Logger.Error(fmt.Sprintf("abci.grpcClient failed to connect to %v.  Retrying...\n", cli.addr))
+			cli.Logger.Error(fmt.Sprintf("abci.grpcClient failed to connect to %v.  Retrying...\n", cli.addr), "err", err)
 			time.Sleep(time.Second * dialRetryIntervalSeconds)
 			continue RETRY_LOOP
 		}
@@ -111,8 +111,8 @@ func (cli *grpcClient) Error() error {
 // NOTE: callback may get internally generated flush responses.
 func (cli *grpcClient) SetResponseCallback(resCb Callback) {
 	cli.mtx.Lock()
-	defer cli.mtx.Unlock()
 	cli.resCb = resCb
+	cli.mtx.Unlock()
 }
 
 //----------------------------------------
@@ -129,7 +129,7 @@ func (cli *grpcClient) EchoAsync(msg string) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Echo{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Echo{Echo: res}})
 }
 
 func (cli *grpcClient) FlushAsync() *ReqRes {
@@ -138,7 +138,7 @@ func (cli *grpcClient) FlushAsync() *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Flush{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Flush{Flush: res}})
 }
 
 func (cli *grpcClient) InfoAsync(params types.RequestInfo) *ReqRes {
@@ -147,7 +147,7 @@ func (cli *grpcClient) InfoAsync(params types.RequestInfo) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Info{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Info{Info: res}})
 }
 
 func (cli *grpcClient) SetOptionAsync(params types.RequestSetOption) *ReqRes {
@@ -156,7 +156,7 @@ func (cli *grpcClient) SetOptionAsync(params types.RequestSetOption) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_SetOption{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_SetOption{SetOption: res}})
 }
 
 func (cli *grpcClient) DeliverTxAsync(tx []byte) *ReqRes {
@@ -165,7 +165,7 @@ func (cli *grpcClient) DeliverTxAsync(tx []byte) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_DeliverTx{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_DeliverTx{DeliverTx: res}})
 }
 
 func (cli *grpcClient) CheckTxAsync(tx []byte) *ReqRes {
@@ -174,7 +174,7 @@ func (cli *grpcClient) CheckTxAsync(tx []byte) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_CheckTx{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_CheckTx{CheckTx: res}})
 }
 
 func (cli *grpcClient) QueryAsync(params types.RequestQuery) *ReqRes {
@@ -183,7 +183,7 @@ func (cli *grpcClient) QueryAsync(params types.RequestQuery) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Query{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Query{Query: res}})
 }
 
 func (cli *grpcClient) CommitAsync() *ReqRes {
@@ -192,7 +192,7 @@ func (cli *grpcClient) CommitAsync() *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Commit{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Commit{Commit: res}})
 }
 
 func (cli *grpcClient) InitChainAsync(params types.RequestInitChain) *ReqRes {
@@ -201,7 +201,7 @@ func (cli *grpcClient) InitChainAsync(params types.RequestInitChain) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_InitChain{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_InitChain{InitChain: res}})
 }
 
 func (cli *grpcClient) BeginBlockAsync(params types.RequestBeginBlock) *ReqRes {
@@ -210,7 +210,7 @@ func (cli *grpcClient) BeginBlockAsync(params types.RequestBeginBlock) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_BeginBlock{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_BeginBlock{BeginBlock: res}})
 }
 
 func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes {
@@ -219,7 +219,7 @@ func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_EndBlock{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_EndBlock{EndBlock: res}})
 }
 
 func (cli *grpcClient) finishAsyncCall(req *types.Request, res *types.Response) *ReqRes {

abci/client/local_client.go

@@ -9,8 +9,13 @@ import (
 
 var _ Client = (*localClient)(nil)
 
+// NOTE: use defer to unlock mutex because Application might panic (e.g., in
+// case of malicious tx or query). It only makes sense for publicly exposed
+// methods like CheckTx (/broadcast_tx_* RPC endpoint) or Query (/abci_query
+// RPC endpoint), but defers are used everywhere for the sake of consistency.
 type localClient struct {
 	cmn.BaseService
+
 	mtx *sync.Mutex
 	types.Application
 	Callback
@@ -30,8 +35,8 @@ func NewLocalClient(mtx *sync.Mutex, app types.Application) *localClient {
 
 func (app *localClient) SetResponseCallback(cb Callback) {
 	app.mtx.Lock()
-	defer app.mtx.Unlock()
 	app.Callback = cb
+	app.mtx.Unlock()
 }
 
 // TODO: change types.Application to include Error()?
@@ -45,6 +50,9 @@ func (app *localClient) FlushAsync() *ReqRes {
 }
 
 func (app *localClient) EchoAsync(msg string) *ReqRes {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	return app.callback(
 		types.ToRequestEcho(msg),
 		types.ToResponseEcho(msg),
@@ -53,8 +61,9 @@ func (app *localClient) EchoAsync(msg string) *ReqRes {
 
 func (app *localClient) InfoAsync(req types.RequestInfo) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.Info(req)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestInfo(req),
 		types.ToResponseInfo(res),
@@ -63,8 +72,9 @@ func (app *localClient) InfoAsync(req types.RequestInfo) *ReqRes {
 
 func (app *localClient) SetOptionAsync(req types.RequestSetOption) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.SetOption(req)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestSetOption(req),
 		types.ToResponseSetOption(res),
@@ -73,8 +83,9 @@ func (app *localClient) SetOptionAsync(req types.RequestSetOption) *ReqRes {
 
 func (app *localClient) DeliverTxAsync(tx []byte) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.DeliverTx(tx)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestDeliverTx(tx),
 		types.ToResponseDeliverTx(res),
@@ -83,8 +94,9 @@ func (app *localClient) DeliverTxAsync(tx []byte) *ReqRes {
 
 func (app *localClient) CheckTxAsync(tx []byte) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.CheckTx(tx)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestCheckTx(tx),
 		types.ToResponseCheckTx(res),
@@ -93,8 +105,9 @@ func (app *localClient) CheckTxAsync(tx []byte) *ReqRes {
 
 func (app *localClient) QueryAsync(req types.RequestQuery) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.Query(req)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestQuery(req),
 		types.ToResponseQuery(res),
@@ -103,8 +116,9 @@ func (app *localClient) QueryAsync(req types.RequestQuery) *ReqRes {
 
 func (app *localClient) CommitAsync() *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.Commit()
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestCommit(),
 		types.ToResponseCommit(res),
@@ -113,19 +127,20 @@ func (app *localClient) CommitAsync() *ReqRes {
 
 func (app *localClient) InitChainAsync(req types.RequestInitChain) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.InitChain(req)
-	reqRes := app.callback(
+	return app.callback(
 		types.ToRequestInitChain(req),
 		types.ToResponseInitChain(res),
 	)
-	app.mtx.Unlock()
-	return reqRes
 }
 
 func (app *localClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.BeginBlock(req)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestBeginBlock(req),
 		types.ToResponseBeginBlock(res),
@@ -134,8 +149,9 @@ func (app *localClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes {
 
 func (app *localClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.EndBlock(req)
-	app.mtx.Unlock()
 	return app.callback(
 		types.ToRequestEndBlock(req),
 		types.ToResponseEndBlock(res),
@@ -154,64 +170,73 @@ func (app *localClient) EchoSync(msg string) (*types.ResponseEcho, error) {
 
 func (app *localClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.Info(req)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.SetOption(req)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) DeliverTxSync(tx []byte) (*types.ResponseDeliverTx, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.DeliverTx(tx)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) CheckTxSync(tx []byte) (*types.ResponseCheckTx, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.CheckTx(tx)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.Query(req)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) CommitSync() (*types.ResponseCommit, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.Commit()
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.InitChain(req)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.BeginBlock(req)
-	app.mtx.Unlock()
 	return &res, nil
 }
 
 func (app *localClient) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) {
 	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
 	res := app.Application.EndBlock(req)
-	app.mtx.Unlock()
 	return &res, nil
 }
 

abci/client/socket_client.go

@@ -26,16 +26,17 @@ var _ Client = (*socketClient)(nil)
 type socketClient struct {
 	cmn.BaseService
 
-	reqQueue    chan *ReqRes
-	flushTimer  *cmn.ThrottleTimer
+	addr        string
 	mustConnect bool
+	conn        net.Conn
+
+	reqQueue   chan *ReqRes
+	flushTimer *cmn.ThrottleTimer
 
 	mtx     sync.Mutex
-	addr    string
-	conn    net.Conn
 	err     error
-	reqSent *list.List
-	resCb   func(*types.Request, *types.Response) // listens to all callbacks
+	reqSent *list.List                            // list of requests sent, waiting for response
+	resCb   func(*types.Request, *types.Response) // called on all requests, if set.
 
 }
 
@@ -67,7 +68,7 @@ RETRY_LOOP:
 			if cli.mustConnect {
 				return err
 			}
-			cli.Logger.Error(fmt.Sprintf("abci.socketClient failed to connect to %v.  Retrying...", cli.addr))
+			cli.Logger.Error(fmt.Sprintf("abci.socketClient failed to connect to %v.  Retrying...", cli.addr), "err", err)
 			time.Sleep(time.Second * dialRetryIntervalSeconds)
 			continue RETRY_LOOP
 		}
@@ -86,6 +87,7 @@ func (cli *socketClient) OnStop() {
 	cli.mtx.Lock()
 	defer cli.mtx.Unlock()
 	if cli.conn != nil {
+		// does this really need a mutex?
 		cli.conn.Close()
 	}
 
@@ -118,8 +120,8 @@ func (cli *socketClient) Error() error {
 // NOTE: callback may get internally generated flush responses.
 func (cli *socketClient) SetResponseCallback(resCb Callback) {
 	cli.mtx.Lock()
-	defer cli.mtx.Unlock()
 	cli.resCb = resCb
+	cli.mtx.Unlock()
 }
 
 //----------------------------------------
@@ -207,12 +209,15 @@ func (cli *socketClient) didRecvResponse(res *types.Response) error {
 	reqres.Done()            // Release waiters
 	cli.reqSent.Remove(next) // Pop first item from linked list
 
-	// Notify reqRes listener if set
+	// Notify reqRes listener if set (request specific callback).
+	// NOTE: it is possible this callback isn't set on the reqres object.
+	// at this point, in which case it will be called after, when it is set.
+	// TODO: should we move this after the resCb call so the order is always consistent?
 	if cb := reqres.GetCallback(); cb != nil {
 		cb(res)
 	}
 
-	// Notify client listener if set
+	// Notify client listener if set (global callback).
 	if cli.resCb != nil {
 		cli.resCb(reqres.Request, res)
 	}

abci/cmd/abci-cli/abci-cli.go

@@ -58,7 +58,7 @@ var RootCmd = &cobra.Command{
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 
 		switch cmd.Use {
-		case "counter", "kvstore", "dummy": // for the examples apps, don't pre-run
+		case "counter", "kvstore": // for the examples apps, don't pre-run
 			return nil
 		case "version": // skip running for version command
 			return nil
@@ -127,10 +127,6 @@ func addCounterFlags() {
 	counterCmd.PersistentFlags().BoolVarP(&flagSerial, "serial", "", false, "enforce incrementing (serial) transactions")
 }
 
-func addDummyFlags() {
-	dummyCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database")
-}
-
 func addKVStoreFlags() {
 	kvstoreCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database")
 }
@@ -152,10 +148,6 @@ func addCommands() {
 	// examples
 	addCounterFlags()
 	RootCmd.AddCommand(counterCmd)
-	// deprecated, left for backwards compatibility
-	addDummyFlags()
-	RootCmd.AddCommand(dummyCmd)
-	// replaces dummy, see issue #196
 	addKVStoreFlags()
 	RootCmd.AddCommand(kvstoreCmd)
 }
@@ -291,18 +283,6 @@ var counterCmd = &cobra.Command{
 	},
 }
 
-// deprecated, left for backwards compatibility
-var dummyCmd = &cobra.Command{
-	Use:        "dummy",
-	Deprecated: "use: [abci-cli kvstore] instead",
-	Short:      "ABCI demo example",
-	Long:       "ABCI demo example",
-	Args:       cobra.ExactArgs(0),
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return cmdKVStore(cmd, args)
-	},
-}
-
 var kvstoreCmd = &cobra.Command{
 	Use:   "kvstore",
 	Short: "ABCI demo example",
@@ -414,7 +394,6 @@ func cmdConsole(cmd *cobra.Command, args []string) error {
 			return err
 		}
 	}
-	return nil
 }
 
 func muxOnCommands(cmd *cobra.Command, pArgs []string) error {
@@ -657,9 +636,7 @@ func cmdQuery(cmd *cobra.Command, args []string) error {
 }
 
 func cmdCounter(cmd *cobra.Command, args []string) error {
-
 	app := counter.NewCounterApplication(flagSerial)
-
 	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
 
 	// Start the listener
@@ -672,12 +649,14 @@ func cmdCounter(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Wait forever
-	cmn.TrapSignal(func() {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
 		// Cleanup
 		srv.Stop()
 	})
-	return nil
+
+	// Run forever.
+	select {}
 }
 
 func cmdKVStore(cmd *cobra.Command, args []string) error {
@@ -702,12 +681,14 @@ func cmdKVStore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Wait forever
-	cmn.TrapSignal(func() {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
 		// Cleanup
 		srv.Stop()
 	})
-	return nil
+
+	// Run forever.
+	select {}
 }
 
 //--------------------------------------------------------------------------------

abci/types/messages_test.go

@@ -83,7 +83,7 @@ func TestWriteReadMessage2(t *testing.T) {
 			Log:       phrase,
 			GasWanted: 10,
 			Tags: []cmn.KVPair{
-				cmn.KVPair{Key: []byte("abc"), Value: []byte("def")},
+				{Key: []byte("abc"), Value: []byte("def")},
 			},
 		},
 		// TODO: add the rest

abci/types/types.proto

@@ -4,9 +4,9 @@ package types;
 // For more information on gogo.proto, see:
 // https://github.com/gogo/protobuf/blob/master/extensions.md
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-import "google/protobuf/timestamp.proto";
-import "github.com/tendermint/tendermint/libs/common/types.proto";
 import "github.com/tendermint/tendermint/crypto/merkle/merkle.proto";
+import "github.com/tendermint/tendermint/libs/common/types.proto";
+import "google/protobuf/timestamp.proto";
 
 // This file is copied from http://github.com/tendermint/abci
 // NOTE: When using custom types, mind the warnings.
@@ -207,13 +207,13 @@ message ResponseCommit {
 // ConsensusParams contains all consensus-relevant parameters
 // that can be adjusted by the abci app
 message ConsensusParams {
-  BlockSizeParams block_size = 1;
+  BlockParams block = 1;
   EvidenceParams evidence = 2;
   ValidatorParams validator = 3;
 }
 
-// BlockSize contains limits on the block size.
-message BlockSizeParams {
+// BlockParams contains limits on the block size.
+message BlockParams {
   // Note: must be greater than 0
   int64 max_bytes = 1;
   // Note: must be greater or equal to -1

abci/types/typespb_test.go

@@ -1479,15 +1479,15 @@ func TestConsensusParamsMarshalTo(t *testing.T) {
 	}
 }
 
-func TestBlockSizeParamsProto(t *testing.T) {
+func TestBlockParamsProto(t *testing.T) {
 	seed := time.Now().UnixNano()
 	popr := math_rand.New(math_rand.NewSource(seed))
-	p := NewPopulatedBlockSizeParams(popr, false)
+	p := NewPopulatedBlockParams(popr, false)
 	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
 	if err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
-	msg := &BlockSizeParams{}
+	msg := &BlockParams{}
 	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
@@ -1510,10 +1510,10 @@ func TestBlockSizeParamsProto(t *testing.T) {
 	}
 }
 
-func TestBlockSizeParamsMarshalTo(t *testing.T) {
+func TestBlockParamsMarshalTo(t *testing.T) {
 	seed := time.Now().UnixNano()
 	popr := math_rand.New(math_rand.NewSource(seed))
-	p := NewPopulatedBlockSizeParams(popr, false)
+	p := NewPopulatedBlockParams(popr, false)
 	size := p.Size()
 	dAtA := make([]byte, size)
 	for i := range dAtA {
@@ -1523,7 +1523,7 @@ func TestBlockSizeParamsMarshalTo(t *testing.T) {
 	if err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
-	msg := &BlockSizeParams{}
+	msg := &BlockParams{}
 	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
@@ -2675,16 +2675,16 @@ func TestConsensusParamsJSON(t *testing.T) {
 		t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
 	}
 }
-func TestBlockSizeParamsJSON(t *testing.T) {
+func TestBlockParamsJSON(t *testing.T) {
 	seed := time.Now().UnixNano()
 	popr := math_rand.New(math_rand.NewSource(seed))
-	p := NewPopulatedBlockSizeParams(popr, true)
+	p := NewPopulatedBlockParams(popr, true)
 	marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
 	jsondata, err := marshaler.MarshalToString(p)
 	if err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
-	msg := &BlockSizeParams{}
+	msg := &BlockParams{}
 	err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
 	if err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
@@ -3637,12 +3637,12 @@ func TestConsensusParamsProtoCompactText(t *testing.T) {
 	}
 }
 
-func TestBlockSizeParamsProtoText(t *testing.T) {
+func TestBlockParamsProtoText(t *testing.T) {
 	seed := time.Now().UnixNano()
 	popr := math_rand.New(math_rand.NewSource(seed))
-	p := NewPopulatedBlockSizeParams(popr, true)
+	p := NewPopulatedBlockParams(popr, true)
 	dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
-	msg := &BlockSizeParams{}
+	msg := &BlockParams{}
 	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
@@ -3651,12 +3651,12 @@ func TestBlockSizeParamsProtoText(t *testing.T) {
 	}
 }
 
-func TestBlockSizeParamsProtoCompactText(t *testing.T) {
+func TestBlockParamsProtoCompactText(t *testing.T) {
 	seed := time.Now().UnixNano()
 	popr := math_rand.New(math_rand.NewSource(seed))
-	p := NewPopulatedBlockSizeParams(popr, true)
+	p := NewPopulatedBlockParams(popr, true)
 	dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
-	msg := &BlockSizeParams{}
+	msg := &BlockParams{}
 	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
 		t.Fatalf("seed = %d, err = %v", seed, err)
 	}
@@ -4573,10 +4573,10 @@ func TestConsensusParamsSize(t *testing.T) {
 	}
 }
 
-func TestBlockSizeParamsSize(t *testing.T) {
+func TestBlockParamsSize(t *testing.T) {
 	seed := time.Now().UnixNano()
 	popr := math_rand.New(math_rand.NewSource(seed))
-	p := NewPopulatedBlockSizeParams(popr, true)
+	p := NewPopulatedBlockParams(popr, true)
 	size2 := github_com_gogo_protobuf_proto.Size(p)
 	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
 	if err != nil {

benchmarks/codec_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 
 	proto "github.com/tendermint/tendermint/benchmarks/proto"
 	"github.com/tendermint/tendermint/crypto/ed25519"

blockchain/pool.go

@@ -69,7 +69,7 @@ type BlockPool struct {
 	height     int64 // the lowest key in requesters.
 	// peers
 	peers         map[p2p.ID]*bpPeer
-	maxPeerHeight int64
+	maxPeerHeight int64 // the biggest reported height
 
 	// atomic
 	numPending int32 // number of requests pending assignment or block response
@@ -78,6 +78,8 @@ type BlockPool struct {
 	errorsCh   chan<- peerError
 }
 
+// NewBlockPool returns a new BlockPool with the height equal to start. Block
+// requests and errors will be sent to requestsCh and errorsCh accordingly.
 func NewBlockPool(start int64, requestsCh chan<- BlockRequest, errorsCh chan<- peerError) *BlockPool {
 	bp := &BlockPool{
 		peers: make(map[p2p.ID]*bpPeer),
@@ -93,15 +95,15 @@ func NewBlockPool(start int64, requestsCh chan<- BlockRequest, errorsCh chan<- p
 	return bp
 }
 
+// OnStart implements cmn.Service by spawning requesters routine and recording
+// pool's start time.
 func (pool *BlockPool) OnStart() error {
 	go pool.makeRequestersRoutine()
 	pool.startTime = time.Now()
 	return nil
 }
 
-func (pool *BlockPool) OnStop() {}
-
-// Run spawns requesters as needed.
+// spawns requesters as needed
 func (pool *BlockPool) makeRequestersRoutine() {
 	for {
 		if !pool.IsRunning() {
@@ -150,6 +152,8 @@ func (pool *BlockPool) removeTimedoutPeers() {
 	}
 }
 
+// GetStatus returns pool's height, numPending requests and the number of
+// requesters.
 func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequesters int) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
@@ -157,6 +161,7 @@ func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequester
 	return pool.height, atomic.LoadInt32(&pool.numPending), len(pool.requesters)
 }
 
+// IsCaughtUp returns true if this node is caught up, false - otherwise.
 // TODO: relax conditions, prevent abuse.
 func (pool *BlockPool) IsCaughtUp() bool {
 	pool.mtx.Lock()
@@ -168,9 +173,13 @@ func (pool *BlockPool) IsCaughtUp() bool {
 		return false
 	}
 
-	// some conditions to determine if we're caught up
-	receivedBlockOrTimedOut := (pool.height > 0 || time.Since(pool.startTime) > 5*time.Second)
-	ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= pool.maxPeerHeight
+	// Some conditions to determine if we're caught up.
+	// Ensures we've either received a block or waited some amount of time,
+	// and that we're synced to the highest known height.
+	// Note we use maxPeerHeight - 1 because to sync block H requires block H+1
+	// to verify the LastCommit.
+	receivedBlockOrTimedOut := pool.height > 0 || time.Since(pool.startTime) > 5*time.Second
+	ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= (pool.maxPeerHeight-1)
 	isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers
 	return isCaughtUp
 }
@@ -252,18 +261,19 @@ func (pool *BlockPool) AddBlock(peerID p2p.ID, block *types.Block, blockSize int
 			peer.decrPending(blockSize)
 		}
 	} else {
-		// Bad peer?
+		pool.Logger.Info("invalid peer", "peer", peerID, "blockHeight", block.Height)
+		pool.sendError(errors.New("invalid peer"), peerID)
 	}
 }
 
-// MaxPeerHeight returns the highest height reported by a peer.
+// MaxPeerHeight returns the highest reported height.
 func (pool *BlockPool) MaxPeerHeight() int64 {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 	return pool.maxPeerHeight
 }
 
-// Sets the peer's alleged blockchain height.
+// SetPeerHeight sets the peer's alleged blockchain height.
 func (pool *BlockPool) SetPeerHeight(peerID p2p.ID, height int64) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
@@ -282,6 +292,8 @@ func (pool *BlockPool) SetPeerHeight(peerID p2p.ID, height int64) {
 	}
 }
 
+// RemovePeer removes the peer with peerID from the pool. If there's no peer
+// with peerID, function is a no-op.
 func (pool *BlockPool) RemovePeer(peerID p2p.ID) {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
@@ -292,10 +304,35 @@ func (pool *BlockPool) RemovePeer(peerID p2p.ID) {
 func (pool *BlockPool) removePeer(peerID p2p.ID) {
 	for _, requester := range pool.requesters {
 		if requester.getPeerID() == peerID {
-			requester.redo()
+			requester.redo(peerID)
 		}
 	}
-	delete(pool.peers, peerID)
+
+	peer, ok := pool.peers[peerID]
+	if ok {
+		if peer.timeout != nil {
+			peer.timeout.Stop()
+		}
+
+		delete(pool.peers, peerID)
+
+		// Find a new peer with the biggest height and update maxPeerHeight if the
+		// peer's height was the biggest.
+		if peer.height == pool.maxPeerHeight {
+			pool.updateMaxPeerHeight()
+		}
+	}
+}
+
+// If no peers are left, maxPeerHeight is set to 0.
+func (pool *BlockPool) updateMaxPeerHeight() {
+	var max int64
+	for _, peer := range pool.peers {
+		if peer.height > max {
+			max = peer.height
+		}
+	}
+	pool.maxPeerHeight = max
 }
 
 // Pick an available peer with at least the given minHeight.
@@ -326,8 +363,11 @@ func (pool *BlockPool) makeNextRequester() {
 	defer pool.mtx.Unlock()
 
 	nextHeight := pool.height + pool.requestersLen()
+	if nextHeight > pool.maxPeerHeight {
+		return
+	}
+
 	request := newBPRequester(pool, nextHeight)
-	// request.SetLogger(pool.Logger.With("height", nextHeight))
 
 	pool.requesters[nextHeight] = request
 	atomic.AddInt32(&pool.numPending, 1)
@@ -356,7 +396,8 @@ func (pool *BlockPool) sendError(err error, peerID p2p.ID) {
 	pool.errorsCh <- peerError{err, peerID}
 }
 
-// unused by tendermint; left for debugging purposes
+// for debugging purposes
+//nolint:unused
 func (pool *BlockPool) debug() string {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
@@ -453,7 +494,7 @@ type bpRequester struct {
 	pool       *BlockPool
 	height     int64
 	gotBlockCh chan struct{}
-	redoCh     chan struct{}
+	redoCh     chan p2p.ID //redo may send multitime, add peerId to identify repeat
 
 	mtx    sync.Mutex
 	peerID p2p.ID
@@ -465,7 +506,7 @@ func newBPRequester(pool *BlockPool, height int64) *bpRequester {
 		pool:       pool,
 		height:     height,
 		gotBlockCh: make(chan struct{}, 1),
-		redoCh:     make(chan struct{}, 1),
+		redoCh:     make(chan p2p.ID, 1),
 
 		peerID: "",
 		block:  nil,
@@ -524,9 +565,9 @@ func (bpr *bpRequester) reset() {
 // Tells bpRequester to pick another peer and try again.
 // NOTE: Nonblocking, and does nothing if another redo
 // was already requested.
-func (bpr *bpRequester) redo() {
+func (bpr *bpRequester) redo(peerId p2p.ID) {
 	select {
-	case bpr.redoCh <- struct{}{}:
+	case bpr.redoCh <- peerId:
 	default:
 	}
 }
@@ -565,9 +606,13 @@ OUTER_LOOP:
 				return
 			case <-bpr.Quit():
 				return
-			case <-bpr.redoCh:
-				bpr.reset()
-				continue OUTER_LOOP
+			case peerID := <-bpr.redoCh:
+				if peerID == bpr.peerID {
+					bpr.reset()
+					continue OUTER_LOOP
+				} else {
+					continue WAIT_LOOP
+				}
 			case <-bpr.gotBlockCh:
 				// We got a block!
 				// Continue the for-loop and wait til Quit.

blockchain/pool_test.go

@@ -1,12 +1,15 @@
 package blockchain
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
-
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
 )
@@ -16,21 +19,59 @@ func init() {
 }
 
 type testPeer struct {
-	id     p2p.ID
-	height int64
+	id        p2p.ID
+	height    int64
+	inputChan chan inputData //make sure each peer's data is sequential
 }
 
-func makePeers(numPeers int, minHeight, maxHeight int64) map[p2p.ID]testPeer {
-	peers := make(map[p2p.ID]testPeer, numPeers)
+type inputData struct {
+	t       *testing.T
+	pool    *BlockPool
+	request BlockRequest
+}
+
+func (p testPeer) runInputRoutine() {
+	go func() {
+		for input := range p.inputChan {
+			p.simulateInput(input)
+		}
+	}()
+}
+
+// Request desired, pretend like we got the block immediately.
+func (p testPeer) simulateInput(input inputData) {
+	block := &types.Block{Header: types.Header{Height: input.request.Height}}
+	input.pool.AddBlock(input.request.PeerID, block, 123)
+	// TODO: uncommenting this creates a race which is detected by: https://github.com/golang/go/blob/2bd767b1022dd3254bcec469f0ee164024726486/src/testing/testing.go#L854-L856
+	// see: https://github.com/tendermint/tendermint/issues/3390#issue-418379890
+	// input.t.Logf("Added block from peer %v (height: %v)", input.request.PeerID, input.request.Height)
+}
+
+type testPeers map[p2p.ID]testPeer
+
+func (ps testPeers) start() {
+	for _, v := range ps {
+		v.runInputRoutine()
+	}
+}
+
+func (ps testPeers) stop() {
+	for _, v := range ps {
+		close(v.inputChan)
+	}
+}
+
+func makePeers(numPeers int, minHeight, maxHeight int64) testPeers {
+	peers := make(testPeers, numPeers)
 	for i := 0; i < numPeers; i++ {
 		peerID := p2p.ID(cmn.RandStr(12))
 		height := minHeight + cmn.RandInt63n(maxHeight-minHeight)
-		peers[peerID] = testPeer{peerID, height}
+		peers[peerID] = testPeer{peerID, height, make(chan inputData, 10)}
 	}
 	return peers
 }
 
-func TestBasic(t *testing.T) {
+func TestBlockPoolBasic(t *testing.T) {
 	start := int64(42)
 	peers := makePeers(10, start+1, 1000)
 	errorsCh := make(chan peerError, 1000)
@@ -45,6 +86,9 @@ func TestBasic(t *testing.T) {
 
 	defer pool.Stop()
 
+	peers.start()
+	defer peers.stop()
+
 	// Introduce each peer.
 	go func() {
 		for _, peer := range peers {
@@ -77,17 +121,13 @@ func TestBasic(t *testing.T) {
 			if request.Height == 300 {
 				return // Done!
 			}
-			// Request desired, pretend like we got the block immediately.
-			go func() {
-				block := &types.Block{Header: types.Header{Height: request.Height}}
-				pool.AddBlock(request.PeerID, block, 123)
-				t.Logf("Added block from peer %v (height: %v)", request.PeerID, request.Height)
-			}()
+
+			peers[request.PeerID].inputChan <- inputData{t, pool, request}
 		}
 	}
 }
 
-func TestTimeout(t *testing.T) {
+func TestBlockPoolTimeout(t *testing.T) {
 	start := int64(42)
 	peers := makePeers(10, start+1, 1000)
 	errorsCh := make(chan peerError, 1000)
@@ -145,3 +185,40 @@ func TestTimeout(t *testing.T) {
 		}
 	}
 }
+
+func TestBlockPoolRemovePeer(t *testing.T) {
+	peers := make(testPeers, 10)
+	for i := 0; i < 10; i++ {
+		peerID := p2p.ID(fmt.Sprintf("%d", i+1))
+		height := int64(i + 1)
+		peers[peerID] = testPeer{peerID, height, make(chan inputData)}
+	}
+	requestsCh := make(chan BlockRequest)
+	errorsCh := make(chan peerError)
+
+	pool := NewBlockPool(1, requestsCh, errorsCh)
+	pool.SetLogger(log.TestingLogger())
+	err := pool.Start()
+	require.NoError(t, err)
+	defer pool.Stop()
+
+	// add peers
+	for peerID, peer := range peers {
+		pool.SetPeerHeight(peerID, peer.height)
+	}
+	assert.EqualValues(t, 10, pool.MaxPeerHeight())
+
+	// remove not-existing peer
+	assert.NotPanics(t, func() { pool.RemovePeer(p2p.ID("Superman")) })
+
+	// remove peer with biggest height
+	pool.RemovePeer(p2p.ID("10"))
+	assert.EqualValues(t, 9, pool.MaxPeerHeight())
+
+	// remove all peers
+	for peerID := range peers {
+		pool.RemovePeer(peerID)
+	}
+
+	assert.EqualValues(t, 0, pool.MaxPeerHeight())
+}

blockchain/reactor.go

@@ -8,7 +8,6 @@ import (
 
 	amino "github.com/tendermint/go-amino"
 
-	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
 	"github.com/tendermint/tendermint/p2p"
 	sm "github.com/tendermint/tendermint/state"
@@ -229,32 +228,40 @@ func (bcR *BlockchainReactor) poolRoutine() {
 
 	didProcessCh := make(chan struct{}, 1)
 
+	go func() {
+		for {
+			select {
+			case <-bcR.Quit():
+				return
+			case <-bcR.pool.Quit():
+				return
+			case request := <-bcR.requestsCh:
+				peer := bcR.Switch.Peers().Get(request.PeerID)
+				if peer == nil {
+					continue
+				}
+				msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height})
+				queued := peer.TrySend(BlockchainChannel, msgBytes)
+				if !queued {
+					bcR.Logger.Debug("Send queue is full, drop block request", "peer", peer.ID(), "height", request.Height)
+				}
+			case err := <-bcR.errorsCh:
+				peer := bcR.Switch.Peers().Get(err.peerID)
+				if peer != nil {
+					bcR.Switch.StopPeerForError(peer, err)
+				}
+
+			case <-statusUpdateTicker.C:
+				// ask for status updates
+				go bcR.BroadcastStatusRequest() // nolint: errcheck
+
+			}
+		}
+	}()
+
 FOR_LOOP:
 	for {
 		select {
-		case request := <-bcR.requestsCh:
-			peer := bcR.Switch.Peers().Get(request.PeerID)
-			if peer == nil {
-				continue FOR_LOOP // Peer has since been disconnected.
-			}
-			msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height})
-			queued := peer.TrySend(BlockchainChannel, msgBytes)
-			if !queued {
-				// We couldn't make the request, send-queue full.
-				// The pool handles timeouts, just let it go.
-				continue FOR_LOOP
-			}
-
-		case err := <-bcR.errorsCh:
-			peer := bcR.Switch.Peers().Get(err.peerID)
-			if peer != nil {
-				bcR.Switch.StopPeerForError(peer, err)
-			}
-
-		case <-statusUpdateTicker.C:
-			// ask for status updates
-			go bcR.BroadcastStatusRequest() // nolint: errcheck
-
 		case <-switchToConsensusTicker.C:
 			height, numPending, lenRequesters := bcR.pool.GetStatus()
 			outbound, inbound, _ := bcR.Switch.NumPeers()
@@ -263,9 +270,12 @@ FOR_LOOP:
 			if bcR.pool.IsCaughtUp() {
 				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
 				bcR.pool.Stop()
-
-				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
-				conR.SwitchToConsensus(state, blocksSynced)
+				conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
+				if ok {
+					conR.SwitchToConsensus(state, blocksSynced)
+				} else {
+					// should only happen during testing
+				}
 
 				break FOR_LOOP
 			}
@@ -298,7 +308,7 @@ FOR_LOOP:
 
 			firstParts := first.MakePartSet(types.BlockPartSizeBytes)
 			firstPartsHeader := firstParts.Header()
-			firstID := types.BlockID{first.Hash(), firstPartsHeader}
+			firstID := types.BlockID{Hash: first.Hash(), PartsHeader: firstPartsHeader}
 			// Finally, verify the first block using the second's commit
 			// NOTE: we can probably make this more efficient, but note that calling
 			// first.Hash() doesn't verify the tx contents, so MakePartSet() is
@@ -314,6 +324,13 @@ FOR_LOOP:
 					// still need to clean up the rest.
 					bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err))
 				}
+				peerID2 := bcR.pool.RedoRequest(second.Height)
+				peer2 := bcR.Switch.Peers().Get(peerID2)
+				if peer2 != nil && peer2 != peer {
+					// NOTE: we've already removed the peer's request, but we
+					// still need to clean up the rest.
+					bcR.Switch.StopPeerForError(peer2, fmt.Errorf("BlockchainReactor validation error: %v", err))
+				}
 				continue FOR_LOOP
 			} else {
 				bcR.pool.PopRequest()
@@ -327,8 +344,7 @@ FOR_LOOP:
 				state, err = bcR.blockExec.ApplyBlock(state, firstID, first)
 				if err != nil {
 					// TODO This is bad, are we zombie?
-					cmn.PanicQ(fmt.Sprintf("Failed to process committed block (%d:%X): %v",
-						first.Height, first.Hash(), err))
+					panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
 				}
 				blocksSynced++
 
@@ -421,11 +437,7 @@ type bcBlockResponseMessage struct {
 
 // ValidateBasic performs basic validation.
 func (m *bcBlockResponseMessage) ValidateBasic() error {
-	if err := m.Block.ValidateBasic(); err != nil {
-		return err
-	}
-
-	return nil
+	return m.Block.ValidateBasic()
 }
 
 func (m *bcBlockResponseMessage) String() string {

blockchain/reactor_test.go

@@ -1,72 +1,156 @@
 package blockchain
 
 import (
-	"net"
+	"os"
+	"sort"
 	"testing"
+	"time"
 
+	"github.com/stretchr/testify/assert"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
-
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/mock"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
 )
 
-func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) {
-	config := cfg.ResetTestRoot("blockchain_reactor_test")
-	// blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB())
-	// stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB())
+var config *cfg.Config
+
+func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
+	validators := make([]types.GenesisValidator, numValidators)
+	privValidators := make([]types.PrivValidator, numValidators)
+	for i := 0; i < numValidators; i++ {
+		val, privVal := types.RandValidator(randPower, minPower)
+		validators[i] = types.GenesisValidator{
+			PubKey: val.PubKey,
+			Power:  val.VotingPower,
+		}
+		privValidators[i] = privVal
+	}
+	sort.Sort(types.PrivValidatorsByAddress(privValidators))
+
+	return &types.GenesisDoc{
+		GenesisTime: tmtime.Now(),
+		ChainID:     config.ChainID(),
+		Validators:  validators,
+	}, privValidators
+}
+
+func makeVote(header *types.Header, blockID types.BlockID, valset *types.ValidatorSet, privVal types.PrivValidator) *types.Vote {
+	addr := privVal.GetPubKey().Address()
+	idx, _ := valset.GetByAddress(addr)
+	vote := &types.Vote{
+		ValidatorAddress: addr,
+		ValidatorIndex:   idx,
+		Height:           header.Height,
+		Round:            1,
+		Timestamp:        tmtime.Now(),
+		Type:             types.PrecommitType,
+		BlockID:          blockID,
+	}
+
+	privVal.SignVote(header.ChainID, vote)
+
+	return vote
+}
+
+type BlockchainReactorPair struct {
+	reactor *BlockchainReactor
+	app     proxy.AppConns
+}
+
+func newBlockchainReactor(logger log.Logger, genDoc *types.GenesisDoc, privVals []types.PrivValidator, maxBlockHeight int64) BlockchainReactorPair {
+	if len(privVals) != 1 {
+		panic("only support one validator")
+	}
+
+	app := &testApp{}
+	cc := proxy.NewLocalClientCreator(app)
+	proxyApp := proxy.NewAppConns(cc)
+	err := proxyApp.Start()
+	if err != nil {
+		panic(cmn.ErrorWrap(err, "error start app"))
+	}
+
 	blockDB := dbm.NewMemDB()
 	stateDB := dbm.NewMemDB()
 	blockStore := NewBlockStore(blockDB)
-	state, err := sm.LoadStateFromDBOrGenesisFile(stateDB, config.GenesisFile())
+
+	state, err := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
 	if err != nil {
 		panic(cmn.ErrorWrap(err, "error constructing state from genesis file"))
 	}
-	return state, blockStore
-}
 
-func newBlockchainReactor(logger log.Logger, maxBlockHeight int64) *BlockchainReactor {
-	state, blockStore := makeStateAndBlockStore(logger)
-
-	// Make the blockchainReactor itself
+	// Make the BlockchainReactor itself.
+	// NOTE we have to create and commit the blocks first because
+	// pool.height is determined from the store.
 	fastSync := true
-	var nilApp proxy.AppConnConsensus
-	blockExec := sm.NewBlockExecutor(dbm.NewMemDB(), log.TestingLogger(), nilApp,
-		sm.MockMempool{}, sm.MockEvidencePool{})
+	db := dbm.NewMemDB()
+	blockExec := sm.NewBlockExecutor(db, log.TestingLogger(), proxyApp.Consensus(),
+		mock.Mempool{}, sm.MockEvidencePool{})
+	sm.SaveState(db, state)
+
+	// let's add some blocks in
+	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
+		lastCommit := types.NewCommit(types.BlockID{}, nil)
+		if blockHeight > 1 {
+			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
+			lastBlock := blockStore.LoadBlock(blockHeight - 1)
+
+			vote := makeVote(&lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVals[0]).CommitSig()
+			lastCommit = types.NewCommit(lastBlockMeta.BlockID, []*types.CommitSig{vote})
+		}
+
+		thisBlock := makeBlock(blockHeight, state, lastCommit)
+
+		thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes)
+		blockID := types.BlockID{thisBlock.Hash(), thisParts.Header()}
+
+		state, err = blockExec.ApplyBlock(state, blockID, thisBlock)
+		if err != nil {
+			panic(cmn.ErrorWrap(err, "error apply block"))
+		}
+
+		blockStore.SaveBlock(thisBlock, thisParts, lastCommit)
+	}
 
 	bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync)
 	bcReactor.SetLogger(logger.With("module", "blockchain"))
 
-	// Next: we need to set a switch in order for peers to be added in
-	bcReactor.Switch = p2p.NewSwitch(cfg.DefaultP2PConfig(), nil)
-
-	// Lastly: let's add some blocks in
-	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
-		firstBlock := makeBlock(blockHeight, state)
-		secondBlock := makeBlock(blockHeight+1, state)
-		firstParts := firstBlock.MakePartSet(types.BlockPartSizeBytes)
-		blockStore.SaveBlock(firstBlock, firstParts, secondBlock.LastCommit)
-	}
-
-	return bcReactor
+	return BlockchainReactorPair{bcReactor, proxyApp}
 }
 
 func TestNoBlockResponse(t *testing.T) {
-	maxBlockHeight := int64(20)
+	config = cfg.ResetTestRoot("blockchain_reactor_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(1, false, 30)
 
-	bcr := newBlockchainReactor(log.TestingLogger(), maxBlockHeight)
-	bcr.Start()
-	defer bcr.Stop()
+	maxBlockHeight := int64(65)
 
-	// Add some peers in
-	peer := newbcrTestPeer(p2p.ID(cmn.RandStr(12)))
-	bcr.AddPeer(peer)
+	reactorPairs := make([]BlockchainReactorPair, 2)
 
-	chID := byte(0x01)
+	reactorPairs[0] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight)
+	reactorPairs[1] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+
+	p2p.MakeConnectedSwitches(config.P2P, 2, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor)
+		return s
+
+	}, p2p.Connect2Switches)
+
+	defer func() {
+		for _, r := range reactorPairs {
+			r.reactor.Stop()
+			r.app.Stop()
+		}
+	}()
 
 	tests := []struct {
 		height   int64
@@ -78,72 +162,101 @@ func TestNoBlockResponse(t *testing.T) {
 		{100, false},
 	}
 
-	// receive a request message from peer,
-	// wait for our response to be received on the peer
-	for _, tt := range tests {
-		reqBlockMsg := &bcBlockRequestMessage{tt.height}
-		reqBlockBytes := cdc.MustMarshalBinaryBare(reqBlockMsg)
-		bcr.Receive(chID, peer, reqBlockBytes)
-		msg := peer.lastBlockchainMessage()
+	for {
+		if reactorPairs[1].reactor.pool.IsCaughtUp() {
+			break
+		}
 
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	assert.Equal(t, maxBlockHeight, reactorPairs[0].reactor.store.Height())
+
+	for _, tt := range tests {
+		block := reactorPairs[1].reactor.store.LoadBlock(tt.height)
 		if tt.existent {
-			if blockMsg, ok := msg.(*bcBlockResponseMessage); !ok {
-				t.Fatalf("Expected to receive a block response for height %d", tt.height)
-			} else if blockMsg.Block.Height != tt.height {
-				t.Fatalf("Expected response to be for height %d, got %d", tt.height, blockMsg.Block.Height)
-			}
+			assert.True(t, block != nil)
 		} else {
-			if noBlockMsg, ok := msg.(*bcNoBlockResponseMessage); !ok {
-				t.Fatalf("Expected to receive a no block response for height %d", tt.height)
-			} else if noBlockMsg.Height != tt.height {
-				t.Fatalf("Expected response to be for height %d, got %d", tt.height, noBlockMsg.Height)
-			}
+			assert.True(t, block == nil)
 		}
 	}
 }
 
-/*
 // NOTE: This is too hard to test without
 // an easy way to add test peer to switch
 // or without significant refactoring of the module.
 // Alternatively we could actually dial a TCP conn but
 // that seems extreme.
 func TestBadBlockStopsPeer(t *testing.T) {
-	maxBlockHeight := int64(20)
+	config = cfg.ResetTestRoot("blockchain_reactor_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(1, false, 30)
 
-	bcr := newBlockchainReactor(log.TestingLogger(), maxBlockHeight)
-	bcr.Start()
-	defer bcr.Stop()
+	maxBlockHeight := int64(148)
 
-	// Add some peers in
-	peer := newbcrTestPeer(p2p.ID(cmn.RandStr(12)))
+	otherChain := newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight)
+	defer func() {
+		otherChain.reactor.Stop()
+		otherChain.app.Stop()
+	}()
 
-	// XXX: This doesn't add the peer to anything,
-	// so it's hard to check that it's later removed
-	bcr.AddPeer(peer)
-	assert.True(t, bcr.Switch.Peers().Size() > 0)
+	reactorPairs := make([]BlockchainReactorPair, 4)
 
-	// send a bad block from the peer
-	// default blocks already dont have commits, so should fail
-	block := bcr.store.LoadBlock(3)
-	msg := &bcBlockResponseMessage{Block: block}
-	peer.Send(BlockchainChannel, struct{ BlockchainMessage }{msg})
+	reactorPairs[0] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight)
+	reactorPairs[1] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+	reactorPairs[2] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+	reactorPairs[3] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
 
-	ticker := time.NewTicker(time.Millisecond * 10)
-	timer := time.NewTimer(time.Second * 2)
-LOOP:
-	for {
-		select {
-		case <-ticker.C:
-			if bcr.Switch.Peers().Size() == 0 {
-				break LOOP
-			}
-		case <-timer.C:
-			t.Fatal("Timed out waiting to disconnect peer")
+	switches := p2p.MakeConnectedSwitches(config.P2P, 4, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor)
+		return s
+
+	}, p2p.Connect2Switches)
+
+	defer func() {
+		for _, r := range reactorPairs {
+			r.reactor.Stop()
+			r.app.Stop()
 		}
+	}()
+
+	for {
+		if reactorPairs[3].reactor.pool.IsCaughtUp() {
+			break
+		}
+
+		time.Sleep(1 * time.Second)
 	}
+
+	//at this time, reactors[0-3] is the newest
+	assert.Equal(t, 3, reactorPairs[1].reactor.Switch.Peers().Size())
+
+	//mark reactorPairs[3] is an invalid peer
+	reactorPairs[3].reactor.store = otherChain.reactor.store
+
+	lastReactorPair := newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+	reactorPairs = append(reactorPairs, lastReactorPair)
+
+	switches = append(switches, p2p.MakeConnectedSwitches(config.P2P, 1, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[len(reactorPairs)-1].reactor)
+		return s
+
+	}, p2p.Connect2Switches)...)
+
+	for i := 0; i < len(reactorPairs)-1; i++ {
+		p2p.Connect2Switches(switches, i, len(reactorPairs)-1)
+	}
+
+	for {
+		if lastReactorPair.reactor.pool.IsCaughtUp() || lastReactorPair.reactor.Switch.Peers().Size() == 0 {
+			break
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+
+	assert.True(t, lastReactorPair.reactor.Switch.Peers().Size() < len(reactorPairs)-1)
 }
-*/
 
 //----------------------------------------------
 // utility funcs
@@ -155,55 +268,41 @@ func makeTxs(height int64) (txs []types.Tx) {
 	return txs
 }
 
-func makeBlock(height int64, state sm.State) *types.Block {
-	block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit), nil, state.Validators.GetProposer().Address)
+func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block {
+	block, _ := state.MakeBlock(height, makeTxs(height), lastCommit, nil, state.Validators.GetProposer().Address)
 	return block
 }
 
-// The Test peer
-type bcrTestPeer struct {
-	cmn.BaseService
-	id p2p.ID
-	ch chan interface{}
+type testApp struct {
+	abci.BaseApplication
 }
 
-var _ p2p.Peer = (*bcrTestPeer)(nil)
+var _ abci.Application = (*testApp)(nil)
 
-func newbcrTestPeer(id p2p.ID) *bcrTestPeer {
-	bcr := &bcrTestPeer{
-		id: id,
-		ch: make(chan interface{}, 2),
-	}
-	bcr.BaseService = *cmn.NewBaseService(nil, "bcrTestPeer", bcr)
-	return bcr
+func (app *testApp) Info(req abci.RequestInfo) (resInfo abci.ResponseInfo) {
+	return abci.ResponseInfo{}
 }
 
-func (tp *bcrTestPeer) lastBlockchainMessage() interface{} { return <-tp.ch }
-
-func (tp *bcrTestPeer) TrySend(chID byte, msgBytes []byte) bool {
-	var msg BlockchainMessage
-	err := cdc.UnmarshalBinaryBare(msgBytes, &msg)
-	if err != nil {
-		panic(cmn.ErrorWrap(err, "Error while trying to parse a BlockchainMessage"))
-	}
-	if _, ok := msg.(*bcStatusResponseMessage); ok {
-		// Discard status response messages since they skew our results
-		// We only want to deal with:
-		// + bcBlockResponseMessage
-		// + bcNoBlockResponseMessage
-	} else {
-		tp.ch <- msg
-	}
-	return true
+func (app *testApp) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginBlock {
+	return abci.ResponseBeginBlock{}
 }
 
-func (tp *bcrTestPeer) Send(chID byte, msgBytes []byte) bool { return tp.TrySend(chID, msgBytes) }
-func (tp *bcrTestPeer) NodeInfo() p2p.NodeInfo               { return p2p.DefaultNodeInfo{} }
-func (tp *bcrTestPeer) Status() p2p.ConnectionStatus         { return p2p.ConnectionStatus{} }
-func (tp *bcrTestPeer) ID() p2p.ID                           { return tp.id }
-func (tp *bcrTestPeer) IsOutbound() bool                     { return false }
-func (tp *bcrTestPeer) IsPersistent() bool                   { return true }
-func (tp *bcrTestPeer) Get(s string) interface{}             { return s }
-func (tp *bcrTestPeer) Set(string, interface{})              {}
-func (tp *bcrTestPeer) RemoteIP() net.IP                     { return []byte{127, 0, 0, 1} }
-func (tp *bcrTestPeer) OriginalAddr() *p2p.NetAddress        { return nil }
+func (app *testApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
+	return abci.ResponseEndBlock{}
+}
+
+func (app *testApp) DeliverTx(tx []byte) abci.ResponseDeliverTx {
+	return abci.ResponseDeliverTx{Tags: []cmn.KVPair{}}
+}
+
+func (app *testApp) CheckTx(tx []byte) abci.ResponseCheckTx {
+	return abci.ResponseCheckTx{}
+}
+
+func (app *testApp) Commit() abci.ResponseCommit {
+	return abci.ResponseCommit{}
+}
+
+func (app *testApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
+	return
+}

blockchain/store.go

@@ -144,14 +144,14 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
 //             most recent height.  Otherwise they'd stall at H-1.
 func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
 	if block == nil {
-		cmn.PanicSanity("BlockStore can only save a non-nil block")
+		panic("BlockStore can only save a non-nil block")
 	}
 	height := block.Height
 	if g, w := height, bs.Height()+1; g != w {
-		cmn.PanicSanity(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", w, g))
+		panic(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", w, g))
 	}
 	if !blockParts.IsComplete() {
-		cmn.PanicSanity(fmt.Sprintf("BlockStore can only save complete block part sets"))
+		panic(fmt.Sprintf("BlockStore can only save complete block part sets"))
 	}
 
 	// Save block meta
@@ -188,7 +188,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
 
 func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) {
 	if height != bs.Height()+1 {
-		cmn.PanicSanity(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
+		panic(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height))
 	}
 	partBytes := cdc.MustMarshalBinaryBare(part)
 	bs.db.Set(calcBlockPartKey(height, index), partBytes)
@@ -224,7 +224,7 @@ type BlockStoreStateJSON struct {
 func (bsj BlockStoreStateJSON) Save(db dbm.DB) {
 	bytes, err := cdc.MarshalJSON(bsj)
 	if err != nil {
-		cmn.PanicSanity(fmt.Sprintf("Could not marshal state bytes: %v", err))
+		panic(fmt.Sprintf("Could not marshal state bytes: %v", err))
 	}
 	db.SetSync(blockStoreKey, bytes)
 }

blockchain/store_test.go

@@ -3,19 +3,48 @@ package blockchain
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"runtime/debug"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	cfg "github.com/tendermint/tendermint/config"
+	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/db"
+	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	sm "github.com/tendermint/tendermint/state"
 
 	"github.com/tendermint/tendermint/types"
 	tmtime "github.com/tendermint/tendermint/types/time"
 )
 
+// A cleanupFunc cleans up any config / test files created for a particular
+// test.
+type cleanupFunc func()
+
+// make a Commit with a single vote containing just the height and a timestamp
+func makeTestCommit(height int64, timestamp time.Time) *types.Commit {
+	commitSigs := []*types.CommitSig{{Height: height, Timestamp: timestamp}}
+	return types.NewCommit(types.BlockID{}, commitSigs)
+}
+
+func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFunc) {
+	config := cfg.ResetTestRoot("blockchain_reactor_test")
+	// blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB())
+	// stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB())
+	blockDB := dbm.NewMemDB()
+	stateDB := dbm.NewMemDB()
+	state, err := sm.LoadStateFromDBOrGenesisFile(stateDB, config.GenesisFile())
+	if err != nil {
+		panic(cmn.ErrorWrap(err, "error constructing state from genesis file"))
+	}
+	return state, NewBlockStore(blockDB), func() { os.RemoveAll(config.RootDir) }
+}
+
 func TestLoadBlockStoreStateJSON(t *testing.T) {
 	db := db.NewMemDB()
 
@@ -63,20 +92,32 @@ func freshBlockStore() (*BlockStore, db.DB) {
 }
 
 var (
-	state, _ = makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
-
-	block       = makeBlock(1, state)
-	partSet     = block.MakePartSet(2)
-	part1       = partSet.GetPart(0)
-	part2       = partSet.GetPart(1)
-	seenCommit1 = &types.Commit{Precommits: []*types.Vote{{Height: 10,
-		Timestamp: tmtime.Now()}}}
+	state       sm.State
+	block       *types.Block
+	partSet     *types.PartSet
+	part1       *types.Part
+	part2       *types.Part
+	seenCommit1 *types.Commit
 )
 
+func TestMain(m *testing.M) {
+	var cleanup cleanupFunc
+	state, _, cleanup = makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	block = makeBlock(1, state, new(types.Commit))
+	partSet = block.MakePartSet(2)
+	part1 = partSet.GetPart(0)
+	part2 = partSet.GetPart(1)
+	seenCommit1 = makeTestCommit(10, tmtime.Now())
+	code := m.Run()
+	cleanup()
+	os.Exit(code)
+}
+
 // TODO: This test should be simplified ...
 
 func TestBlockStoreSaveLoadBlock(t *testing.T) {
-	state, bs := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	state, bs, cleanup := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	defer cleanup()
 	require.Equal(t, bs.Height(), int64(0), "initially the height should be zero")
 
 	// check there are no blocks at various heights
@@ -88,10 +129,9 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 	}
 
 	// save a block
-	block := makeBlock(bs.Height()+1, state)
+	block := makeBlock(bs.Height()+1, state, new(types.Commit))
 	validPartSet := block.MakePartSet(2)
-	seenCommit := &types.Commit{Precommits: []*types.Vote{{Height: 10,
-		Timestamp: tmtime.Now()}}}
+	seenCommit := makeTestCommit(10, tmtime.Now())
 	bs.SaveBlock(block, partSet, seenCommit)
 	require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed")
 
@@ -110,8 +150,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 
 	// End of setup, test data
 
-	commitAtH10 := &types.Commit{Precommits: []*types.Vote{{Height: 10,
-		Timestamp: tmtime.Now()}}}
+	commitAtH10 := makeTestCommit(10, tmtime.Now())
 	tuples := []struct {
 		block      *types.Block
 		parts      *types.PartSet
@@ -294,7 +333,7 @@ func TestLoadBlockPart(t *testing.T) {
 	gotPart, _, panicErr := doFn(loadPart)
 	require.Nil(t, panicErr, "an existent and proper block should not panic")
 	require.Nil(t, res, "a properly saved block should return a proper block")
-	require.Equal(t, gotPart.(*types.Part).Hash(), part1.Hash(),
+	require.Equal(t, gotPart.(*types.Part), part1,
 		"expecting successful retrieval of previously saved block")
 }
 
@@ -329,14 +368,13 @@ func TestLoadBlockMeta(t *testing.T) {
 }
 
 func TestBlockFetchAtHeight(t *testing.T) {
-	state, bs := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	state, bs, cleanup := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
+	defer cleanup()
 	require.Equal(t, bs.Height(), int64(0), "initially the height should be zero")
-	block := makeBlock(bs.Height()+1, state)
+	block := makeBlock(bs.Height()+1, state, new(types.Commit))
 
 	partSet := block.MakePartSet(2)
-	seenCommit := &types.Commit{Precommits: []*types.Vote{{Height: 10,
-		Timestamp: tmtime.Now()}}}
-
+	seenCommit := makeTestCommit(10, tmtime.Now())
 	bs.SaveBlock(block, partSet, seenCommit)
 	require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed")
 

blockchain/wire.go

@@ -1,7 +1,7 @@
 package blockchain
 
 import (
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 	"github.com/tendermint/tendermint/types"
 )
 

cmd/priv_val_server/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"os"
+	"time"
 
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
@@ -13,9 +14,10 @@ import (
 
 func main() {
 	var (
-		addr        = flag.String("addr", ":26659", "Address of client to connect to")
-		chainID     = flag.String("chain-id", "mychain", "chain id")
-		privValPath = flag.String("priv", "", "priv val file path")
+		addr             = flag.String("addr", ":26659", "Address of client to connect to")
+		chainID          = flag.String("chain-id", "mychain", "chain id")
+		privValKeyPath   = flag.String("priv-key", "", "priv val key file path")
+		privValStatePath = flag.String("priv-state", "", "priv val state file path")
 
 		logger = log.NewTMLogger(
 			log.NewSyncWriter(os.Stdout),
@@ -27,27 +29,39 @@ func main() {
 		"Starting private validator",
 		"addr", *addr,
 		"chainID", *chainID,
-		"privPath", *privValPath,
+		"privKeyPath", *privValKeyPath,
+		"privStatePath", *privValStatePath,
 	)
 
-	pv := privval.LoadFilePV(*privValPath)
+	pv := privval.LoadFilePV(*privValKeyPath, *privValStatePath)
 
-	rs := privval.NewRemoteSigner(
-		logger,
-		*chainID,
-		*addr,
-		pv,
-		ed25519.GenPrivKey(),
-	)
+	var dialer privval.SocketDialer
+	protocol, address := cmn.ProtocolAndAddress(*addr)
+	switch protocol {
+	case "unix":
+		dialer = privval.DialUnixFn(address)
+	case "tcp":
+		connTimeout := 3 * time.Second // TODO
+		dialer = privval.DialTCPFn(address, connTimeout, ed25519.GenPrivKey())
+	default:
+		logger.Error("Unknown protocol", "protocol", protocol)
+		os.Exit(1)
+	}
+
+	rs := privval.NewSignerServiceEndpoint(logger, *chainID, pv, dialer)
 	err := rs.Start()
 	if err != nil {
 		panic(err)
 	}
 
-	cmn.TrapSignal(func() {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
 		err := rs.Stop()
 		if err != nil {
 			panic(err)
 		}
 	})
+
+	// Run forever.
+	select {}
 }

cmd/tendermint/commands/gen_validator.go

@@ -17,7 +17,7 @@ var GenValidatorCmd = &cobra.Command{
 }
 
 func genValidator(cmd *cobra.Command, args []string) {
-	pv := privval.GenFilePV("")
+	pv := privval.GenFilePV("", "")
 	jsbz, err := cdc.MarshalJSON(pv)
 	if err != nil {
 		panic(err)

cmd/tendermint/commands/init.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
-
 	cfg "github.com/tendermint/tendermint/config"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/p2p"
@@ -26,15 +25,18 @@ func initFiles(cmd *cobra.Command, args []string) error {
 
 func initFilesWithConfig(config *cfg.Config) error {
 	// private validator
-	privValFile := config.PrivValidatorFile()
+	privValKeyFile := config.PrivValidatorKeyFile()
+	privValStateFile := config.PrivValidatorStateFile()
 	var pv *privval.FilePV
-	if cmn.FileExists(privValFile) {
-		pv = privval.LoadFilePV(privValFile)
-		logger.Info("Found private validator", "path", privValFile)
+	if cmn.FileExists(privValKeyFile) {
+		pv = privval.LoadFilePV(privValKeyFile, privValStateFile)
+		logger.Info("Found private validator", "keyFile", privValKeyFile,
+			"stateFile", privValStateFile)
 	} else {
-		pv = privval.GenFilePV(privValFile)
+		pv = privval.GenFilePV(privValKeyFile, privValStateFile)
 		pv.Save()
-		logger.Info("Generated private validator", "path", privValFile)
+		logger.Info("Generated private validator", "keyFile", privValKeyFile,
+			"stateFile", privValStateFile)
 	}
 
 	nodeKeyFile := config.NodeKeyFile()
@@ -57,9 +59,10 @@ func initFilesWithConfig(config *cfg.Config) error {
 			GenesisTime:     tmtime.Now(),
 			ConsensusParams: types.DefaultConsensusParams(),
 		}
+		key := pv.GetPubKey()
 		genDoc.Validators = []types.GenesisValidator{{
-			Address: pv.GetPubKey().Address(),
-			PubKey:  pv.GetPubKey(),
+			Address: key.Address(),
+			PubKey:  key,
 			Power:   10,
 		}}
 

cmd/tendermint/commands/lite.go

@@ -43,7 +43,7 @@ func init() {
 	LiteCmd.Flags().IntVar(&cacheSize, "cache-size", 10, "Specify the memory trust store cache size")
 }
 
-func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) {
+func EnsureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) {
 	u, err := url.Parse(addr)
 	if err != nil {
 		return "", err
@@ -59,11 +59,16 @@ func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) {
 }
 
 func runProxy(cmd *cobra.Command, args []string) error {
-	nodeAddr, err := ensureAddrHasSchemeOrDefaultToTCP(nodeAddr)
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
+		// TODO: close up shop
+	})
+
+	nodeAddr, err := EnsureAddrHasSchemeOrDefaultToTCP(nodeAddr)
 	if err != nil {
 		return err
 	}
-	listenAddr, err := ensureAddrHasSchemeOrDefaultToTCP(listenAddr)
+	listenAddr, err := EnsureAddrHasSchemeOrDefaultToTCP(listenAddr)
 	if err != nil {
 		return err
 	}
@@ -86,9 +91,6 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		return cmn.ErrorWrap(err, "starting proxy")
 	}
 
-	cmn.TrapSignal(func() {
-		// TODO: close up shop
-	})
-
-	return nil
+	// Run forever
+	select {}
 }

cmd/tendermint/commands/reset_priv_validator.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
 	"github.com/tendermint/tendermint/privval"
 )
@@ -17,6 +18,12 @@ var ResetAllCmd = &cobra.Command{
 	Run:   resetAll,
 }
 
+var keepAddrBook bool
+
+func init() {
+	ResetAllCmd.Flags().BoolVar(&keepAddrBook, "keep-addr-book", false, "Keep the address book intact")
+}
+
 // ResetPrivValidatorCmd resets the private validator files.
 var ResetPrivValidatorCmd = &cobra.Command{
 	Use:   "unsafe_reset_priv_validator",
@@ -27,36 +34,45 @@ var ResetPrivValidatorCmd = &cobra.Command{
 // XXX: this is totally unsafe.
 // it's only suitable for testnets.
 func resetAll(cmd *cobra.Command, args []string) {
-	ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorFile(), logger)
+	ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorKeyFile(),
+		config.PrivValidatorStateFile(), logger)
 }
 
 // XXX: this is totally unsafe.
 // it's only suitable for testnets.
 func resetPrivValidator(cmd *cobra.Command, args []string) {
-	resetFilePV(config.PrivValidatorFile(), logger)
+	resetFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile(), logger)
 }
 
-// ResetAll removes the privValidator and address book files plus all data.
+// ResetAll removes address book files plus all data, and resets the privValdiator data.
 // Exported so other CLI tools can use it.
-func ResetAll(dbDir, addrBookFile, privValFile string, logger log.Logger) {
-	resetFilePV(privValFile, logger)
-	removeAddrBook(addrBookFile, logger)
+func ResetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile string, logger log.Logger) {
+	if keepAddrBook {
+		logger.Info("The address book remains intact")
+	} else {
+		removeAddrBook(addrBookFile, logger)
+	}
 	if err := os.RemoveAll(dbDir); err == nil {
 		logger.Info("Removed all blockchain history", "dir", dbDir)
 	} else {
 		logger.Error("Error removing all blockchain history", "dir", dbDir, "err", err)
 	}
+	// recreate the dbDir since the privVal state needs to live there
+	cmn.EnsureDir(dbDir, 0700)
+	resetFilePV(privValKeyFile, privValStateFile, logger)
 }
 
-func resetFilePV(privValFile string, logger log.Logger) {
-	if _, err := os.Stat(privValFile); err == nil {
-		pv := privval.LoadFilePV(privValFile)
+func resetFilePV(privValKeyFile, privValStateFile string, logger log.Logger) {
+	if _, err := os.Stat(privValKeyFile); err == nil {
+		pv := privval.LoadFilePVEmptyState(privValKeyFile, privValStateFile)
 		pv.Reset()
-		logger.Info("Reset private validator file to genesis state", "file", privValFile)
+		logger.Info("Reset private validator file to genesis state", "keyFile", privValKeyFile,
+			"stateFile", privValStateFile)
 	} else {
-		pv := privval.GenFilePV(privValFile)
+		pv := privval.GenFilePV(privValKeyFile, privValStateFile)
 		pv.Save()
-		logger.Info("Generated private validator file", "file", privValFile)
+		logger.Info("Generated private validator file", "keyFile", privValKeyFile,
+			"stateFile", privValStateFile)
 	}
 }
 

cmd/tendermint/commands/root.go

@@ -54,6 +54,9 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		if config.LogFormat == cfg.LogFormatJSON {
+			logger = log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout))
+		}
 		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel())
 		if err != nil {
 			return err

cmd/tendermint/commands/root_test.go

@@ -22,10 +22,6 @@ var (
 	defaultRoot = os.ExpandEnv("$HOME/.some/test/dir")
 )
 
-const (
-	rootName = "root"
-)
-
 // clearConfig clears env vars, the given root dir, and resets viper.
 func clearConfig(dir string) {
 	if err := os.Unsetenv("TMHOME"); err != nil {

cmd/tendermint/commands/run_node.go

@@ -2,12 +2,10 @@ package commands
 
 import (
 	"fmt"
-	"os"
-	"os/signal"
-	"syscall"
 
 	"github.com/spf13/cobra"
 
+	cmn "github.com/tendermint/tendermint/libs/common"
 	nm "github.com/tendermint/tendermint/node"
 )
 
@@ -24,7 +22,7 @@ func AddNodeFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing")
 
 	// abci flags
-	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.")
+	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or one of: 'kvstore', 'persistent_kvstore', 'counter', 'counter_serial' or 'noop' for local testing.")
 	cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)")
 
 	// rpc flags
@@ -57,28 +55,20 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command {
 				return fmt.Errorf("Failed to create node: %v", err)
 			}
 
-			// Stop upon receiving SIGTERM or CTRL-C
-			c := make(chan os.Signal, 1)
-			signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-			go func() {
-				for sig := range c {
-					logger.Error(fmt.Sprintf("captured %v, exiting...", sig))
-					if n.IsRunning() {
-						n.Stop()
-					}
-					os.Exit(1)
+			// Stop upon receiving SIGTERM or CTRL-C.
+			cmn.TrapSignal(logger, func() {
+				if n.IsRunning() {
+					n.Stop()
 				}
-			}()
+			})
 
 			if err := n.Start(); err != nil {
 				return fmt.Errorf("Failed to start node: %v", err)
 			}
 			logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
 
-			// Run forever
+			// Run forever.
 			select {}
-
-			return nil
 		},
 	}
 

cmd/tendermint/commands/show_node_id.go

@@ -16,12 +16,11 @@ var ShowNodeIDCmd = &cobra.Command{
 }
 
 func showNodeID(cmd *cobra.Command, args []string) error {
-
 	nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile())
 	if err != nil {
 		return err
 	}
-	fmt.Println(nodeKey.ID())
 
+	fmt.Println(nodeKey.ID())
 	return nil
 }

cmd/tendermint/commands/show_validator.go

@@ -3,8 +3,10 @@ package commands
 import (
 	"fmt"
 
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
+	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/privval"
 )
 
@@ -12,11 +14,21 @@ import (
 var ShowValidatorCmd = &cobra.Command{
 	Use:   "show_validator",
 	Short: "Show this node's validator info",
-	Run:   showValidator,
+	RunE:  showValidator,
 }
 
-func showValidator(cmd *cobra.Command, args []string) {
-	privValidator := privval.LoadOrGenFilePV(config.PrivValidatorFile())
-	pubKeyJSONBytes, _ := cdc.MarshalJSON(privValidator.GetPubKey())
-	fmt.Println(string(pubKeyJSONBytes))
+func showValidator(cmd *cobra.Command, args []string) error {
+	keyFilePath := config.PrivValidatorKeyFile()
+	if !cmn.FileExists(keyFilePath) {
+		return fmt.Errorf("private validator file %s does not exist", keyFilePath)
+	}
+
+	pv := privval.LoadFilePV(keyFilePath, config.PrivValidatorStateFile())
+	bz, err := cdc.MarshalJSON(pv.GetPubKey())
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal private validator pubkey")
+	}
+
+	fmt.Println(string(bz))
+	return nil
 }

cmd/tendermint/commands/testnet.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	cfg "github.com/tendermint/tendermint/config"
 	cmn "github.com/tendermint/tendermint/libs/common"
@@ -20,13 +21,17 @@ import (
 var (
 	nValidators    int
 	nNonValidators int
+	configFile     string
 	outputDir      string
 	nodeDirPrefix  string
 
 	populatePersistentPeers bool
 	hostnamePrefix          string
+	hostnameSuffix          string
 	startingIPAddress       string
+	hostnames               []string
 	p2pPort                 int
+	randomMonikers          bool
 )
 
 const (
@@ -36,6 +41,8 @@ const (
 func init() {
 	TestnetFilesCmd.Flags().IntVar(&nValidators, "v", 4,
 		"Number of validators to initialize the testnet with")
+	TestnetFilesCmd.Flags().StringVar(&configFile, "config", "",
+		"Config file to use (note some options may be overwritten)")
 	TestnetFilesCmd.Flags().IntVar(&nNonValidators, "n", 0,
 		"Number of non-validators to initialize the testnet with")
 	TestnetFilesCmd.Flags().StringVar(&outputDir, "o", "./mytestnet",
@@ -46,11 +53,17 @@ func init() {
 	TestnetFilesCmd.Flags().BoolVar(&populatePersistentPeers, "populate-persistent-peers", true,
 		"Update config of each node with the list of persistent peers build using either hostname-prefix or starting-ip-address")
 	TestnetFilesCmd.Flags().StringVar(&hostnamePrefix, "hostname-prefix", "node",
-		"Hostname prefix (node results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
+		"Hostname prefix (\"node\" results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
+	TestnetFilesCmd.Flags().StringVar(&hostnameSuffix, "hostname-suffix", "",
+		"Hostname suffix (\".xyz.com\" results in persistent peers list ID0@node0.xyz.com:26656, ID1@node1.xyz.com:26656, ...)")
 	TestnetFilesCmd.Flags().StringVar(&startingIPAddress, "starting-ip-address", "",
-		"Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:26656, ID1@192.168.0.2:26656, ...)")
+		"Starting IP address (\"192.168.0.1\" results in persistent peers list ID0@192.168.0.1:26656, ID1@192.168.0.2:26656, ...)")
+	TestnetFilesCmd.Flags().StringArrayVar(&hostnames, "hostname", []string{},
+		"Manually override all hostnames of validators and non-validators (use --hostname multiple times for multiple hosts)")
 	TestnetFilesCmd.Flags().IntVar(&p2pPort, "p2p-port", 26656,
 		"P2P Port")
+	TestnetFilesCmd.Flags().BoolVar(&randomMonikers, "random-monikers", false,
+		"Randomize the moniker for each generated node")
 }
 
 // TestnetFilesCmd allows initialisation of files for a Tendermint testnet.
@@ -72,7 +85,29 @@ Example:
 }
 
 func testnetFiles(cmd *cobra.Command, args []string) error {
+	if len(hostnames) > 0 && len(hostnames) != (nValidators+nNonValidators) {
+		return fmt.Errorf(
+			"testnet needs precisely %d hostnames (number of validators plus non-validators) if --hostname parameter is used",
+			nValidators+nNonValidators,
+		)
+	}
+
 	config := cfg.DefaultConfig()
+
+	// overwrite default config if set and valid
+	if configFile != "" {
+		viper.SetConfigFile(configFile)
+		if err := viper.ReadInConfig(); err != nil {
+			return err
+		}
+		if err := viper.Unmarshal(config); err != nil {
+			return err
+		}
+		if err := config.ValidateBasic(); err != nil {
+			return err
+		}
+	}
+
 	genVals := make([]types.GenesisValidator, nValidators)
 
 	for i := 0; i < nValidators; i++ {
@@ -85,11 +120,18 @@ func testnetFiles(cmd *cobra.Command, args []string) error {
 			_ = os.RemoveAll(outputDir)
 			return err
 		}
+		err = os.MkdirAll(filepath.Join(nodeDir, "data"), nodeDirPerm)
+		if err != nil {
+			_ = os.RemoveAll(outputDir)
+			return err
+		}
 
 		initFilesWithConfig(config)
 
-		pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator)
-		pv := privval.LoadFilePV(pvFile)
+		pvKeyFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidatorKey)
+		pvStateFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidatorState)
+
+		pv := privval.LoadFilePV(pvKeyFile, pvStateFile)
 		genVals[i] = types.GenesisValidator{
 			Address: pv.GetPubKey().Address(),
 			PubKey:  pv.GetPubKey(),
@@ -108,6 +150,12 @@ func testnetFiles(cmd *cobra.Command, args []string) error {
 			return err
 		}
 
+		err = os.MkdirAll(filepath.Join(nodeDir, "data"), nodeDirPerm)
+		if err != nil {
+			_ = os.RemoveAll(outputDir)
+			return err
+		}
+
 		initFilesWithConfig(config)
 	}
 
@@ -127,58 +175,84 @@ func testnetFiles(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	// Gather persistent peer addresses.
+	var (
+		persistentPeers string
+		err             error
+	)
 	if populatePersistentPeers {
-		err := populatePersistentPeersInConfigAndWriteIt(config)
+		persistentPeers, err = persistentPeersString(config)
 		if err != nil {
 			_ = os.RemoveAll(outputDir)
 			return err
 		}
 	}
 
+	// Overwrite default config.
+	for i := 0; i < nValidators+nNonValidators; i++ {
+		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
+		config.SetRoot(nodeDir)
+		config.P2P.AddrBookStrict = false
+		config.P2P.AllowDuplicateIP = true
+		if populatePersistentPeers {
+			config.P2P.PersistentPeers = persistentPeers
+		}
+		config.Moniker = moniker(i)
+
+		cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config)
+	}
+
 	fmt.Printf("Successfully initialized %v node directories\n", nValidators+nNonValidators)
 	return nil
 }
 
 func hostnameOrIP(i int) string {
-	if startingIPAddress != "" {
-		ip := net.ParseIP(startingIPAddress)
-		ip = ip.To4()
-		if ip == nil {
-			fmt.Printf("%v: non ipv4 address\n", startingIPAddress)
-			os.Exit(1)
-		}
-
-		for j := 0; j < i; j++ {
-			ip[3]++
-		}
-		return ip.String()
+	if len(hostnames) > 0 && i < len(hostnames) {
+		return hostnames[i]
+	}
+	if startingIPAddress == "" {
+		return fmt.Sprintf("%s%d%s", hostnamePrefix, i, hostnameSuffix)
+	}
+	ip := net.ParseIP(startingIPAddress)
+	ip = ip.To4()
+	if ip == nil {
+		fmt.Printf("%v: non ipv4 address\n", startingIPAddress)
+		os.Exit(1)
 	}
 
-	return fmt.Sprintf("%s%d", hostnamePrefix, i)
+	for j := 0; j < i; j++ {
+		ip[3]++
+	}
+	return ip.String()
 }
 
-func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error {
+func persistentPeersString(config *cfg.Config) (string, error) {
 	persistentPeers := make([]string, nValidators+nNonValidators)
 	for i := 0; i < nValidators+nNonValidators; i++ {
 		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
 		config.SetRoot(nodeDir)
 		nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile())
 		if err != nil {
-			return err
+			return "", err
 		}
 		persistentPeers[i] = p2p.IDAddressString(nodeKey.ID(), fmt.Sprintf("%s:%d", hostnameOrIP(i), p2pPort))
 	}
-	persistentPeersList := strings.Join(persistentPeers, ",")
-
-	for i := 0; i < nValidators+nNonValidators; i++ {
-		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
-		config.SetRoot(nodeDir)
-		config.P2P.PersistentPeers = persistentPeersList
-		config.P2P.AddrBookStrict = false
-
-		// overwrite default config
-		cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config)
-	}
-
-	return nil
+	return strings.Join(persistentPeers, ","), nil
+}
+
+func moniker(i int) string {
+	if randomMonikers {
+		return randomMoniker()
+	}
+	if len(hostnames) > 0 && i < len(hostnames) {
+		return hostnames[i]
+	}
+	if startingIPAddress == "" {
+		return fmt.Sprintf("%s%d%s", hostnamePrefix, i, hostnameSuffix)
+	}
+	return randomMoniker()
+}
+
+func randomMoniker() string {
+	return cmn.HexBytes(cmn.RandBytes(8)).String()
 }

cmd/tendermint/commands/wire.go

@@ -1,7 +1,7 @@
 package commands
 
 import (
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 

config/config.go

@@ -14,6 +14,11 @@ const (
 	FuzzModeDrop = iota
 	// FuzzModeDelay is a mode in which we randomly sleep
 	FuzzModeDelay
+
+	// LogFormatPlain is a format for colored text
+	LogFormatPlain = "plain"
+	// LogFormatJSON is a format for json output
+	LogFormatJSON = "json"
 )
 
 // NOTE: Most of the structs & relevant comments + the
@@ -30,15 +35,24 @@ var (
 	defaultConfigFileName  = "config.toml"
 	defaultGenesisJSONName = "genesis.json"
 
-	defaultPrivValName  = "priv_validator.json"
+	defaultPrivValKeyName   = "priv_validator_key.json"
+	defaultPrivValStateName = "priv_validator_state.json"
+
 	defaultNodeKeyName  = "node_key.json"
 	defaultAddrBookName = "addrbook.json"
 
-	defaultConfigFilePath  = filepath.Join(defaultConfigDir, defaultConfigFileName)
-	defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName)
-	defaultPrivValPath     = filepath.Join(defaultConfigDir, defaultPrivValName)
-	defaultNodeKeyPath     = filepath.Join(defaultConfigDir, defaultNodeKeyName)
-	defaultAddrBookPath    = filepath.Join(defaultConfigDir, defaultAddrBookName)
+	defaultConfigFilePath   = filepath.Join(defaultConfigDir, defaultConfigFileName)
+	defaultGenesisJSONPath  = filepath.Join(defaultConfigDir, defaultGenesisJSONName)
+	defaultPrivValKeyPath   = filepath.Join(defaultConfigDir, defaultPrivValKeyName)
+	defaultPrivValStatePath = filepath.Join(defaultDataDir, defaultPrivValStateName)
+
+	defaultNodeKeyPath  = filepath.Join(defaultConfigDir, defaultNodeKeyName)
+	defaultAddrBookPath = filepath.Join(defaultConfigDir, defaultAddrBookName)
+)
+
+var (
+	oldPrivVal     = "priv_validator.json"
+	oldPrivValPath = filepath.Join(defaultConfigDir, oldPrivVal)
 )
 
 // Config defines the top level configuration for a Tendermint node
@@ -94,6 +108,9 @@ func (cfg *Config) SetRoot(root string) *Config {
 // ValidateBasic performs basic validation (checking param bounds, etc.) and
 // returns an error if any check fails.
 func (cfg *Config) ValidateBasic() error {
+	if err := cfg.BaseConfig.ValidateBasic(); err != nil {
+		return err
+	}
 	if err := cfg.RPC.ValidateBasic(); err != nil {
 		return errors.Wrap(err, "Error in [rpc] section")
 	}
@@ -136,7 +153,18 @@ type BaseConfig struct {
 	// and verifying their commits
 	FastSync bool `mapstructure:"fast_sync"`
 
-	// Database backend: leveldb | memdb | cleveldb
+	// Database backend: goleveldb | cleveldb | boltdb
+	// * goleveldb (github.com/syndtr/goleveldb - most popular implementation)
+	//   - pure go
+	//   - stable
+	// * cleveldb (uses levigo wrapper)
+	//   - fast
+	//   - requires gcc
+	//   - use cleveldb build tag (go build -tags cleveldb)
+	// * boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt)
+	//   - EXPERIMENTAL
+	//   - may be faster is some use-cases (random reads - indexer)
+	//   - use boltdb build tag (go build -tags boltdb)
 	DBBackend string `mapstructure:"db_backend"`
 
 	// Database directory
@@ -145,11 +173,17 @@ type BaseConfig struct {
 	// Output level for logging
 	LogLevel string `mapstructure:"log_level"`
 
+	// Output format: 'plain' (colored text) or 'json'
+	LogFormat string `mapstructure:"log_format"`
+
 	// Path to the JSON file containing the initial validator set and other meta data
 	Genesis string `mapstructure:"genesis_file"`
 
 	// Path to the JSON file containing the private key to use as a validator in the consensus protocol
-	PrivValidator string `mapstructure:"priv_validator_file"`
+	PrivValidatorKey string `mapstructure:"priv_validator_key_file"`
+
+	// Path to the JSON file containing the last sign state of a validator
+	PrivValidatorState string `mapstructure:"priv_validator_state_file"`
 
 	// TCP or UNIX socket address for Tendermint to listen on for
 	// connections from an external PrivValidator process
@@ -172,18 +206,20 @@ type BaseConfig struct {
 // DefaultBaseConfig returns a default base configuration for a Tendermint node
 func DefaultBaseConfig() BaseConfig {
 	return BaseConfig{
-		Genesis:           defaultGenesisJSONPath,
-		PrivValidator:     defaultPrivValPath,
-		NodeKey:           defaultNodeKeyPath,
-		Moniker:           defaultMoniker,
-		ProxyApp:          "tcp://127.0.0.1:26658",
-		ABCI:              "socket",
-		LogLevel:          DefaultPackageLogLevels(),
-		ProfListenAddress: "",
-		FastSync:          true,
-		FilterPeers:       false,
-		DBBackend:         "leveldb",
-		DBPath:            "data",
+		Genesis:            defaultGenesisJSONPath,
+		PrivValidatorKey:   defaultPrivValKeyPath,
+		PrivValidatorState: defaultPrivValStatePath,
+		NodeKey:            defaultNodeKeyPath,
+		Moniker:            defaultMoniker,
+		ProxyApp:           "tcp://127.0.0.1:26658",
+		ABCI:               "socket",
+		LogLevel:           DefaultPackageLogLevels(),
+		LogFormat:          LogFormatPlain,
+		ProfListenAddress:  "",
+		FastSync:           true,
+		FilterPeers:        false,
+		DBBackend:          "goleveldb",
+		DBPath:             "data",
 	}
 }
 
@@ -206,9 +242,20 @@ func (cfg BaseConfig) GenesisFile() string {
 	return rootify(cfg.Genesis, cfg.RootDir)
 }
 
-// PrivValidatorFile returns the full path to the priv_validator.json file
-func (cfg BaseConfig) PrivValidatorFile() string {
-	return rootify(cfg.PrivValidator, cfg.RootDir)
+// PrivValidatorKeyFile returns the full path to the priv_validator_key.json file
+func (cfg BaseConfig) PrivValidatorKeyFile() string {
+	return rootify(cfg.PrivValidatorKey, cfg.RootDir)
+}
+
+// PrivValidatorFile returns the full path to the priv_validator_state.json file
+func (cfg BaseConfig) PrivValidatorStateFile() string {
+	return rootify(cfg.PrivValidatorState, cfg.RootDir)
+}
+
+// OldPrivValidatorFile returns the full path of the priv_validator.json from pre v0.28.0.
+// TODO: eventually remove.
+func (cfg BaseConfig) OldPrivValidatorFile() string {
+	return rootify(oldPrivValPath, cfg.RootDir)
 }
 
 // NodeKeyFile returns the full path to the node_key.json file
@@ -221,6 +268,17 @@ func (cfg BaseConfig) DBDir() string {
 	return rootify(cfg.DBPath, cfg.RootDir)
 }
 
+// ValidateBasic performs basic validation (checking param bounds, etc.) and
+// returns an error if any check fails.
+func (cfg BaseConfig) ValidateBasic() error {
+	switch cfg.LogFormat {
+	case LogFormatPlain, LogFormatJSON:
+	default:
+		return errors.New("unknown log_format (must be 'plain' or 'json')")
+	}
+	return nil
+}
+
 // DefaultLogLevel returns a default log level of "error"
 func DefaultLogLevel() string {
 	return "error"
@@ -242,13 +300,25 @@ type RPCConfig struct {
 	// TCP or UNIX socket address for the RPC server to listen on
 	ListenAddress string `mapstructure:"laddr"`
 
+	// A list of origins a cross-domain request can be executed from.
+	// If the special '*' value is present in the list, all origins will be allowed.
+	// An origin may contain a wildcard (*) to replace 0 or more characters (i.e.: http://*.domain.com).
+	// Only one wildcard can be used per origin.
+	CORSAllowedOrigins []string `mapstructure:"cors_allowed_origins"`
+
+	// A list of methods the client is allowed to use with cross-domain requests.
+	CORSAllowedMethods []string `mapstructure:"cors_allowed_methods"`
+
+	// A list of non simple headers the client is allowed to use with cross-domain requests.
+	CORSAllowedHeaders []string `mapstructure:"cors_allowed_headers"`
+
 	// TCP or UNIX socket address for the gRPC server to listen on
 	// NOTE: This server only supports /broadcast_tx_commit
 	GRPCListenAddress string `mapstructure:"grpc_laddr"`
 
 	// Maximum number of simultaneous connections.
 	// Does not include RPC (HTTP&WebSocket) connections. See max_open_connections
-	// If you want to accept more significant number than the default, make sure
+	// If you want to accept a larger number than the default, make sure
 	// you increase your OS limits.
 	// 0 - unlimited.
 	GRPCMaxOpenConnections int `mapstructure:"grpc_max_open_connections"`
@@ -258,24 +328,63 @@ type RPCConfig struct {
 
 	// Maximum number of simultaneous connections (including WebSocket).
 	// Does not include gRPC connections. See grpc_max_open_connections
-	// If you want to accept more significant number than the default, make sure
+	// If you want to accept a larger number than the default, make sure
 	// you increase your OS limits.
 	// 0 - unlimited.
 	// Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
 	// 1024 - 40 - 10 - 50 = 924 = ~900
 	MaxOpenConnections int `mapstructure:"max_open_connections"`
+
+	// Maximum number of unique clientIDs that can /subscribe
+	// If you're using /broadcast_tx_commit, set to the estimated maximum number
+	// of broadcast_tx_commit calls per block.
+	MaxSubscriptionClients int `mapstructure:"max_subscription_clients"`
+
+	// Maximum number of unique queries a given client can /subscribe to
+	// If you're using GRPC (or Local RPC client) and /broadcast_tx_commit, set
+	// to the estimated maximum number of broadcast_tx_commit calls per block.
+	MaxSubscriptionsPerClient int `mapstructure:"max_subscriptions_per_client"`
+
+	// How long to wait for a tx to be committed during /broadcast_tx_commit
+	// WARNING: Using a value larger than 10s will result in increasing the
+	// global HTTP write timeout, which applies to all connections and endpoints.
+	// See https://github.com/tendermint/tendermint/issues/3435
+	TimeoutBroadcastTxCommit time.Duration `mapstructure:"timeout_broadcast_tx_commit"`
+
+	// The name of a file containing certificate that is used to create the HTTPS server.
+	//
+	// If the certificate is signed by a certificate authority,
+	// the certFile should be the concatenation of the server's certificate, any intermediates,
+	// and the CA's certificate.
+	//
+	// NOTE: both tls_cert_file and tls_key_file must be present for Tendermint to create HTTPS server. Otherwise, HTTP server is run.
+	TLSCertFile string `mapstructure:"tls_cert_file"`
+
+	// The name of a file containing matching private key that is used to create the HTTPS server.
+	//
+	// NOTE: both tls_cert_file and tls_key_file must be present for Tendermint to create HTTPS server. Otherwise, HTTP server is run.
+	TLSKeyFile string `mapstructure:"tls_key_file"`
 }
 
 // DefaultRPCConfig returns a default configuration for the RPC server
 func DefaultRPCConfig() *RPCConfig {
 	return &RPCConfig{
-		ListenAddress: "tcp://0.0.0.0:26657",
-
+		ListenAddress:          "tcp://0.0.0.0:26657",
+		CORSAllowedOrigins:     []string{},
+		CORSAllowedMethods:     []string{"HEAD", "GET", "POST"},
+		CORSAllowedHeaders:     []string{"Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time"},
 		GRPCListenAddress:      "",
 		GRPCMaxOpenConnections: 900,
 
 		Unsafe:             false,
 		MaxOpenConnections: 900,
+
+		MaxSubscriptionClients:    100,
+		MaxSubscriptionsPerClient: 5,
+		TimeoutBroadcastTxCommit:  10 * time.Second,
+
+		TLSCertFile: "",
+		TLSKeyFile:  "",
 	}
 }
 
@@ -297,9 +406,35 @@ func (cfg *RPCConfig) ValidateBasic() error {
 	if cfg.MaxOpenConnections < 0 {
 		return errors.New("max_open_connections can't be negative")
 	}
+	if cfg.MaxSubscriptionClients < 0 {
+		return errors.New("max_subscription_clients can't be negative")
+	}
+	if cfg.MaxSubscriptionsPerClient < 0 {
+		return errors.New("max_subscriptions_per_client can't be negative")
+	}
+	if cfg.TimeoutBroadcastTxCommit < 0 {
+		return errors.New("timeout_broadcast_tx_commit can't be negative")
+	}
 	return nil
 }
 
+// IsCorsEnabled returns true if cross-origin resource sharing is enabled.
+func (cfg *RPCConfig) IsCorsEnabled() bool {
+	return len(cfg.CORSAllowedOrigins) != 0
+}
+
+func (cfg RPCConfig) KeyFile() string {
+	return rootify(filepath.Join(defaultConfigDir, cfg.TLSKeyFile), cfg.RootDir)
+}
+
+func (cfg RPCConfig) CertFile() string {
+	return rootify(filepath.Join(defaultConfigDir, cfg.TLSCertFile), cfg.RootDir)
+}
+
+func (cfg RPCConfig) IsTLSEnabled() bool {
+	return cfg.TLSCertFile != "" && cfg.TLSKeyFile != ""
+}
+
 //-----------------------------------------------------------------------------
 // P2PConfig
 
@@ -392,7 +527,7 @@ func DefaultP2PConfig() *P2PConfig {
 		RecvRate:                5120000, // 5 mB/s
 		PexReactor:              true,
 		SeedMode:                false,
-		AllowDuplicateIP:        true, // so non-breaking yet
+		AllowDuplicateIP:        false,
 		HandshakeTimeout:        20 * time.Second,
 		DialTimeout:             3 * time.Second,
 		TestDialFail:            false,
@@ -464,12 +599,13 @@ func DefaultFuzzConnConfig() *FuzzConnConfig {
 
 // MempoolConfig defines the configuration options for the Tendermint mempool
 type MempoolConfig struct {
-	RootDir   string `mapstructure:"home"`
-	Recheck   bool   `mapstructure:"recheck"`
-	Broadcast bool   `mapstructure:"broadcast"`
-	WalPath   string `mapstructure:"wal_dir"`
-	Size      int    `mapstructure:"size"`
-	CacheSize int    `mapstructure:"cache_size"`
+	RootDir     string `mapstructure:"home"`
+	Recheck     bool   `mapstructure:"recheck"`
+	Broadcast   bool   `mapstructure:"broadcast"`
+	WalPath     string `mapstructure:"wal_dir"`
+	Size        int    `mapstructure:"size"`
+	MaxTxsBytes int64  `mapstructure:"max_txs_bytes"`
+	CacheSize   int    `mapstructure:"cache_size"`
 }
 
 // DefaultMempoolConfig returns a default configuration for the Tendermint mempool
@@ -478,10 +614,11 @@ func DefaultMempoolConfig() *MempoolConfig {
 		Recheck:   true,
 		Broadcast: true,
 		WalPath:   "",
-		// Each signature verification takes .5ms, size reduced until we implement
+		// Each signature verification takes .5ms, Size reduced until we implement
 		// ABCI Recheck
-		Size:      5000,
-		CacheSize: 10000,
+		Size:        5000,
+		MaxTxsBytes: 1024 * 1024 * 1024, // 1GB
+		CacheSize:   10000,
 	}
 }
 
@@ -497,12 +634,20 @@ func (cfg *MempoolConfig) WalDir() string {
 	return rootify(cfg.WalPath, cfg.RootDir)
 }
 
+// WalEnabled returns true if the WAL is enabled.
+func (cfg *MempoolConfig) WalEnabled() bool {
+	return cfg.WalPath != ""
+}
+
 // ValidateBasic performs basic validation (checking param bounds, etc.) and
 // returns an error if any check fails.
 func (cfg *MempoolConfig) ValidateBasic() error {
 	if cfg.Size < 0 {
 		return errors.New("size can't be negative")
 	}
+	if cfg.MaxTxsBytes < 0 {
+		return errors.New("max_txs_bytes can't be negative")
+	}
 	if cfg.CacheSize < 0 {
 		return errors.New("cache_size can't be negative")
 	}
@@ -537,9 +682,6 @@ type ConsensusConfig struct {
 	// Reactor sleep duration parameters
 	PeerGossipSleepDuration     time.Duration `mapstructure:"peer_gossip_sleep_duration"`
 	PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer_query_maj23_sleep_duration"`
-
-	// Block time parameters. Corresponds to the minimum time increment between consecutive blocks.
-	BlockTimeIota time.Duration `mapstructure:"blocktime_iota"`
 }
 
 // DefaultConsensusConfig returns a default configuration for the consensus service
@@ -558,7 +700,6 @@ func DefaultConsensusConfig() *ConsensusConfig {
 		CreateEmptyBlocksInterval:   0 * time.Second,
 		PeerGossipSleepDuration:     100 * time.Millisecond,
 		PeerQueryMaj23SleepDuration: 2000 * time.Millisecond,
-		BlockTimeIota:               1000 * time.Millisecond,
 	}
 }
 
@@ -575,16 +716,9 @@ func TestConsensusConfig() *ConsensusConfig {
 	cfg.SkipTimeoutCommit = true
 	cfg.PeerGossipSleepDuration = 5 * time.Millisecond
 	cfg.PeerQueryMaj23SleepDuration = 250 * time.Millisecond
-	cfg.BlockTimeIota = 10 * time.Millisecond
 	return cfg
 }
 
-// MinValidVoteTime returns the minimum acceptable block time.
-// See the [BFT time spec](https://godoc.org/github.com/tendermint/tendermint/docs/spec/consensus/bft-time.md).
-func (cfg *ConsensusConfig) MinValidVoteTime(lastBlockTime time.Time) time.Time {
-	return lastBlockTime.Add(cfg.BlockTimeIota)
-}
-
 // WaitForTxs returns true if the consensus should wait for transactions before entering the propose step
 func (cfg *ConsensusConfig) WaitForTxs() bool {
 	return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0
@@ -662,9 +796,6 @@ func (cfg *ConsensusConfig) ValidateBasic() error {
 	if cfg.PeerQueryMaj23SleepDuration < 0 {
 		return errors.New("peer_query_maj23_sleep_duration can't be negative")
 	}
-	if cfg.BlockTimeIota < 0 {
-		return errors.New("blocktime_iota can't be negative")
-	}
 	return nil
 }
 
@@ -727,12 +858,12 @@ type InstrumentationConfig struct {
 	PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
 
 	// Maximum number of simultaneous connections.
-	// If you want to accept more significant number than the default, make sure
+	// If you want to accept a larger number than the default, make sure
 	// you increase your OS limits.
 	// 0 - unlimited.
 	MaxOpenConnections int `mapstructure:"max_open_connections"`
 
-	// Tendermint instrumentation namespace.
+	// Instrumentation namespace.
 	Namespace string `mapstructure:"namespace"`
 }
 

config/toml.go

@@ -2,13 +2,17 @@ package config
 
 import (
 	"bytes"
-	"os"
+	"fmt"
+	"io/ioutil"
 	"path/filepath"
 	"text/template"
 
 	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
+// DefaultDirPerm is the default permissions used when creating directories.
+const DefaultDirPerm = 0700
+
 var configTemplate *template.Template
 
 func init() {
@@ -23,14 +27,14 @@ func init() {
 // EnsureRoot creates the root, config, and data directories if they don't exist,
 // and panics if it fails.
 func EnsureRoot(rootDir string) {
-	if err := cmn.EnsureDir(rootDir, 0700); err != nil {
-		cmn.PanicSanity(err.Error())
+	if err := cmn.EnsureDir(rootDir, DefaultDirPerm); err != nil {
+		panic(err.Error())
 	}
-	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), 0700); err != nil {
-		cmn.PanicSanity(err.Error())
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), DefaultDirPerm); err != nil {
+		panic(err.Error())
 	}
-	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), 0700); err != nil {
-		cmn.PanicSanity(err.Error())
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), DefaultDirPerm); err != nil {
+		panic(err.Error())
 	}
 
 	configFilePath := filepath.Join(rootDir, defaultConfigFilePath)
@@ -77,7 +81,18 @@ moniker = "{{ .BaseConfig.Moniker }}"
 # and verifying their commits
 fast_sync = {{ .BaseConfig.FastSync }}
 
-# Database backend: leveldb | memdb | cleveldb
+# Database backend: goleveldb | cleveldb | boltdb
+# * goleveldb (github.com/syndtr/goleveldb - most popular implementation)
+#   - pure go
+#   - stable
+# * cleveldb (uses levigo wrapper)
+#   - fast
+#   - requires gcc
+#   - use cleveldb build tag (go build -tags cleveldb)
+# * boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt)
+#   - EXPERIMENTAL
+#   - may be faster is some use-cases (random reads - indexer)
+#   - use boltdb build tag (go build -tags boltdb)
 db_backend = "{{ .BaseConfig.DBBackend }}"
 
 # Database directory
@@ -86,13 +101,19 @@ db_dir = "{{ js .BaseConfig.DBPath }}"
 # Output level for logging, including package level options
 log_level = "{{ .BaseConfig.LogLevel }}"
 
+# Output format: 'plain' (colored text) or 'json'
+log_format = "{{ .BaseConfig.LogFormat }}"
+
 ##### additional base config options #####
 
 # Path to the JSON file containing the initial validator set and other meta data
 genesis_file = "{{ js .BaseConfig.Genesis }}"
 
 # Path to the JSON file containing the private key to use as a validator in the consensus protocol
-priv_validator_file = "{{ js .BaseConfig.PrivValidator }}"
+priv_validator_key_file = "{{ js .BaseConfig.PrivValidatorKey }}"
+
+# Path to the JSON file containing the last sign state of a validator
+priv_validator_state_file = "{{ js .BaseConfig.PrivValidatorState }}"
 
 # TCP or UNIX socket address for Tendermint to listen on for
 # connections from an external PrivValidator process
@@ -119,13 +140,24 @@ filter_peers = {{ .BaseConfig.FilterPeers }}
 # TCP or UNIX socket address for the RPC server to listen on
 laddr = "{{ .RPC.ListenAddress }}"
 
+# A list of origins a cross-domain request can be executed from
+# Default value '[]' disables cors support
+# Use '["*"]' to allow any origin
+cors_allowed_origins = [{{ range .RPC.CORSAllowedOrigins }}{{ printf "%q, " . }}{{end}}]
+
+# A list of methods the client is allowed to use with cross-domain requests
+cors_allowed_methods = [{{ range .RPC.CORSAllowedMethods }}{{ printf "%q, " . }}{{end}}]
+
+# A list of non simple headers the client is allowed to use with cross-domain requests
+cors_allowed_headers = [{{ range .RPC.CORSAllowedHeaders }}{{ printf "%q, " . }}{{end}}]
+
 # TCP or UNIX socket address for the gRPC server to listen on
 # NOTE: This server only supports /broadcast_tx_commit
 grpc_laddr = "{{ .RPC.GRPCListenAddress }}"
 
 # Maximum number of simultaneous connections.
 # Does not include RPC (HTTP&WebSocket) connections. See max_open_connections
-# If you want to accept more significant number than the default, make sure
+# If you want to accept a larger number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
 # Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
@@ -137,13 +169,40 @@ unsafe = {{ .RPC.Unsafe }}
 
 # Maximum number of simultaneous connections (including WebSocket).
 # Does not include gRPC connections. See grpc_max_open_connections
-# If you want to accept more significant number than the default, make sure
+# If you want to accept a larger number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
 # Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
 # 1024 - 40 - 10 - 50 = 924 = ~900
 max_open_connections = {{ .RPC.MaxOpenConnections }}
 
+# Maximum number of unique clientIDs that can /subscribe
+# If you're using /broadcast_tx_commit, set to the estimated maximum number
+# of broadcast_tx_commit calls per block.
+max_subscription_clients = {{ .RPC.MaxSubscriptionClients }}
+
+# Maximum number of unique queries a given client can /subscribe to
+# If you're using GRPC (or Local RPC client) and /broadcast_tx_commit, set to
+# the estimated # maximum number of broadcast_tx_commit calls per block.
+max_subscriptions_per_client = {{ .RPC.MaxSubscriptionsPerClient }}
+
+# How long to wait for a tx to be committed during /broadcast_tx_commit.
+# WARNING: Using a value larger than 10s will result in increasing the
+# global HTTP write timeout, which applies to all connections and endpoints.
+# See https://github.com/tendermint/tendermint/issues/3435
+timeout_broadcast_tx_commit = "{{ .RPC.TimeoutBroadcastTxCommit }}"
+
+# The name of a file containing certificate that is used to create the HTTPS server.
+# If the certificate is signed by a certificate authority,
+# the certFile should be the concatenation of the server's certificate, any intermediates,
+# and the CA's certificate.
+# NOTE: both tls_cert_file and tls_key_file must be present for Tendermint to create HTTPS server. Otherwise, HTTP server is run.
+tls_cert_file = "{{ .RPC.TLSCertFile }}"
+
+# The name of a file containing matching private key that is used to create the HTTPS server.
+# NOTE: both tls_cert_file and tls_key_file must be present for Tendermint to create HTTPS server. Otherwise, HTTP server is run.
+tls_key_file = "{{ .RPC.TLSKeyFile }}"
+
 ##### peer to peer configuration options #####
 [p2p]
 
@@ -216,10 +275,15 @@ recheck = {{ .Mempool.Recheck }}
 broadcast = {{ .Mempool.Broadcast }}
 wal_dir = "{{ js .Mempool.WalPath }}"
 
-# size of the mempool
+# Maximum number of transactions in the mempool
 size = {{ .Mempool.Size }}
 
-# size of the cache (used to filter transactions we saw earlier)
+# Limit the total size of all txs in the mempool.
+# This only accounts for raw transactions (e.g. given 1MB transactions and
+# max_txs_bytes=5MB, mempool will only accept 5 transactions).
+max_txs_bytes = {{ .Mempool.MaxTxsBytes }}
+
+# Size of the cache (used to filter transactions we saw earlier) in transactions
 cache_size = {{ .Mempool.CacheSize }}
 
 ##### consensus configuration options #####
@@ -252,8 +316,8 @@ peer_query_maj23_sleep_duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"
 # What indexer to use for transactions
 #
 # Options:
-#   1) "null" (default)
-#   2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
+#   1) "null"
+#   2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
 indexer = "{{ .TxIndex.Indexer }}"
 
 # Comma-separated list of tags to index (by default the only tag is "tx.hash")
@@ -285,7 +349,7 @@ prometheus = {{ .Instrumentation.Prometheus }}
 prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
 
 # Maximum number of simultaneous connections.
-# If you want to accept more significant number than the default, make sure
+# If you want to accept a larger number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
 max_open_connections = {{ .Instrumentation.MaxOpenConnections }}
@@ -297,53 +361,51 @@ namespace = "{{ .Instrumentation.Namespace }}"
 /****** these are for test settings ***********/
 
 func ResetTestRoot(testName string) *Config {
-	rootDir := os.ExpandEnv("$HOME/.tendermint_test")
-	rootDir = filepath.Join(rootDir, testName)
-	// Remove ~/.tendermint_test_bak
-	if cmn.FileExists(rootDir + "_bak") {
-		if err := os.RemoveAll(rootDir + "_bak"); err != nil {
-			cmn.PanicSanity(err.Error())
-		}
+	return ResetTestRootWithChainID(testName, "")
+}
+
+func ResetTestRootWithChainID(testName string, chainID string) *Config {
+	// create a unique, concurrency-safe test directory under os.TempDir()
+	rootDir, err := ioutil.TempDir("", fmt.Sprintf("%s-%s_", chainID, testName))
+	if err != nil {
+		panic(err)
 	}
-	// Move ~/.tendermint_test to ~/.tendermint_test_bak
-	if cmn.FileExists(rootDir) {
-		if err := os.Rename(rootDir, rootDir+"_bak"); err != nil {
-			cmn.PanicSanity(err.Error())
-		}
+	// ensure config and data subdirs are created
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), DefaultDirPerm); err != nil {
+		panic(err)
 	}
-	// Create new dir
-	if err := cmn.EnsureDir(rootDir, 0700); err != nil {
-		cmn.PanicSanity(err.Error())
-	}
-	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), 0700); err != nil {
-		cmn.PanicSanity(err.Error())
-	}
-	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), 0700); err != nil {
-		cmn.PanicSanity(err.Error())
+	if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), DefaultDirPerm); err != nil {
+		panic(err)
 	}
 
 	baseConfig := DefaultBaseConfig()
 	configFilePath := filepath.Join(rootDir, defaultConfigFilePath)
 	genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis)
-	privFilePath := filepath.Join(rootDir, baseConfig.PrivValidator)
+	privKeyFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorKey)
+	privStateFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorState)
 
 	// Write default config file if missing.
 	if !cmn.FileExists(configFilePath) {
 		writeDefaultConfigFile(configFilePath)
 	}
 	if !cmn.FileExists(genesisFilePath) {
+		if chainID == "" {
+			chainID = "tendermint_test"
+		}
+		testGenesis := fmt.Sprintf(testGenesisFmt, chainID)
 		cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644)
 	}
 	// we always overwrite the priv val
-	cmn.MustWriteFile(privFilePath, []byte(testPrivValidator), 0644)
+	cmn.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644)
+	cmn.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644)
 
 	config := TestConfig().SetRoot(rootDir)
 	return config
 }
 
-var testGenesis = `{
+var testGenesisFmt = `{
   "genesis_time": "2018-10-10T08:20:13.695936996Z",
-  "chain_id": "tendermint_test",
+  "chain_id": "%s",
   "validators": [
     {
       "pub_key": {
@@ -357,7 +419,7 @@ var testGenesis = `{
   "app_hash": ""
 }`
 
-var testPrivValidator = `{
+var testPrivValidatorKey = `{
   "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145",
   "pub_key": {
     "type": "tendermint/PubKeyEd25519",
@@ -366,8 +428,11 @@ var testPrivValidator = `{
   "priv_key": {
     "type": "tendermint/PrivKeyEd25519",
     "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="
-  },
-  "last_height": "0",
-  "last_round": "0",
-  "last_step": 0
+  }
+}`
+
+var testPrivValidatorState = `{
+  "height": "0",
+  "round": "0",
+  "step": 0
 }`

config/toml_test.go

@@ -48,6 +48,7 @@ func TestEnsureTestRoot(t *testing.T) {
 
 	// create root dir
 	cfg := ResetTestRoot(testName)
+	defer os.RemoveAll(cfg.RootDir)
 	rootDir := cfg.RootDir
 
 	// make sure config is set properly
@@ -60,7 +61,7 @@ func TestEnsureTestRoot(t *testing.T) {
 
 	// TODO: make sure the cfg returned and testconfig are the same!
 	baseConfig := DefaultBaseConfig()
-	ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidator)
+	ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidatorKey, baseConfig.PrivValidatorState)
 }
 
 func checkConfig(configFile string) bool {

consensus/byzantine_test.go

@@ -10,13 +10,10 @@ import (
 	"github.com/stretchr/testify/require"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/p2p"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
-func init() {
-	config = ResetConfig("consensus_byzantine_test")
-}
-
 //----------------------------------------------
 // byzantine failures
 
@@ -29,7 +26,8 @@ func init() {
 func TestByzantine(t *testing.T) {
 	N := 4
 	logger := consensusLogger().With("test", "byzantine")
-	css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter)
+	css, cleanup := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter)
+	defer cleanup()
 
 	// give the byzantine validator a normal ticker
 	ticker := NewTimeoutTicker()
@@ -49,7 +47,7 @@ func TestByzantine(t *testing.T) {
 		switches[i].SetLogger(p2pLogger.With("validator", i))
 	}
 
-	eventChans := make([]chan interface{}, N)
+	blocksSubs := make([]types.Subscription, N)
 	reactors := make([]p2p.Reactor, N)
 	for i := 0; i < N; i++ {
 		// make first val byzantine
@@ -68,16 +66,15 @@ func TestByzantine(t *testing.T) {
 		eventBus := css[i].eventBus
 		eventBus.SetLogger(logger.With("module", "events", "validator", i))
 
-		eventChans[i] = make(chan interface{}, 1)
-		err := eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		var err error
+		blocksSubs[i], err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock)
 		require.NoError(t, err)
 
-		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
+		conR := NewConsensusReactor(css[i], true) // so we don't start the consensus states
 		conR.SetLogger(logger.With("validator", i))
 		conR.SetEventBus(eventBus)
 
-		var conRI p2p.Reactor // nolint: gotype, gosimple
-		conRI = conR
+		var conRI p2p.Reactor = conR
 
 		// make first val byzantine
 		if i == 0 {
@@ -85,6 +82,7 @@ func TestByzantine(t *testing.T) {
 		}
 
 		reactors[i] = conRI
+		sm.SaveState(css[i].blockExec.DB(), css[i].state) //for save height 1's validators info
 	}
 
 	defer func() {
@@ -135,7 +133,7 @@ func TestByzantine(t *testing.T) {
 	p2p.Connect2Switches(switches, ind1, ind2)
 
 	// wait for someone in the big partition (B) to make a block
-	<-eventChans[ind2]
+	<-blocksSubs[ind2].Out()
 
 	t.Log("A block has been committed. Healing partition")
 	p2p.Connect2Switches(switches, ind0, ind1)
@@ -147,7 +145,7 @@ func TestByzantine(t *testing.T) {
 	wg.Add(2)
 	for i := 1; i < N-1; i++ {
 		go func(j int) {
-			<-eventChans[j]
+			<-blocksSubs[j].Out()
 			wg.Done()
 		}(i)
 	}
@@ -272,3 +270,4 @@ func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
 func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) {
 	br.reactor.Receive(chID, peer, msgBytes)
 }
+func (br *ByzantineReactor) InitPeer(peer p2p.Peer) p2p.Peer { return peer }

consensus/common_test.go

@@ -6,14 +6,19 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
-	"reflect"
+	"path/filepath"
 	"sort"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/go-kit/kit/log/term"
+
+	"path"
+
 	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/example/counter"
+	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
@@ -21,25 +26,26 @@ import (
 	cmn "github.com/tendermint/tendermint/libs/common"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/privval"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 	tmtime "github.com/tendermint/tendermint/types/time"
-
-	"github.com/tendermint/tendermint/abci/example/counter"
-	"github.com/tendermint/tendermint/abci/example/kvstore"
-
-	"github.com/go-kit/kit/log/term"
 )
 
 const (
 	testSubscriber = "test-client"
 )
 
+// A cleanupFunc cleans up any config / test files created for a particular
+// test.
+type cleanupFunc func()
+
 // genesis, chain_id, priv_val
 var config *cfg.Config // NOTE: must be reset for each _test.go file
+var consensusReplayConfig *cfg.Config
 var ensureTimeout = time.Millisecond * 100
 
 func ensureDir(dir string, mode os.FileMode) {
@@ -72,9 +78,10 @@ func NewValidatorStub(privValidator types.PrivValidator, valIndex int) *validato
 }
 
 func (vs *validatorStub) signVote(voteType types.SignedMsgType, hash []byte, header types.PartSetHeader) (*types.Vote, error) {
+	addr := vs.PrivValidator.GetPubKey().Address()
 	vote := &types.Vote{
 		ValidatorIndex:   vs.Index,
-		ValidatorAddress: vs.PrivValidator.GetAddress(),
+		ValidatorAddress: addr,
 		Height:           vs.Height,
 		Round:            vs.Round,
 		Timestamp:        tmtime.Now(),
@@ -114,6 +121,24 @@ func incrementRound(vss ...*validatorStub) {
 	}
 }
 
+type ValidatorStubsByAddress []*validatorStub
+
+func (vss ValidatorStubsByAddress) Len() int {
+	return len(vss)
+}
+
+func (vss ValidatorStubsByAddress) Less(i, j int) bool {
+	return bytes.Compare(vss[i].GetPubKey().Address(), vss[j].GetPubKey().Address()) == -1
+}
+
+func (vss ValidatorStubsByAddress) Swap(i, j int) {
+	it := vss[i]
+	vss[i] = vss[j]
+	vss[i].Index = i
+	vss[j] = it
+	vss[j].Index = j
+}
+
 //-------------------------------------------------------------------------------
 // Functions for transitioning the consensus state
 
@@ -122,17 +147,21 @@ func startTestRound(cs *ConsensusState, height int64, round int) {
 	cs.startRoutines(0)
 }
 
-// Create proposal block from cs1 but sign it with vs
+// Create proposal block from cs1 but sign it with vs.
 func decideProposal(cs1 *ConsensusState, vs *validatorStub, height int64, round int) (proposal *types.Proposal, block *types.Block) {
+	cs1.mtx.Lock()
 	block, blockParts := cs1.createProposalBlock()
-	if block == nil { // on error
-		panic("error creating proposal block")
+	validRound := cs1.ValidRound
+	chainID := cs1.state.ChainID
+	cs1.mtx.Unlock()
+	if block == nil {
+		panic("Failed to createProposalBlock. Did you forget to add commit for previous block?")
 	}
 
 	// Make proposal
-	polRound, propBlockID := cs1.ValidRound, types.BlockID{block.Hash(), blockParts.Header()}
+	polRound, propBlockID := validRound, types.BlockID{block.Hash(), blockParts.Header()}
 	proposal = types.NewProposal(height, round, polRound, propBlockID)
-	if err := vs.SignProposal(cs1.state.ChainID, proposal); err != nil {
+	if err := vs.SignProposal(chainID, proposal); err != nil {
 		panic(err)
 	}
 	return
@@ -151,8 +180,9 @@ func signAddVotes(to *ConsensusState, voteType types.SignedMsgType, hash []byte,
 
 func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *validatorStub, blockHash []byte) {
 	prevotes := cs.Votes.Prevotes(round)
+	address := privVal.GetPubKey().Address()
 	var vote *types.Vote
-	if vote = prevotes.GetByAddress(privVal.GetAddress()); vote == nil {
+	if vote = prevotes.GetByAddress(address); vote == nil {
 		panic("Failed to find prevote from validator")
 	}
 	if blockHash == nil {
@@ -168,8 +198,9 @@ func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *valid
 
 func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorStub, blockHash []byte) {
 	votes := cs.LastCommit
+	address := privVal.GetPubKey().Address()
 	var vote *types.Vote
-	if vote = votes.GetByAddress(privVal.GetAddress()); vote == nil {
+	if vote = votes.GetByAddress(address); vote == nil {
 		panic("Failed to find precommit from validator")
 	}
 	if !bytes.Equal(vote.BlockID.Hash, blockHash) {
@@ -179,8 +210,9 @@ func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorS
 
 func validatePrecommit(t *testing.T, cs *ConsensusState, thisRound, lockRound int, privVal *validatorStub, votedBlockHash, lockedBlockHash []byte) {
 	precommits := cs.Votes.Precommits(thisRound)
+	address := privVal.GetPubKey().Address()
 	var vote *types.Vote
-	if vote = precommits.GetByAddress(privVal.GetAddress()); vote == nil {
+	if vote = precommits.GetByAddress(address); vote == nil {
 		panic("Failed to find precommit from validator")
 	}
 
@@ -215,30 +247,29 @@ func validatePrevoteAndPrecommit(t *testing.T, cs *ConsensusState, thisRound, lo
 	cs.mtx.Unlock()
 }
 
-// genesis
-func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} {
-	voteCh0 := make(chan interface{})
-	err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryVote, voteCh0)
+func subscribeToVoter(cs *ConsensusState, addr []byte) <-chan tmpubsub.Message {
+	votesSub, err := cs.eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, types.EventQueryVote)
 	if err != nil {
 		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote))
 	}
-	voteCh := make(chan interface{})
+	ch := make(chan tmpubsub.Message)
 	go func() {
-		for v := range voteCh0 {
-			vote := v.(types.EventDataVote)
+		for msg := range votesSub.Out() {
+			vote := msg.Data().(types.EventDataVote)
 			// we only fire for our own votes
 			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
-				voteCh <- v
+				ch <- msg
 			}
 		}
 	}()
-	return voteCh
+	return ch
 }
 
 //-------------------------------------------------------------------------------
 // consensus states
 
 func newConsensusState(state sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState {
+	config := cfg.ResetTestRoot("consensus_state_test")
 	return newConsensusStateWithConfig(config, state, pv, app)
 }
 
@@ -257,7 +288,7 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S
 	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
 
 	// Make Mempool
-	mempool := mempl.NewMempool(thisConfig.Mempool, proxyAppConnMem, 0)
+	mempool := mempl.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
 	mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
 	if thisConfig.Consensus.WaitForTxs() {
 		mempool.EnableTxsAvailable()
@@ -267,7 +298,8 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S
 	evpool := sm.MockEvidencePool{}
 
 	// Make ConsensusState
-	stateDB := dbm.NewMemDB()
+	stateDB := blockDB
+	sm.SaveState(stateDB, state) //for save height 1's validators info
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
 	cs := NewConsensusState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
 	cs.SetLogger(log.TestingLogger().With("module", "consensus"))
@@ -281,9 +313,10 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S
 }
 
 func loadPrivValidator(config *cfg.Config) *privval.FilePV {
-	privValidatorFile := config.PrivValidatorFile()
-	ensureDir(path.Dir(privValidatorFile), 0700)
-	privValidator := privval.LoadOrGenFilePV(privValidatorFile)
+	privValidatorKeyFile := config.PrivValidatorKeyFile()
+	ensureDir(filepath.Dir(privValidatorKeyFile), 0700)
+	privValidatorStateFile := config.PrivValidatorStateFile()
+	privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
 	privValidator.Reset()
 	return privValidator
 }
@@ -307,7 +340,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 
 //-------------------------------------------------------------------------------
 
-func ensureNoNewEvent(ch <-chan interface{}, timeout time.Duration,
+func ensureNoNewEvent(ch <-chan tmpubsub.Message, timeout time.Duration,
 	errorMessage string) {
 	select {
 	case <-time.After(timeout):
@@ -317,169 +350,186 @@ func ensureNoNewEvent(ch <-chan interface{}, timeout time.Duration,
 	}
 }
 
-func ensureNoNewEventOnChannel(ch <-chan interface{}) {
+func ensureNoNewEventOnChannel(ch <-chan tmpubsub.Message) {
 	ensureNoNewEvent(
 		ch,
 		ensureTimeout,
 		"We should be stuck waiting, not receiving new event on the channel")
 }
 
-func ensureNoNewRoundStep(stepCh <-chan interface{}) {
+func ensureNoNewRoundStep(stepCh <-chan tmpubsub.Message) {
 	ensureNoNewEvent(
 		stepCh,
 		ensureTimeout,
 		"We should be stuck waiting, not receiving NewRoundStep event")
 }
 
-func ensureNoNewUnlock(unlockCh <-chan interface{}) {
+func ensureNoNewUnlock(unlockCh <-chan tmpubsub.Message) {
 	ensureNoNewEvent(
 		unlockCh,
 		ensureTimeout,
 		"We should be stuck waiting, not receiving Unlock event")
 }
 
-func ensureNoNewTimeout(stepCh <-chan interface{}, timeout int64) {
-	timeoutDuration := time.Duration(timeout*5) * time.Nanosecond
+func ensureNoNewTimeout(stepCh <-chan tmpubsub.Message, timeout int64) {
+	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
 	ensureNoNewEvent(
 		stepCh,
 		timeoutDuration,
 		"We should be stuck waiting, not receiving NewTimeout event")
 }
 
-func ensureNewEvent(
-	ch <-chan interface{},
-	height int64,
-	round int,
-	timeout time.Duration,
-	errorMessage string) {
-
+func ensureNewEvent(ch <-chan tmpubsub.Message, height int64, round int, timeout time.Duration, errorMessage string) {
 	select {
 	case <-time.After(timeout):
 		panic(errorMessage)
-	case ev := <-ch:
-		rs, ok := ev.(types.EventDataRoundState)
+	case msg := <-ch:
+		roundStateEvent, ok := msg.Data().(types.EventDataRoundState)
 		if !ok {
-			panic(
-				fmt.Sprintf(
-					"expected a EventDataRoundState, got %v.Wrong subscription channel?",
-					reflect.TypeOf(rs)))
+			panic(fmt.Sprintf("expected a EventDataRoundState, got %T. Wrong subscription channel?",
+				msg.Data()))
 		}
-		if rs.Height != height {
-			panic(fmt.Sprintf("expected height %v, got %v", height, rs.Height))
+		if roundStateEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, roundStateEvent.Height))
 		}
-		if rs.Round != round {
-			panic(fmt.Sprintf("expected round %v, got %v", round, rs.Round))
+		if roundStateEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, roundStateEvent.Round))
 		}
 		// TODO: We could check also for a step at this point!
 	}
 }
 
-func ensureNewRoundStep(stepCh <-chan interface{}, height int64, round int) {
-	ensureNewEvent(
-		stepCh,
-		height,
-		round,
-		ensureTimeout,
-		"Timeout expired while waiting for NewStep event")
-}
-
-func ensureNewVote(voteCh <-chan interface{}, height int64, round int) {
+func ensureNewRound(roundCh <-chan tmpubsub.Message, height int64, round int) {
 	select {
 	case <-time.After(ensureTimeout):
-		break
-	case v := <-voteCh:
-		edv, ok := v.(types.EventDataVote)
+		panic("Timeout expired while waiting for NewRound event")
+	case msg := <-roundCh:
+		newRoundEvent, ok := msg.Data().(types.EventDataNewRound)
 		if !ok {
-			panic(fmt.Sprintf("expected a *types.Vote, "+
-				"got %v. wrong subscription channel?",
-				reflect.TypeOf(v)))
+			panic(fmt.Sprintf("expected a EventDataNewRound, got %T. Wrong subscription channel?",
+				msg.Data()))
 		}
-		vote := edv.Vote
-		if vote.Height != height {
-			panic(fmt.Sprintf("expected height %v, got %v", height, vote.Height))
+		if newRoundEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, newRoundEvent.Height))
 		}
-		if vote.Round != round {
-			panic(fmt.Sprintf("expected round %v, got %v", round, vote.Round))
+		if newRoundEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, newRoundEvent.Round))
 		}
 	}
 }
 
-func ensureNewRound(roundCh <-chan interface{}, height int64, round int) {
-	ensureNewEvent(roundCh, height, round, ensureTimeout,
-		"Timeout expired while waiting for NewRound event")
-}
-
-func ensureNewTimeout(timeoutCh <-chan interface{}, height int64, round int, timeout int64) {
-	timeoutDuration := time.Duration(timeout*3) * time.Nanosecond
+func ensureNewTimeout(timeoutCh <-chan tmpubsub.Message, height int64, round int, timeout int64) {
+	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
 	ensureNewEvent(timeoutCh, height, round, timeoutDuration,
 		"Timeout expired while waiting for NewTimeout event")
 }
 
-func ensureNewProposal(proposalCh <-chan interface{}, height int64, round int) {
-	ensureNewEvent(proposalCh, height, round, ensureTimeout,
-		"Timeout expired while waiting for NewProposal event")
+func ensureNewProposal(proposalCh <-chan tmpubsub.Message, height int64, round int) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewProposal event")
+	case msg := <-proposalCh:
+		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if proposalEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
+		}
+		if proposalEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
+		}
+	}
 }
 
-func ensureNewValidBlock(validBlockCh <-chan interface{}, height int64, round int) {
+func ensureNewValidBlock(validBlockCh <-chan tmpubsub.Message, height int64, round int) {
 	ensureNewEvent(validBlockCh, height, round, ensureTimeout,
 		"Timeout expired while waiting for NewValidBlock event")
 }
 
-func ensureNewBlock(blockCh <-chan interface{}, height int64) {
+func ensureNewBlock(blockCh <-chan tmpubsub.Message, height int64) {
 	select {
 	case <-time.After(ensureTimeout):
 		panic("Timeout expired while waiting for NewBlock event")
-	case ev := <-blockCh:
-		block, ok := ev.(types.EventDataNewBlock)
+	case msg := <-blockCh:
+		blockEvent, ok := msg.Data().(types.EventDataNewBlock)
 		if !ok {
-			panic(fmt.Sprintf("expected a *types.EventDataNewBlock, "+
-				"got %v. wrong subscription channel?",
-				reflect.TypeOf(block)))
+			panic(fmt.Sprintf("expected a EventDataNewBlock, got %T. Wrong subscription channel?",
+				msg.Data()))
 		}
-		if block.Block.Height != height {
-			panic(fmt.Sprintf("expected height %v, got %v", height, block.Block.Height))
+		if blockEvent.Block.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, blockEvent.Block.Height))
 		}
 	}
 }
 
-func ensureNewBlockHeader(blockCh <-chan interface{}, height int64, blockHash cmn.HexBytes) {
+func ensureNewBlockHeader(blockCh <-chan tmpubsub.Message, height int64, blockHash cmn.HexBytes) {
 	select {
 	case <-time.After(ensureTimeout):
 		panic("Timeout expired while waiting for NewBlockHeader event")
-	case ev := <-blockCh:
-		blockHeader, ok := ev.(types.EventDataNewBlockHeader)
+	case msg := <-blockCh:
+		blockHeaderEvent, ok := msg.Data().(types.EventDataNewBlockHeader)
 		if !ok {
-			panic(fmt.Sprintf("expected a *types.EventDataNewBlockHeader, "+
-				"got %v. wrong subscription channel?",
-				reflect.TypeOf(blockHeader)))
+			panic(fmt.Sprintf("expected a EventDataNewBlockHeader, got %T. Wrong subscription channel?",
+				msg.Data()))
 		}
-		if blockHeader.Header.Height != height {
-			panic(fmt.Sprintf("expected height %v, got %v", height, blockHeader.Header.Height))
+		if blockHeaderEvent.Header.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, blockHeaderEvent.Header.Height))
 		}
-		if !bytes.Equal(blockHeader.Header.Hash(), blockHash) {
-			panic(fmt.Sprintf("expected header %X, got %X", blockHash, blockHeader.Header.Hash()))
+		if !bytes.Equal(blockHeaderEvent.Header.Hash(), blockHash) {
+			panic(fmt.Sprintf("expected header %X, got %X", blockHash, blockHeaderEvent.Header.Hash()))
 		}
 	}
 }
 
-func ensureNewUnlock(unlockCh <-chan interface{}, height int64, round int) {
+func ensureNewUnlock(unlockCh <-chan tmpubsub.Message, height int64, round int) {
 	ensureNewEvent(unlockCh, height, round, ensureTimeout,
 		"Timeout expired while waiting for NewUnlock event")
 }
 
-func ensureVote(voteCh <-chan interface{}, height int64, round int,
+func ensureProposal(proposalCh <-chan tmpubsub.Message, height int64, round int, propID types.BlockID) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewProposal event")
+	case msg := <-proposalCh:
+		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if proposalEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
+		}
+		if proposalEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
+		}
+		if !proposalEvent.BlockID.Equals(propID) {
+			panic("Proposed block does not match expected block")
+		}
+	}
+}
+
+func ensurePrecommit(voteCh <-chan tmpubsub.Message, height int64, round int) {
+	ensureVote(voteCh, height, round, types.PrecommitType)
+}
+
+func ensurePrevote(voteCh <-chan tmpubsub.Message, height int64, round int) {
+	ensureVote(voteCh, height, round, types.PrevoteType)
+}
+
+func ensureVote(voteCh <-chan tmpubsub.Message, height int64, round int,
 	voteType types.SignedMsgType) {
 	select {
 	case <-time.After(ensureTimeout):
 		panic("Timeout expired while waiting for NewVote event")
-	case v := <-voteCh:
-		edv, ok := v.(types.EventDataVote)
+	case msg := <-voteCh:
+		voteEvent, ok := msg.Data().(types.EventDataVote)
 		if !ok {
-			panic(fmt.Sprintf("expected a *types.Vote, "+
-				"got %v. wrong subscription channel?",
-				reflect.TypeOf(v)))
+			panic(fmt.Sprintf("expected a EventDataVote, got %T. Wrong subscription channel?",
+				msg.Data()))
 		}
-		vote := edv.Vote
+		vote := voteEvent.Vote
 		if vote.Height != height {
 			panic(fmt.Sprintf("expected height %v, got %v", height, vote.Height))
 		}
@@ -492,15 +542,7 @@ func ensureVote(voteCh <-chan interface{}, height int64, round int,
 	}
 }
 
-func ensurePrecommit(voteCh <-chan interface{}, height int64, round int) {
-	ensureVote(voteCh, height, round, types.PrecommitType)
-}
-
-func ensurePrevote(voteCh <-chan interface{}, height int64, round int) {
-	ensureVote(voteCh, height, round, types.PrevoteType)
-}
-
-func ensureNewEventOnChannel(ch <-chan interface{}) {
+func ensureNewEventOnChannel(ch <-chan tmpubsub.Message) {
 	select {
 	case <-time.After(ensureTimeout):
 		panic("Timeout expired while waiting for new activity on the channel")
@@ -524,59 +566,85 @@ func consensusLogger() log.Logger {
 	}).With("module", "consensus")
 }
 
-func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application, configOpts ...func(*cfg.Config)) []*ConsensusState {
+func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker,
+	appFunc func() abci.Application, configOpts ...func(*cfg.Config)) ([]*ConsensusState, cleanupFunc) {
 	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
 	css := make([]*ConsensusState, nValidators)
 	logger := consensusLogger()
+	configRootDirs := make([]string, 0, nValidators)
 	for i := 0; i < nValidators; i++ {
 		stateDB := dbm.NewMemDB() // each state needs its own db
 		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
 		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		configRootDirs = append(configRootDirs, thisConfig.RootDir)
 		for _, opt := range configOpts {
 			opt(thisConfig)
 		}
-		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
 		app := appFunc()
 		vals := types.TM2PB.ValidatorUpdates(state.Validators)
 		app.InitChain(abci.RequestInitChain{Validators: vals})
 
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], app)
+		css[i] = newConsensusStateWithConfigAndBlockStore(thisConfig, state, privVals[i], app, stateDB)
 		css[i].SetTimeoutTicker(tickerFunc())
 		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
 	}
-	return css
+	return css, func() {
+		for _, dir := range configRootDirs {
+			os.RemoveAll(dir)
+		}
+	}
 }
 
 // nPeers = nValidators + nNotValidator
-func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application) []*ConsensusState {
+func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func(string) abci.Application) ([]*ConsensusState, *types.GenesisDoc, *cfg.Config, cleanupFunc) {
 	genDoc, privVals := randGenesisDoc(nValidators, false, testMinPower)
 	css := make([]*ConsensusState, nPeers)
 	logger := consensusLogger()
+	var peer0Config *cfg.Config
+	configRootDirs := make([]string, 0, nPeers)
 	for i := 0; i < nPeers; i++ {
 		stateDB := dbm.NewMemDB() // each state needs its own db
 		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
 		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
-		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		configRootDirs = append(configRootDirs, thisConfig.RootDir)
+		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		if i == 0 {
+			peer0Config = thisConfig
+		}
 		var privVal types.PrivValidator
 		if i < nValidators {
 			privVal = privVals[i]
 		} else {
-			tempFile, err := ioutil.TempFile("", "priv_validator_")
+			tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_")
 			if err != nil {
 				panic(err)
 			}
-			privVal = privval.GenFilePV(tempFile.Name())
+			tempStateFile, err := ioutil.TempFile("", "priv_validator_state_")
+			if err != nil {
+				panic(err)
+			}
+
+			privVal = privval.GenFilePV(tempKeyFile.Name(), tempStateFile.Name())
 		}
 
-		app := appFunc()
+		app := appFunc(path.Join(config.DBDir(), fmt.Sprintf("%s_%d", testName, i)))
 		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		if _, ok := app.(*kvstore.PersistentKVStoreApplication); ok {
+			state.Version.Consensus.App = kvstore.ProtocolVersion //simulate handshake, receive app version. If don't do this, replay test will fail
+		}
 		app.InitChain(abci.RequestInitChain{Validators: vals})
+		//sm.SaveState(stateDB,state)	//height 1's validatorsInfo already saved in LoadStateFromDBOrGenesisDoc above
 
 		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, app)
 		css[i].SetTimeoutTicker(tickerFunc())
 		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
 	}
-	return css
+	return css, genDoc, peer0Config, func() {
+		for _, dir := range configRootDirs {
+			os.RemoveAll(dir)
+		}
+	}
 }
 
 func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
@@ -586,7 +654,6 @@ func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
 		}
 	}
 	panic("didnt find peer in switches")
-	return -1
 }
 
 //-------------------------------------------------------------------------------
@@ -664,8 +731,7 @@ func (m *mockTicker) Chan() <-chan timeoutInfo {
 	return m.c
 }
 
-func (mockTicker) SetLogger(log.Logger) {
-}
+func (*mockTicker) SetLogger(log.Logger) {}
 
 //------------------------------------
 
@@ -674,6 +740,13 @@ func newCounter() abci.Application {
 }
 
 func newPersistentKVStore() abci.Application {
-	dir, _ := ioutil.TempDir("/tmp", "persistent-kvstore")
+	dir, err := ioutil.TempDir("", "persistent-kvstore")
+	if err != nil {
+		panic(err)
+	}
 	return kvstore.NewPersistentKVStoreApplication(dir)
 }
+
+func newPersistentKVStoreWithPath(dbDir string) abci.Application {
+	return kvstore.NewPersistentKVStoreApplication(dbDir)
+}

consensus/mempool_test.go

@@ -3,6 +3,7 @@ package consensus
 import (
 	"encoding/binary"
 	"fmt"
+	"os"
 	"testing"
 	"time"
 
@@ -10,19 +11,24 @@ import (
 
 	"github.com/tendermint/tendermint/abci/example/code"
 	abci "github.com/tendermint/tendermint/abci/types"
+	dbm "github.com/tendermint/tendermint/libs/db"
+	mempl "github.com/tendermint/tendermint/mempool"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
-func init() {
-	config = ResetConfig("consensus_mempool_test")
+// for testing
+func assertMempool(txn txNotifier) mempl.Mempool {
+	return txn.(mempl.Mempool)
 }
 
 func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
 	config := ResetConfig("consensus_mempool_txs_available_test")
+	defer os.RemoveAll(config.RootDir)
 	config.Consensus.CreateEmptyBlocks = false
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
-	cs.mempool.EnableTxsAvailable()
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
 	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	startTestRound(cs, height, round)
@@ -37,10 +43,11 @@ func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
 
 func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
 	config := ResetConfig("consensus_mempool_txs_available_test")
+	defer os.RemoveAll(config.RootDir)
 	config.Consensus.CreateEmptyBlocksInterval = ensureTimeout
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
-	cs.mempool.EnableTxsAvailable()
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
 	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	startTestRound(cs, height, round)
@@ -52,10 +59,11 @@ func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
 
 func TestMempoolProgressInHigherRound(t *testing.T) {
 	config := ResetConfig("consensus_mempool_txs_available_test")
+	defer os.RemoveAll(config.RootDir)
 	config.Consensus.CreateEmptyBlocks = false
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
-	cs.mempool.EnableTxsAvailable()
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
 	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
@@ -71,19 +79,19 @@ func TestMempoolProgressInHigherRound(t *testing.T) {
 	}
 	startTestRound(cs, height, round)
 
-	ensureNewRoundStep(newRoundCh, height, round) // first round at first height
-	ensureNewEventOnChannel(newBlockCh)           // first block gets committed
+	ensureNewRound(newRoundCh, height, round) // first round at first height
+	ensureNewEventOnChannel(newBlockCh)       // first block gets committed
 
 	height = height + 1 // moving to the next height
 	round = 0
 
-	ensureNewRoundStep(newRoundCh, height, round) // first round at next height
-	deliverTxsRange(cs, 0, 1)                     // we deliver txs, but dont set a proposal so we get the next round
+	ensureNewRound(newRoundCh, height, round) // first round at next height
+	deliverTxsRange(cs, 0, 1)                 // we deliver txs, but dont set a proposal so we get the next round
 	ensureNewTimeout(timeoutCh, height, round, cs.config.TimeoutPropose.Nanoseconds())
 
-	round = round + 1                             // moving to the next round
-	ensureNewRoundStep(newRoundCh, height, round) // wait for the next round
-	ensureNewEventOnChannel(newBlockCh)           // now we can commit the block
+	round = round + 1                         // moving to the next round
+	ensureNewRound(newRoundCh, height, round) // wait for the next round
+	ensureNewEventOnChannel(newBlockCh)       // now we can commit the block
 }
 
 func deliverTxsRange(cs *ConsensusState, start, end int) {
@@ -91,7 +99,7 @@ func deliverTxsRange(cs *ConsensusState, start, end int) {
 	for i := start; i < end; i++ {
 		txBytes := make([]byte, 8)
 		binary.BigEndian.PutUint64(txBytes, uint64(i))
-		err := cs.mempool.CheckTx(txBytes, nil)
+		err := assertMempool(cs.txNotifier).CheckTx(txBytes, nil)
 		if err != nil {
 			panic(fmt.Sprintf("Error after CheckTx: %v", err))
 		}
@@ -100,7 +108,9 @@ func deliverTxsRange(cs *ConsensusState, start, end int) {
 
 func TestMempoolTxConcurrentWithCommit(t *testing.T) {
 	state, privVals := randGenesisState(1, false, 10)
-	cs := newConsensusState(state, privVals[0], NewCounterApplication())
+	blockDB := dbm.NewMemDB()
+	cs := newConsensusStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockDB)
+	sm.SaveState(blockDB, state)
 	height, round := cs.Height, cs.Round
 	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
 
@@ -111,9 +121,9 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) {
 	for nTxs := 0; nTxs < NTxs; {
 		ticker := time.NewTicker(time.Second * 30)
 		select {
-		case b := <-newBlockCh:
-			evt := b.(types.EventDataNewBlock)
-			nTxs += int(evt.Block.Header.NumTxs)
+		case msg := <-newBlockCh:
+			blockEvent := msg.Data().(types.EventDataNewBlock)
+			nTxs += int(blockEvent.Block.Header.NumTxs)
 		case <-ticker.C:
 			panic("Timed out waiting to commit blocks with transactions")
 		}
@@ -123,7 +133,9 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) {
 func TestMempoolRmBadTx(t *testing.T) {
 	state, privVals := randGenesisState(1, false, 10)
 	app := NewCounterApplication()
-	cs := newConsensusState(state, privVals[0], app)
+	blockDB := dbm.NewMemDB()
+	cs := newConsensusStateWithConfigAndBlockStore(config, state, privVals[0], app, blockDB)
+	sm.SaveState(blockDB, state)
 
 	// increment the counter by 1
 	txBytes := make([]byte, 8)
@@ -141,7 +153,7 @@ func TestMempoolRmBadTx(t *testing.T) {
 		// Try to send the tx through the mempool.
 		// CheckTx should not err, but the app should return a bad abci code
 		// and the tx should get removed from the pool
-		err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) {
+		err := assertMempool(cs.txNotifier).CheckTx(txBytes, func(r *abci.Response) {
 			if r.GetCheckTx().Code != code.CodeTypeBadNonce {
 				t.Fatalf("expected checktx to return bad nonce, got %v", r)
 			}
@@ -153,7 +165,7 @@ func TestMempoolRmBadTx(t *testing.T) {
 
 		// check for the tx
 		for {
-			txs := cs.mempool.ReapMaxBytesMaxGas(int64(len(txBytes)), -1)
+			txs := assertMempool(cs.txNotifier).ReapMaxBytesMaxGas(int64(len(txBytes)), -1)
 			if len(txs) == 0 {
 				emptyMempoolCh <- struct{}{}
 				return

consensus/metrics.go

@@ -8,7 +8,11 @@ import (
 	stdprometheus "github.com/prometheus/client_golang/prometheus"
 )
 
-const MetricsSubsystem = "consensus"
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "consensus"
+)
 
 // Metrics contains metrics exposed by this package.
 type Metrics struct {
@@ -50,101 +54,107 @@ type Metrics struct {
 }
 
 // PrometheusMetrics returns Metrics build using Prometheus client library.
-func PrometheusMetrics(namespace string) *Metrics {
+// Optionally, labels can be provided along with their values ("foo",
+// "fooValue").
+func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
+	labels := []string{}
+	for i := 0; i < len(labelsAndValues); i += 2 {
+		labels = append(labels, labelsAndValues[i])
+	}
 	return &Metrics{
 		Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "height",
 			Help:      "Height of the chain.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "rounds",
 			Help:      "Number of rounds.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 
 		Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "validators",
 			Help:      "Number of validators.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "validators_power",
 			Help:      "Total power of all validators.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "missing_validators",
 			Help:      "Number of validators who did not sign.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "missing_validators_power",
 			Help:      "Total power of the missing validators.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "byzantine_validators",
 			Help:      "Number of validators who tried to double sign.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "byzantine_validators_power",
 			Help:      "Total power of the byzantine validators.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 
 		BlockIntervalSeconds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "block_interval_seconds",
 			Help:      "Time between this and the last block.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 
 		NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "num_txs",
 			Help:      "Number of transactions.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		BlockSizeBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "block_size_bytes",
 			Help:      "Size of the block.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "total_txs",
 			Help:      "Total number of transactions.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "latest_block_height",
 			Help:      "The latest block height.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		FastSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "fast_syncing",
 			Help:      "Whether or not a node is fast syncing. 1 if yes, 0 if no.",
-		}, []string{}),
+		}, labels).With(labelsAndValues...),
 		BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
 			Name:      "block_parts",
 			Help:      "Number of blockparts transmitted by peer.",
-		}, []string{"peer_id"}),
+		}, append(labels, "peer_id")).With(labelsAndValues...),
 	}
 }
 

consensus/reactor.go

@@ -155,16 +155,24 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor {
 	}
 }
 
-// AddPeer implements Reactor
+// InitPeer implements Reactor by creating a state for the peer.
+func (conR *ConsensusReactor) InitPeer(peer p2p.Peer) p2p.Peer {
+	peerState := NewPeerState(peer).SetLogger(conR.Logger)
+	peer.Set(types.PeerStateKey, peerState)
+	return peer
+}
+
+// AddPeer implements Reactor by spawning multiple gossiping goroutines for the
+// peer.
 func (conR *ConsensusReactor) AddPeer(peer p2p.Peer) {
 	if !conR.IsRunning() {
 		return
 	}
 
-	// Create peerState for peer
-	peerState := NewPeerState(peer).SetLogger(conR.Logger)
-	peer.Set(types.PeerStateKey, peerState)
-
+	peerState, ok := peer.Get(types.PeerStateKey).(*PeerState)
+	if !ok {
+		panic(fmt.Sprintf("peer %v has no state", peer))
+	}
 	// Begin routines for this peer.
 	go conR.gossipDataRoutine(peer, peerState)
 	go conR.gossipVotesRoutine(peer, peerState)
@@ -177,13 +185,17 @@ func (conR *ConsensusReactor) AddPeer(peer p2p.Peer) {
 	}
 }
 
-// RemovePeer implements Reactor
+// RemovePeer is a noop.
 func (conR *ConsensusReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
 	if !conR.IsRunning() {
 		return
 	}
 	// TODO
-	//peer.Get(PeerStateKey).(*PeerState).Disconnect()
+	// ps, ok := peer.Get(PeerStateKey).(*PeerState)
+	// if !ok {
+	// 	panic(fmt.Sprintf("Peer %v has no state", peer))
+	// }
+	// ps.Disconnect()
 }
 
 // Receive implements Reactor
@@ -214,7 +226,10 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 	conR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg)
 
 	// Get peer states
-	ps := src.Get(types.PeerStateKey).(*PeerState)
+	ps, ok := src.Get(types.PeerStateKey).(*PeerState)
+	if !ok {
+		panic(fmt.Sprintf("Peer %v has no state", src))
+	}
 
 	switch chID {
 	case StateChannel:
@@ -257,11 +272,6 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 				BlockID: msg.BlockID,
 				Votes:   ourVotes,
 			}))
-		case *ProposalHeartbeatMessage:
-			hb := msg.Heartbeat
-			conR.Logger.Debug("Received proposal heartbeat message",
-				"height", hb.Height, "round", hb.Round, "sequence", hb.Sequence,
-				"valIdx", hb.ValidatorIndex, "valAddr", hb.ValidatorAddress)
 		default:
 			conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg)))
 		}
@@ -293,9 +303,9 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 		switch msg := msg.(type) {
 		case *VoteMessage:
 			cs := conR.conS
-			cs.mtx.Lock()
+			cs.mtx.RLock()
 			height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size()
-			cs.mtx.Unlock()
+			cs.mtx.RUnlock()
 			ps.EnsureVoteBitArrays(height, valSize)
 			ps.EnsureVoteBitArrays(height-1, lastCommitSize)
 			ps.SetHasVote(msg.Vote)
@@ -362,8 +372,8 @@ func (conR *ConsensusReactor) FastSync() bool {
 
 //--------------------------------------
 
-// subscribeToBroadcastEvents subscribes for new round steps, votes and
-// proposal heartbeats using internal pubsub defined on state to broadcast
+// subscribeToBroadcastEvents subscribes for new round steps and votes
+// using internal pubsub defined on state to broadcast
 // them to peers upon receiving.
 func (conR *ConsensusReactor) subscribeToBroadcastEvents() {
 	const subscriber = "consensus-reactor"
@@ -382,10 +392,6 @@ func (conR *ConsensusReactor) subscribeToBroadcastEvents() {
 			conR.broadcastHasVoteMessage(data.(*types.Vote))
 		})
 
-	conR.conS.evsw.AddListenerForEvent(subscriber, types.EventProposalHeartbeat,
-		func(data tmevents.EventData) {
-			conR.broadcastProposalHeartbeatMessage(data.(*types.Heartbeat))
-		})
 }
 
 func (conR *ConsensusReactor) unsubscribeFromBroadcastEvents() {
@@ -393,13 +399,6 @@ func (conR *ConsensusReactor) unsubscribeFromBroadcastEvents() {
 	conR.conS.evsw.RemoveListener(subscriber)
 }
 
-func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(hb *types.Heartbeat) {
-	conR.Logger.Debug("Broadcasting proposal heartbeat message",
-		"height", hb.Height, "round", hb.Round, "sequence", hb.Sequence)
-	msg := &ProposalHeartbeatMessage{hb}
-	conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(msg))
-}
-
 func (conR *ConsensusReactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) {
 	nrsMsg := makeRoundStepMessage(rs)
 	conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg))
@@ -428,7 +427,10 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) {
 	/*
 		// TODO: Make this broadcast more selective.
 		for _, peer := range conR.Switch.Peers().List() {
-			ps := peer.Get(PeerStateKey).(*PeerState)
+			ps, ok := peer.Get(PeerStateKey).(*PeerState)
+			if !ok {
+				panic(fmt.Sprintf("Peer %v has no state", peer))
+			}
 			prs := ps.GetRoundState()
 			if prs.Height == vote.Height {
 				// TODO: Also filter on round?
@@ -444,9 +446,9 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) {
 
 func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage) {
 	nrsMsg = &NewRoundStepMessage{
-		Height: rs.Height,
-		Round:  rs.Round,
-		Step:   rs.Step,
+		Height:                rs.Height,
+		Round:                 rs.Round,
+		Step:                  rs.Step,
 		SecondsSinceStartTime: int(time.Since(rs.StartTime).Seconds()),
 		LastCommitRound:       rs.LastCommit.Round(),
 	}
@@ -497,7 +499,7 @@ OUTER_LOOP:
 			if prs.ProposalBlockParts == nil {
 				blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height)
 				if blockMeta == nil {
-					cmn.PanicCrisis(fmt.Sprintf("Failed to load block %d when blockStore is at %d",
+					panic(fmt.Sprintf("Failed to load block %d when blockStore is at %d",
 						prs.Height, conR.conS.blockStore.Height()))
 				}
 				ps.InitProposalBlockParts(blockMeta.BlockID.PartsHeader)
@@ -826,7 +828,10 @@ func (conR *ConsensusReactor) peerStatsRoutine() {
 				continue
 			}
 			// Get peer state
-			ps := peer.Get(types.PeerStateKey).(*PeerState)
+			ps, ok := peer.Get(types.PeerStateKey).(*PeerState)
+			if !ok {
+				panic(fmt.Sprintf("Peer %v has no state", peer))
+			}
 			switch msg.Msg.(type) {
 			case *VoteMessage:
 				if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 {
@@ -859,7 +864,10 @@ func (conR *ConsensusReactor) StringIndented(indent string) string {
 	s := "ConsensusReactor{\n"
 	s += indent + "  " + conR.conS.StringIndented(indent+"  ") + "\n"
 	for _, peer := range conR.Switch.Peers().List() {
-		ps := peer.Get(types.PeerStateKey).(*PeerState)
+		ps, ok := peer.Get(types.PeerStateKey).(*PeerState)
+		if !ok {
+			panic(fmt.Sprintf("Peer %v has no state", peer))
+		}
 		s += indent + "  " + ps.StringIndented(indent+"  ") + "\n"
 	}
 	s += indent + "}"
@@ -896,7 +904,7 @@ type PeerState struct {
 	peer   p2p.Peer
 	logger log.Logger
 
-	mtx   sync.Mutex             `json:"-"`           // NOTE: Modify below using setters, never directly.
+	mtx   sync.Mutex             // NOTE: Modify below using setters, never directly.
 	PRS   cstypes.PeerRoundState `json:"round_state"` // Exposed.
 	Stats *peerStateStats        `json:"stats"`       // Exposed.
 }
@@ -1017,7 +1025,11 @@ func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool {
 	if vote, ok := ps.PickVoteToSend(votes); ok {
 		msg := &VoteMessage{vote}
 		ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote)
-		return ps.peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(msg))
+		if ps.peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(msg)) {
+			ps.SetHasVote(vote)
+			return true
+		}
+		return false
 	}
 	return false
 }
@@ -1046,7 +1058,6 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote
 		return nil, false // Not something worth sending
 	}
 	if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok {
-		ps.setHasVote(height, round, type_, index)
 		return votes.GetByIndex(index), true
 	}
 	return nil, false
@@ -1107,7 +1118,7 @@ func (ps *PeerState) ensureCatchupCommitRound(height int64, round int, numValida
 		NOTE: This is wrong, 'round' could change.
 		e.g. if orig round is not the same as block LastCommit round.
 		if ps.CatchupCommitRound != -1 && ps.CatchupCommitRound != round {
-			cmn.PanicSanity(fmt.Sprintf("Conflicting CatchupCommitRound. Height: %v, Orig: %v, New: %v", height, ps.CatchupCommitRound, round))
+			panic(fmt.Sprintf("Conflicting CatchupCommitRound. Height: %v, Orig: %v, New: %v", height, ps.CatchupCommitRound, round))
 		}
 	*/
 	if ps.PRS.CatchupCommitRound == round {
@@ -1368,7 +1379,6 @@ func RegisterConsensusMessages(cdc *amino.Codec) {
 	cdc.RegisterConcrete(&HasVoteMessage{}, "tendermint/HasVote", nil)
 	cdc.RegisterConcrete(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23", nil)
 	cdc.RegisterConcrete(&VoteSetBitsMessage{}, "tendermint/VoteSetBits", nil)
-	cdc.RegisterConcrete(&ProposalHeartbeatMessage{}, "tendermint/ProposalHeartbeat", nil)
 }
 
 func decodeMsg(bz []byte) (msg ConsensusMessage, err error) {
@@ -1645,18 +1655,3 @@ func (m *VoteSetBitsMessage) String() string {
 }
 
 //-------------------------------------
-
-// ProposalHeartbeatMessage is sent to signal that a node is alive and waiting for transactions for a proposal.
-type ProposalHeartbeatMessage struct {
-	Heartbeat *types.Heartbeat
-}
-
-// ValidateBasic performs basic validation.
-func (m *ProposalHeartbeatMessage) ValidateBasic() error {
-	return m.Heartbeat.ValidateBasic()
-}
-
-// String returns a string representation.
-func (m *ProposalHeartbeatMessage) String() string {
-	return fmt.Sprintf("[HEARTBEAT %v]", m.Heartbeat)
-}

consensus/reactor_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/tendermint/tendermint/abci/client"
+	abcicli "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
 	bc "github.com/tendermint/tendermint/blockchain"
@@ -23,20 +23,21 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/p2p/mock"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
-func init() {
-	config = ResetConfig("consensus_reactor_test")
-}
-
 //----------------------------------------------
 // in-process testnets
 
-func startConsensusNet(t *testing.T, css []*ConsensusState, N int) ([]*ConsensusReactor, []chan interface{}, []*types.EventBus) {
+func startConsensusNet(t *testing.T, css []*ConsensusState, N int) (
+	[]*ConsensusReactor,
+	[]types.Subscription,
+	[]*types.EventBus,
+) {
 	reactors := make([]*ConsensusReactor, N)
-	eventChans := make([]chan interface{}, N)
+	blocksSubs := make([]types.Subscription, 0)
 	eventBuses := make([]*types.EventBus, N)
 	for i := 0; i < N; i++ {
 		/*logger, err := tmflags.ParseLogLevel("consensus:info,*:error", logger, "info")
@@ -48,9 +49,13 @@ func startConsensusNet(t *testing.T, css []*ConsensusState, N int) ([]*Consensus
 		eventBuses[i] = css[i].eventBus
 		reactors[i].SetEventBus(eventBuses[i])
 
-		eventChans[i] = make(chan interface{}, 1)
-		err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, eventChans[i])
+		blocksSub, err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock)
 		require.NoError(t, err)
+		blocksSubs = append(blocksSubs, blocksSub)
+
+		if css[i].state.LastBlockHeight == 0 { //simulate handle initChain in handshake
+			sm.SaveState(css[i].blockExec.DB(), css[i].state)
+		}
 	}
 	// make connected switches and start all reactors
 	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
@@ -67,7 +72,7 @@ func startConsensusNet(t *testing.T, css []*ConsensusState, N int) ([]*Consensus
 		s := reactors[i].conS.GetState()
 		reactors[i].SwitchToConsensus(s, 0)
 	}
-	return reactors, eventChans, eventBuses
+	return reactors, blocksSubs, eventBuses
 }
 
 func stopConsensusNet(logger log.Logger, reactors []*ConsensusReactor, eventBuses []*types.EventBus) {
@@ -86,12 +91,13 @@ func stopConsensusNet(logger log.Logger, reactors []*ConsensusReactor, eventBuse
 // Ensure a testnet makes blocks
 func TestReactorBasic(t *testing.T) {
 	N := 4
-	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
 	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N, func(j int) {
-		<-eventChans[j]
+		<-blocksSubs[j].Out()
 	}, css)
 }
 
@@ -116,6 +122,7 @@ func TestReactorWithEvidence(t *testing.T) {
 		stateDB := dbm.NewMemDB() // each state needs its own db
 		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
 		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		defer os.RemoveAll(thisConfig.RootDir)
 		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
 		app := appFunc()
 		vals := types.TM2PB.ValidatorUpdates(state.Validators)
@@ -134,7 +141,7 @@ func TestReactorWithEvidence(t *testing.T) {
 		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
 
 		// Make Mempool
-		mempool := mempl.NewMempool(thisConfig.Mempool, proxyAppConnMem, 0)
+		mempool := mempl.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
 		mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
 		if thisConfig.Consensus.WaitForTxs() {
 			mempool.EnableTxsAvailable()
@@ -143,7 +150,8 @@ func TestReactorWithEvidence(t *testing.T) {
 		// mock the evidence pool
 		// everyone includes evidence of another double signing
 		vIdx := (i + 1) % nValidators
-		evpool := newMockEvidencePool(privVals[vIdx].GetAddress())
+		addr := privVals[vIdx].GetPubKey().Address()
+		evpool := newMockEvidencePool(addr)
 
 		// Make ConsensusState
 		blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
@@ -162,20 +170,20 @@ func TestReactorWithEvidence(t *testing.T) {
 		css[i] = cs
 	}
 
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, nValidators)
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nValidators)
 	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 
 	// wait till everyone makes the first new block with no evidence
 	timeoutWaitGroup(t, nValidators, func(j int) {
-		blockI := <-eventChans[j]
-		block := blockI.(types.EventDataNewBlock).Block
+		msg := <-blocksSubs[j].Out()
+		block := msg.Data().(types.EventDataNewBlock).Block
 		assert.True(t, len(block.Evidence.Evidence) == 0)
 	}, css)
 
 	// second block should have evidence
 	timeoutWaitGroup(t, nValidators, func(j int) {
-		blockI := <-eventChans[j]
-		block := blockI.(types.EventDataNewBlock).Block
+		msg := <-blocksSubs[j].Out()
+		block := msg.Data().(types.EventDataNewBlock).Block
 		assert.True(t, len(block.Evidence.Evidence) > 0)
 	}, css)
 }
@@ -210,51 +218,86 @@ func (m *mockEvidencePool) Update(block *types.Block, state sm.State) {
 	}
 	m.height++
 }
+func (m *mockEvidencePool) IsCommitted(types.Evidence) bool { return false }
 
 //------------------------------------
 
-// Ensure a testnet sends proposal heartbeats and makes blocks when there are txs
-func TestReactorProposalHeartbeats(t *testing.T) {
+// Ensure a testnet makes blocks when there are txs
+func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) {
 	N := 4
-	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter,
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter,
 		func(c *cfg.Config) {
 			c.Consensus.CreateEmptyBlocks = false
 		})
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
 	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
-	heartbeatChans := make([]chan interface{}, N)
-	var err error
-	for i := 0; i < N; i++ {
-		heartbeatChans[i] = make(chan interface{}, 1)
-		err = eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryProposalHeartbeat, heartbeatChans[i])
-		require.NoError(t, err)
-	}
-	// wait till everyone sends a proposal heartbeat
-	timeoutWaitGroup(t, N, func(j int) {
-		<-heartbeatChans[j]
-	}, css)
 
 	// send a tx
-	if err := css[3].mempool.CheckTx([]byte{1, 2, 3}, nil); err != nil {
+	if err := assertMempool(css[3].txNotifier).CheckTx([]byte{1, 2, 3}, nil); err != nil {
 		//t.Fatal(err)
 	}
 
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N, func(j int) {
-		<-eventChans[j]
+		<-blocksSubs[j].Out()
 	}, css)
 }
 
+func TestReactorReceiveDoesNotPanicIfAddPeerHasntBeenCalledYet(t *testing.T) {
+	N := 1
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, _, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	var (
+		reactor = reactors[0]
+		peer    = mock.NewPeer(nil)
+		msg     = cdc.MustMarshalBinaryBare(&HasVoteMessage{Height: 1, Round: 1, Index: 1, Type: types.PrevoteType})
+	)
+
+	reactor.InitPeer(peer)
+
+	// simulate switch calling Receive before AddPeer
+	assert.NotPanics(t, func() {
+		reactor.Receive(StateChannel, peer, msg)
+		reactor.AddPeer(peer)
+	})
+}
+
+func TestReactorReceivePanicsIfInitPeerHasntBeenCalledYet(t *testing.T) {
+	N := 1
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, _, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	var (
+		reactor = reactors[0]
+		peer    = mock.NewPeer(nil)
+		msg     = cdc.MustMarshalBinaryBare(&HasVoteMessage{Height: 1, Round: 1, Index: 1, Type: types.PrevoteType})
+	)
+
+	// we should call InitPeer here
+
+	// simulate switch calling Receive before AddPeer
+	assert.Panics(t, func() {
+		reactor.Receive(StateChannel, peer, msg)
+	})
+}
+
 // Test we record stats about votes and block parts from other peers.
 func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
 	N := 4
-	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
 	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N, func(j int) {
-		<-eventChans[j]
+		<-blocksSubs[j].Out()
 	}, css)
 
 	// Get peer
@@ -272,19 +315,21 @@ func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
 func TestReactorVotingPowerChange(t *testing.T) {
 	nVals := 4
 	logger := log.TestingLogger()
-	css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentKVStore)
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, nVals)
+	css, cleanup := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentKVStore)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nVals)
 	defer stopConsensusNet(logger, reactors, eventBuses)
 
 	// map of active validators
 	activeVals := make(map[string]struct{})
 	for i := 0; i < nVals; i++ {
-		activeVals[string(css[i].privValidator.GetAddress())] = struct{}{}
+		addr := css[i].privValidator.GetPubKey().Address()
+		activeVals[string(addr)] = struct{}{}
 	}
 
 	// wait till everyone makes block 1
 	timeoutWaitGroup(t, nVals, func(j int) {
-		<-eventChans[j]
+		<-blocksSubs[j].Out()
 	}, css)
 
 	//---------------------------------------------------------------------------
@@ -295,10 +340,10 @@ func TestReactorVotingPowerChange(t *testing.T) {
 	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25)
 	previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower()
 
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlockWithTx(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
 
 	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
@@ -307,10 +352,10 @@ func TestReactorVotingPowerChange(t *testing.T) {
 	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2)
 	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
 
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlockWithTx(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
 
 	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
@@ -319,10 +364,10 @@ func TestReactorVotingPowerChange(t *testing.T) {
 	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26)
 	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
 
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlockWithTx(t, nVals, activeVals, eventChans, css, updateValidatorTx)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
-	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
 
 	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
 		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
@@ -332,22 +377,24 @@ func TestReactorVotingPowerChange(t *testing.T) {
 func TestReactorValidatorSetChanges(t *testing.T) {
 	nPeers := 7
 	nVals := 4
-	css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentKVStore)
+	css, _, _, cleanup := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentKVStoreWithPath)
 
+	defer cleanup()
 	logger := log.TestingLogger()
 
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, nPeers)
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nPeers)
 	defer stopConsensusNet(logger, reactors, eventBuses)
 
 	// map of active validators
 	activeVals := make(map[string]struct{})
 	for i := 0; i < nVals; i++ {
-		activeVals[string(css[i].privValidator.GetAddress())] = struct{}{}
+		addr := css[i].privValidator.GetPubKey().Address()
+		activeVals[string(addr)] = struct{}{}
 	}
 
 	// wait till everyone makes block 1
 	timeoutWaitGroup(t, nPeers, func(j int) {
-		<-eventChans[j]
+		<-blocksSubs[j].Out()
 	}, css)
 
 	//---------------------------------------------------------------------------
@@ -360,22 +407,22 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	// wait till everyone makes block 2
 	// ensure the commit includes all validators
 	// send newValTx to change vals in block 3
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx1)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, newValidatorTx1)
 
 	// wait till everyone makes block 3.
 	// it includes the commit for block 2, which is by the original validator set
-	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, newValidatorTx1)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, newValidatorTx1)
 
 	// wait till everyone makes block 4.
 	// it includes the commit for block 3, which is by the original validator set
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
 
 	// the commits for block 4 should be with the updated validator set
 	activeVals[string(newValidatorPubKey1.Address())] = struct{}{}
 
 	// wait till everyone makes block 5
 	// it includes the commit for block 4, which should have the updated validator set
-	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
 
 	//---------------------------------------------------------------------------
 	logger.Info("---------------------------- Testing changing the voting power of one validator")
@@ -385,10 +432,10 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
 	previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower()
 
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
-	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
-	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
 
 	if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
 		t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].GetRoundState().LastValidators.TotalVotingPower())
@@ -405,12 +452,12 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	newVal3ABCI := types.TM2PB.PubKey(newValidatorPubKey3)
 	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
 
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
-	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
 	activeVals[string(newValidatorPubKey2.Address())] = struct{}{}
 	activeVals[string(newValidatorPubKey3.Address())] = struct{}{}
-	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
 
 	//---------------------------------------------------------------------------
 	logger.Info("---------------------------- Testing removing two validators at once")
@@ -418,61 +465,70 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
 	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
 
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)
-	waitForAndValidateBlockWithTx(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
 	delete(activeVals, string(newValidatorPubKey2.Address()))
 	delete(activeVals, string(newValidatorPubKey3.Address()))
-	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, eventChans, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
 }
 
 // Check we can make blocks with skip_timeout_commit=false
 func TestReactorWithTimeoutCommit(t *testing.T) {
 	N := 4
-	css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter)
+	css, cleanup := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter)
+	defer cleanup()
 	// override default SkipTimeoutCommit == true for tests
 	for i := 0; i < N; i++ {
 		css[i].config.SkipTimeoutCommit = false
 	}
 
-	reactors, eventChans, eventBuses := startConsensusNet(t, css, N-1)
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N-1)
 	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 
 	// wait till everyone makes the first new block
 	timeoutWaitGroup(t, N-1, func(j int) {
-		<-eventChans[j]
+		<-blocksSubs[j].Out()
 	}, css)
 }
 
-func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
+func waitForAndValidateBlock(
+	t *testing.T,
+	n int,
+	activeVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*ConsensusState,
+	txs ...[]byte,
+) {
 	timeoutWaitGroup(t, n, func(j int) {
 		css[j].Logger.Debug("waitForAndValidateBlock")
-		newBlockI, ok := <-eventChans[j]
-		if !ok {
-			return
-		}
-		newBlock := newBlockI.(types.EventDataNewBlock).Block
+		msg := <-blocksSubs[j].Out()
+		newBlock := msg.Data().(types.EventDataNewBlock).Block
 		css[j].Logger.Debug("waitForAndValidateBlock: Got block", "height", newBlock.Height)
 		err := validateBlock(newBlock, activeVals)
 		assert.Nil(t, err)
 		for _, tx := range txs {
-			err := css[j].mempool.CheckTx(tx, nil)
+			err := assertMempool(css[j].txNotifier).CheckTx(tx, nil)
 			assert.Nil(t, err)
 		}
 	}, css)
 }
 
-func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
+func waitForAndValidateBlockWithTx(
+	t *testing.T,
+	n int,
+	activeVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*ConsensusState,
+	txs ...[]byte,
+) {
 	timeoutWaitGroup(t, n, func(j int) {
 		ntxs := 0
 	BLOCK_TX_LOOP:
 		for {
 			css[j].Logger.Debug("waitForAndValidateBlockWithTx", "ntxs", ntxs)
-			newBlockI, ok := <-eventChans[j]
-			if !ok {
-				return
-			}
-			newBlock := newBlockI.(types.EventDataNewBlock).Block
+			msg := <-blocksSubs[j].Out()
+			newBlock := msg.Data().(types.EventDataNewBlock).Block
 			css[j].Logger.Debug("waitForAndValidateBlockWithTx: Got block", "height", newBlock.Height)
 			err := validateBlock(newBlock, activeVals)
 			assert.Nil(t, err)
@@ -493,18 +549,21 @@ func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]st
 	}, css)
 }
 
-func waitForBlockWithUpdatedValsAndValidateIt(t *testing.T, n int, updatedVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState) {
+func waitForBlockWithUpdatedValsAndValidateIt(
+	t *testing.T,
+	n int,
+	updatedVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*ConsensusState,
+) {
 	timeoutWaitGroup(t, n, func(j int) {
 
 		var newBlock *types.Block
 	LOOP:
 		for {
 			css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt")
-			newBlockI, ok := <-eventChans[j]
-			if !ok {
-				return
-			}
-			newBlock = newBlockI.(types.EventDataNewBlock).Block
+			msg := <-blocksSubs[j].Out()
+			newBlock = msg.Data().(types.EventDataNewBlock).Block
 			if newBlock.LastCommit.Size() == len(updatedVals) {
 				css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt: Got block", "height", newBlock.Height)
 				break LOOP

consensus/replay.go

@@ -6,20 +6,21 @@ import (
 	"hash/crc32"
 	"io"
 	"reflect"
+
 	//"strconv"
 	//"strings"
 	"time"
 
 	abci "github.com/tendermint/tendermint/abci/types"
-	"github.com/tendermint/tendermint/version"
 	//auto "github.com/tendermint/tendermint/libs/autofile"
-	cmn "github.com/tendermint/tendermint/libs/common"
+
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
-
+	"github.com/tendermint/tendermint/mock"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/version"
 )
 
 var crc32c = crc32.MakeTable(crc32.Castagnoli)
@@ -41,7 +42,7 @@ var crc32c = crc32.MakeTable(crc32.Castagnoli)
 // Unmarshal and apply a single message to the consensus state as if it were
 // received in receiveRoutine.  Lines that start with "#" are ignored.
 // NOTE: receiveRoutine should not be running.
-func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan interface{}) error {
+func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscription) error {
 	// Skip meta messages which exist for demarcating boundaries.
 	if _, ok := msg.Msg.(EndHeightMessage); ok {
 		return nil
@@ -53,15 +54,17 @@ func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan
 		cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
 		// these are playback checks
 		ticker := time.After(time.Second * 2)
-		if newStepCh != nil {
+		if newStepSub != nil {
 			select {
-			case mi := <-newStepCh:
-				m2 := mi.(types.EventDataRoundState)
+			case stepMsg := <-newStepSub.Out():
+				m2 := stepMsg.Data().(types.EventDataRoundState)
 				if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
 					return fmt.Errorf("RoundState mismatch. Got %v; Expected %v", m2, m)
 				}
+			case <-newStepSub.Cancelled():
+				return fmt.Errorf("Failed to read off newStepSub.Out(). newStepSub was cancelled")
 			case <-ticker:
-				return fmt.Errorf("Failed to read off newStepCh")
+				return fmt.Errorf("Failed to read off newStepSub.Out()")
 			}
 		}
 	case msgInfo:
@@ -143,8 +146,8 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error {
 		if err == io.EOF {
 			break
 		} else if IsDataCorruptionError(err) {
-			cs.Logger.Debug("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight)
-			panic(fmt.Sprintf("data has been corrupted (%v) in last height %d of consensus WAL", err, csHeight))
+			cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight)
+			return err
 		} else if err != nil {
 			return err
 		}
@@ -196,6 +199,7 @@ type Handshaker struct {
 	stateDB      dbm.DB
 	initialState sm.State
 	store        sm.BlockStore
+	eventBus     types.BlockEventPublisher
 	genDoc       *types.GenesisDoc
 	logger       log.Logger
 
@@ -209,6 +213,7 @@ func NewHandshaker(stateDB dbm.DB, state sm.State,
 		stateDB:      stateDB,
 		initialState: state,
 		store:        store,
+		eventBus:     types.NopEventBus{},
 		genDoc:       genDoc,
 		logger:       log.NewNopLogger(),
 		nBlocks:      0,
@@ -219,6 +224,13 @@ func (h *Handshaker) SetLogger(l log.Logger) {
 	h.logger = l
 }
 
+// SetEventBus - sets the event bus for publishing block related events.
+// If not called, it defaults to types.NopEventBus.
+func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) {
+	h.eventBus = eventBus
+}
+
+// NBlocks returns the number of blocks applied to the state.
 func (h *Handshaker) NBlocks() int {
 	return h.nBlocks
 }
@@ -246,12 +258,15 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 	)
 
 	// Set AppVersion on the state.
-	h.initialState.Version.Consensus.App = version.Protocol(res.AppVersion)
+	if h.initialState.Version.Consensus.App != version.Protocol(res.AppVersion) {
+		h.initialState.Version.Consensus.App = version.Protocol(res.AppVersion)
+		sm.SaveState(h.stateDB, h.initialState)
+	}
 
 	// Replay blocks up to the latest in the blockstore.
 	_, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp)
 	if err != nil {
-		return fmt.Errorf("Error on replay: %v", err)
+		return fmt.Errorf("error on replay: %v", err)
 	}
 
 	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced",
@@ -262,7 +277,8 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 	return nil
 }
 
-// Replay all blocks since appBlockHeight and ensure the result matches the current state.
+// ReplayBlocks replays all blocks since appBlockHeight and ensures the result
+// matches the current state.
 // Returns the final AppHash or an error.
 func (h *Handshaker) ReplayBlocks(
 	state sm.State,
@@ -276,7 +292,12 @@ func (h *Handshaker) ReplayBlocks(
 
 	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain.
 	if appBlockHeight == 0 {
-		nextVals := types.TM2PB.ValidatorUpdates(state.NextValidators) // state.Validators would work too.
+		validators := make([]*types.Validator, len(h.genDoc.Validators))
+		for i, val := range h.genDoc.Validators {
+			validators[i] = types.NewValidator(val.PubKey, val.Power)
+		}
+		validatorSet := types.NewValidatorSet(validators)
+		nextVals := types.TM2PB.ValidatorUpdates(validatorSet)
 		csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams)
 		req := abci.RequestInitChain{
 			Time:            h.genDoc.GenesisTime,
@@ -290,36 +311,45 @@ func (h *Handshaker) ReplayBlocks(
 			return nil, err
 		}
 
-		// If the app returned validators or consensus params, update the state.
-		if len(res.Validators) > 0 {
-			vals, err := types.PB2TM.ValidatorUpdates(res.Validators)
-			if err != nil {
-				return nil, err
+		if stateBlockHeight == 0 { //we only update state when we are in initial state
+			// If the app returned validators or consensus params, update the state.
+			if len(res.Validators) > 0 {
+				vals, err := types.PB2TM.ValidatorUpdates(res.Validators)
+				if err != nil {
+					return nil, err
+				}
+				state.Validators = types.NewValidatorSet(vals)
+				state.NextValidators = types.NewValidatorSet(vals)
+			} else {
+				// If validator set is not set in genesis and still empty after InitChain, exit.
+				if len(h.genDoc.Validators) == 0 {
+					return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain")
+				}
 			}
-			state.Validators = types.NewValidatorSet(vals)
-			state.NextValidators = types.NewValidatorSet(vals)
+
+			if res.ConsensusParams != nil {
+				state.ConsensusParams = state.ConsensusParams.Update(res.ConsensusParams)
+			}
+			sm.SaveState(h.stateDB, state)
 		}
-		if res.ConsensusParams != nil {
-			state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams)
-		}
-		sm.SaveState(h.stateDB, state)
 	}
 
 	// First handle edge cases and constraints on the storeBlockHeight.
 	if storeBlockHeight == 0 {
-		return appHash, checkAppHash(state, appHash)
+		assertAppHashEqualsOneFromState(appHash, state)
+		return appHash, nil
 
 	} else if storeBlockHeight < appBlockHeight {
 		// the app should never be ahead of the store (but this is under app's control)
-		return appHash, sm.ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
+		return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight}
 
 	} else if storeBlockHeight < stateBlockHeight {
 		// the state should never be ahead of the store (this is under tendermint's control)
-		cmn.PanicSanity(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
+		panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
 
 	} else if storeBlockHeight > stateBlockHeight+1 {
 		// store should be at most one ahead of the state (this is under tendermint's control)
-		cmn.PanicSanity(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
+		panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
 	}
 
 	var err error
@@ -334,7 +364,8 @@ func (h *Handshaker) ReplayBlocks(
 
 		} else if appBlockHeight == storeBlockHeight {
 			// We're good!
-			return appHash, checkAppHash(state, appHash)
+			assertAppHashEqualsOneFromState(appHash, state)
+			return appHash, nil
 		}
 
 	} else if storeBlockHeight == stateBlockHeight+1 {
@@ -355,7 +386,7 @@ func (h *Handshaker) ReplayBlocks(
 			return state.AppHash, err
 
 		} else if appBlockHeight == storeBlockHeight {
-			// We ran Commit, but didn't save the state, so replayBlock with mock app
+			// We ran Commit, but didn't save the state, so replayBlock with mock app.
 			abciResponses, err := sm.LoadABCIResponses(h.stateDB, storeBlockHeight)
 			if err != nil {
 				return nil, err
@@ -368,8 +399,8 @@ func (h *Handshaker) ReplayBlocks(
 
 	}
 
-	cmn.PanicSanity("Should never happen")
-	return nil, nil
+	panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d",
+		appBlockHeight, storeBlockHeight, stateBlockHeight))
 }
 
 func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int64, mutateState bool) ([]byte, error) {
@@ -392,7 +423,12 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl
 	for i := appBlockHeight + 1; i <= finalBlock; i++ {
 		h.logger.Info("Applying block", "height", i)
 		block := h.store.LoadBlock(i)
-		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, state.LastValidators, h.stateDB)
+		// Extra check to ensure the app was not changed in a way it shouldn't have.
+		if len(appHash) > 0 {
+			assertAppHashEqualsOneFromBlock(appHash, block)
+		}
+
+		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, h.stateDB)
 		if err != nil {
 			return nil, err
 		}
@@ -409,7 +445,8 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl
 		appHash = state.AppHash
 	}
 
-	return appHash, checkAppHash(state, appHash)
+	assertAppHashEqualsOneFromState(appHash, state)
+	return appHash, nil
 }
 
 // ApplyBlock on the proxyApp with the last block.
@@ -417,7 +454,8 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap
 	block := h.store.LoadBlock(height)
 	meta := h.store.LoadBlockMeta(height)
 
-	blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, sm.MockMempool{}, sm.MockEvidencePool{})
+	blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, mock.Mempool{}, sm.MockEvidencePool{})
+	blockExec.SetEventBus(h.eventBus)
 
 	var err error
 	state, err = blockExec.ApplyBlock(state, meta.BlockID, block)
@@ -430,11 +468,26 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap
 	return state, nil
 }
 
-func checkAppHash(state sm.State, appHash []byte) error {
-	if !bytes.Equal(state.AppHash, appHash) {
-		panic(fmt.Errorf("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, state.AppHash).Error())
+func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) {
+	if !bytes.Equal(appHash, block.AppHash) {
+		panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X.
+
+Block: %v
+`,
+			appHash, block.AppHash, block))
+	}
+}
+
+func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) {
+	if !bytes.Equal(appHash, state.AppHash) {
+		panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got
+%X, expected %X.
+
+State: %v
+
+Did you reset Tendermint without resetting your application's data?`,
+			appHash, state.AppHash, state))
 	}
-	return nil
 }
 
 //--------------------------------------------------------------------------------
@@ -465,6 +518,9 @@ type mockProxyApp struct {
 func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx {
 	r := mock.abciResponses.DeliverTx[mock.txCount]
 	mock.txCount++
+	if r == nil { //it could be nil because of amino unMarshall, it will cause an empty ResponseDeliverTx to become nil
+		return abci.ResponseDeliverTx{}
+	}
 	return *r
 }
 

consensus/replay_file.go

@@ -16,6 +16,7 @@ import (
 	cmn "github.com/tendermint/tendermint/libs/common"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/mock"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -51,10 +52,9 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 	cs.startForReplay()
 
 	// ensure all new step events are regenerated as expected
-	newStepCh := make(chan interface{}, 1)
 
 	ctx := context.Background()
-	err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, newStepCh)
+	newStepSub, err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep)
 	if err != nil {
 		return errors.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)
 	}
@@ -83,7 +83,7 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 			return err
 		}
 
-		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {
+		if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil {
 			return err
 		}
 
@@ -92,7 +92,6 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 		}
 		pb.count++
 	}
-	return nil
 }
 
 //------------------------------------------------
@@ -121,12 +120,12 @@ func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState sm.S
 }
 
 // go back count steps by resetting the state and running (pb.count - count) steps
-func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
+func (pb *playback) replayReset(count int, newStepSub types.Subscription) error {
 	pb.cs.Stop()
 	pb.cs.Wait()
 
 	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec,
-		pb.cs.blockStore, pb.cs.mempool, pb.cs.evpool)
+		pb.cs.blockStore, pb.cs.txNotifier, pb.cs.evpool)
 	newCS.SetEventBus(pb.cs.eventBus)
 	newCS.startForReplay()
 
@@ -151,7 +150,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
 		} else if err != nil {
 			return err
 		}
-		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {
+		if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil {
 			return err
 		}
 		pb.count++
@@ -215,16 +214,15 @@ func (pb *playback) replayConsoleLoop() int {
 
 			ctx := context.Background()
 			// ensure all new step events are regenerated as expected
-			newStepCh := make(chan interface{}, 1)
 
-			err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep, newStepCh)
+			newStepSub, err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep)
 			if err != nil {
 				cmn.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep))
 			}
 			defer pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep)
 
 			if len(tokens) == 1 {
-				if err := pb.replayReset(1, newStepCh); err != nil {
+				if err := pb.replayReset(1, newStepSub); err != nil {
 					pb.cs.Logger.Error("Replay reset error", "err", err)
 				}
 			} else {
@@ -234,7 +232,7 @@ func (pb *playback) replayConsoleLoop() int {
 				} else if i > pb.count {
 					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
 				} else {
-					if err := pb.replayReset(i, newStepCh); err != nil {
+					if err := pb.replayReset(i, newStepSub); err != nil {
 						pb.cs.Logger.Error("Replay reset error", "err", err)
 					}
 				}
@@ -273,7 +271,6 @@ func (pb *playback) replayConsoleLoop() int {
 			fmt.Println(pb.count)
 		}
 	}
-	return 0
 }
 
 //--------------------------------------------------------------------------------
@@ -304,18 +301,19 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 		cmn.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err))
 	}
 
-	handshaker := NewHandshaker(stateDB, state, blockStore, gdoc)
-	err = handshaker.Handshake(proxyApp)
-	if err != nil {
-		cmn.Exit(fmt.Sprintf("Error on handshake: %v", err))
-	}
-
 	eventBus := types.NewEventBus()
 	if err := eventBus.Start(); err != nil {
 		cmn.Exit(fmt.Sprintf("Failed to start event bus: %v", err))
 	}
 
-	mempool, evpool := sm.MockMempool{}, sm.MockEvidencePool{}
+	handshaker := NewHandshaker(stateDB, state, blockStore, gdoc)
+	handshaker.SetEventBus(eventBus)
+	err = handshaker.Handshake(proxyApp)
+	if err != nil {
+		cmn.Exit(fmt.Sprintf("Error on handshake: %v", err))
+	}
+
+	mempool, evpool := mock.Mempool{}, sm.MockEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 
 	consensusState := NewConsensusState(csConfig, state.Copy(), blockExec,

consensus/replay_test.go

@@ -7,7 +7,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"path"
+	"path/filepath"
 	"runtime"
 	"testing"
 	"time"
@@ -15,25 +15,37 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"sort"
+
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
-	crypto "github.com/tendermint/tendermint/crypto"
-	auto "github.com/tendermint/tendermint/libs/autofile"
-	dbm "github.com/tendermint/tendermint/libs/db"
-	"github.com/tendermint/tendermint/version"
-
 	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/crypto"
+	cmn "github.com/tendermint/tendermint/libs/common"
+	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/mock"
 	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+	"github.com/tendermint/tendermint/version"
 )
 
-var consensusReplayConfig *cfg.Config
-
-func init() {
+func TestMain(m *testing.M) {
+	config = ResetConfig("consensus_reactor_test")
 	consensusReplayConfig = ResetConfig("consensus_replay_test")
+	configStateTest := ResetConfig("consensus_state_test")
+	configMempoolTest := ResetConfig("consensus_mempool_test")
+	configByzantineTest := ResetConfig("consensus_byzantine_test")
+	code := m.Run()
+	os.RemoveAll(config.RootDir)
+	os.RemoveAll(consensusReplayConfig.RootDir)
+	os.RemoveAll(configStateTest.RootDir)
+	os.RemoveAll(configMempoolTest.RootDir)
+	os.RemoveAll(configByzantineTest.RootDir)
+	os.Exit(code)
 }
 
 // These tests ensure we can always recover from failure at any part of the consensus process.
@@ -51,7 +63,8 @@ func init() {
 // and which ones we need the wal for - then we'd also be able to only flush the
 // wal writer when we need to, instead of with every message.
 
-func startNewConsensusStateAndWaitForBlock(t *testing.T, lastBlockHeight int64, blockDB dbm.DB, stateDB dbm.DB) {
+func startNewConsensusStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config,
+	lastBlockHeight int64, blockDB dbm.DB, stateDB dbm.DB) {
 	logger := log.TestingLogger()
 	state, _ := sm.LoadStateFromDBOrGenesisFile(stateDB, consensusReplayConfig.GenesisFile())
 	privValidator := loadPrivValidator(consensusReplayConfig)
@@ -59,7 +72,6 @@ func startNewConsensusStateAndWaitForBlock(t *testing.T, lastBlockHeight int64,
 	cs.SetLogger(logger)
 
 	bytes, _ := ioutil.ReadFile(cs.config.WalFile())
-	// fmt.Printf("====== WAL: \n\r%s\n", bytes)
 	t.Logf("====== WAL: \n\r%X\n", bytes)
 
 	err := cs.Start()
@@ -70,24 +82,25 @@ func startNewConsensusStateAndWaitForBlock(t *testing.T, lastBlockHeight int64,
 	// in the WAL itself. Assuming the consensus state is running, replay of any
 	// WAL, including the empty one, should eventually be followed by a new
 	// block, or else something is wrong.
-	newBlockCh := make(chan interface{}, 1)
-	err = cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, newBlockCh)
+	newBlockSub, err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock)
 	require.NoError(t, err)
 	select {
-	case <-newBlockCh:
-	case <-time.After(60 * time.Second):
-		t.Fatalf("Timed out waiting for new block (see trace above)")
+	case <-newBlockSub.Out():
+	case <-newBlockSub.Cancelled():
+		t.Fatal("newBlockSub was cancelled")
+	case <-time.After(120 * time.Second):
+		t.Fatal("Timed out waiting for new block (see trace above)")
 	}
 }
 
-func sendTxs(cs *ConsensusState, ctx context.Context) {
+func sendTxs(ctx context.Context, cs *ConsensusState) {
 	for i := 0; i < 256; i++ {
 		select {
 		case <-ctx.Done():
 			return
 		default:
 			tx := []byte{byte(i)}
-			cs.mempool.CheckTx(tx, nil)
+			assertMempool(cs.txNotifier).CheckTx(tx, nil)
 			i++
 		}
 	}
@@ -105,34 +118,35 @@ func TestWALCrash(t *testing.T) {
 			1},
 		{"many non-empty blocks",
 			func(stateDB dbm.DB, cs *ConsensusState, ctx context.Context) {
-				go sendTxs(cs, ctx)
+				go sendTxs(ctx, cs)
 			},
 			3},
 	}
 
-	for _, tc := range testCases {
+	for i, tc := range testCases {
+		consensusReplayConfig := ResetConfig(fmt.Sprintf("%s_%d", t.Name(), i))
 		t.Run(tc.name, func(t *testing.T) {
-			crashWALandCheckLiveness(t, tc.initFn, tc.heightToStop)
+			crashWALandCheckLiveness(t, consensusReplayConfig, tc.initFn, tc.heightToStop)
 		})
 	}
 }
 
-func crashWALandCheckLiveness(t *testing.T, initFn func(dbm.DB, *ConsensusState, context.Context), heightToStop int64) {
-	walPaniced := make(chan error)
-	crashingWal := &crashingWAL{panicCh: walPaniced, heightToStop: heightToStop}
+func crashWALandCheckLiveness(t *testing.T, consensusReplayConfig *cfg.Config,
+	initFn func(dbm.DB, *ConsensusState, context.Context), heightToStop int64) {
+	walPanicked := make(chan error)
+	crashingWal := &crashingWAL{panicCh: walPanicked, heightToStop: heightToStop}
 
 	i := 1
 LOOP:
 	for {
-		// fmt.Printf("====== LOOP %d\n", i)
 		t.Logf("====== LOOP %d\n", i)
 
 		// create consensus state from a clean slate
 		logger := log.NewNopLogger()
-		stateDB := dbm.NewMemDB()
+		blockDB := dbm.NewMemDB()
+		stateDB := blockDB
 		state, _ := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile())
 		privValidator := loadPrivValidator(consensusReplayConfig)
-		blockDB := dbm.NewMemDB()
 		cs := newConsensusStateWithConfigAndBlockStore(consensusReplayConfig, state, privValidator, kvstore.NewKVStoreApplication(), blockDB)
 		cs.SetLogger(logger)
 
@@ -159,11 +173,11 @@ LOOP:
 		i++
 
 		select {
-		case err := <-walPaniced:
-			t.Logf("WAL paniced: %v", err)
+		case err := <-walPanicked:
+			t.Logf("WAL panicked: %v", err)
 
 			// make sure we can make blocks after a crash
-			startNewConsensusStateAndWaitForBlock(t, cs.Height, blockDB, stateDB)
+			startNewConsensusStateAndWaitForBlock(t, consensusReplayConfig, cs.Height, blockDB, stateDB)
 
 			// stop consensus state and transactions sender (initFn)
 			cs.Stop()
@@ -181,16 +195,18 @@ LOOP:
 
 // crashingWAL is a WAL which crashes or rather simulates a crash during Save
 // (before and after). It remembers a message for which we last panicked
-// (lastPanicedForMsgIndex), so we don't panic for it in subsequent iterations.
+// (lastPanickedForMsgIndex), so we don't panic for it in subsequent iterations.
 type crashingWAL struct {
 	next         WAL
 	panicCh      chan error
 	heightToStop int64
 
-	msgIndex               int // current message index
-	lastPanicedForMsgIndex int // last message for which we panicked
+	msgIndex                int // current message index
+	lastPanickedForMsgIndex int // last message for which we panicked
 }
 
+var _ WAL = &crashingWAL{}
+
 // WALWriteError indicates a WAL crash.
 type WALWriteError struct {
 	msg string
@@ -223,8 +239,8 @@ func (w *crashingWAL) Write(m WALMessage) {
 		return
 	}
 
-	if w.msgIndex > w.lastPanicedForMsgIndex {
-		w.lastPanicedForMsgIndex = w.msgIndex
+	if w.msgIndex > w.lastPanickedForMsgIndex {
+		w.lastPanickedForMsgIndex = w.msgIndex
 		_, file, line, _ := runtime.Caller(1)
 		w.panicCh <- WALWriteError{fmt.Sprintf("failed to write %T to WAL (fileline: %s:%d)", m, file, line)}
 		runtime.Goexit()
@@ -238,8 +254,9 @@ func (w *crashingWAL) WriteSync(m WALMessage) {
 	w.Write(m)
 }
 
-func (w *crashingWAL) Group() *auto.Group { return w.next.Group() }
-func (w *crashingWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+func (w *crashingWAL) FlushAndSync() error { return w.next.FlushAndSync() }
+
+func (w *crashingWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
 	return w.next.SearchForEndHeight(height, options)
 }
 
@@ -248,15 +265,23 @@ func (w *crashingWAL) Stop() error  { return w.next.Stop() }
 func (w *crashingWAL) Wait()        { w.next.Wait() }
 
 //------------------------------------------------------------------------------------------
-// Handshake Tests
+type testSim struct {
+	GenesisState sm.State
+	Config       *cfg.Config
+	Chain        []*types.Block
+	Commits      []*types.Commit
+	CleanupFunc  cleanupFunc
+}
 
 const (
-	NUM_BLOCKS = 6
+	numBlocks = 6
 )
 
 var (
-	mempool = sm.MockMempool{}
+	mempool = mock.Mempool{}
 	evpool  = sm.MockEvidencePool{}
+
+	sim testSim
 )
 
 //---------------------------------------
@@ -267,94 +292,356 @@ var (
 // 2 - save block and committed but state is behind
 var modes = []uint{0, 1, 2}
 
+// This is actually not a test, it's for storing validator change tx data for testHandshakeReplay
+func TestSimulateValidatorsChange(t *testing.T) {
+	nPeers := 7
+	nVals := 4
+	css, genDoc, config, cleanup := randConsensusNetWithPeers(nVals, nPeers, "replay_test", newMockTickerFunc(true), newPersistentKVStoreWithPath)
+	sim.Config = config
+	sim.GenesisState, _ = sm.MakeGenesisState(genDoc)
+	sim.CleanupFunc = cleanup
+
+	partSize := types.BlockPartSizeBytes
+
+	newRoundCh := subscribe(css[0].eventBus, types.EventQueryNewRound)
+	proposalCh := subscribe(css[0].eventBus, types.EventQueryCompleteProposal)
+
+	vss := make([]*validatorStub, nPeers)
+	for i := 0; i < nPeers; i++ {
+		vss[i] = NewValidatorStub(css[i].privValidator, i)
+	}
+	height, round := css[0].Height, css[0].Round
+	// start the machine
+	startTestRound(css[0], height, round)
+	incrementHeight(vss...)
+	ensureNewRound(newRoundCh, height, 0)
+	ensureNewProposal(proposalCh, height, round)
+	rs := css[0].GetRoundState()
+	signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...)
+	ensureNewRound(newRoundCh, height+1, 0)
+
+	//height 2
+	height++
+	incrementHeight(vss...)
+	newValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
+	valPubKey1ABCI := types.TM2PB.PubKey(newValidatorPubKey1)
+	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
+	err := assertMempool(css[0].txNotifier).CheckTx(newValidatorTx1, nil)
+	assert.Nil(t, err)
+	propBlock, _ := css[0].createProposalBlock() //changeProposer(t, cs1, vs2)
+	propBlockParts := propBlock.MakePartSet(partSize)
+	blockID := types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()}
+	proposal := types.NewProposal(vss[1].Height, round, -1, blockID)
+	if err := vss[1].SignProposal(config.ChainID(), proposal); err != nil {
+		t.Fatal("failed to sign bad proposal", err)
+	}
+
+	// set the proposal block
+	if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
+	ensureNewProposal(proposalCh, height, round)
+	rs = css[0].GetRoundState()
+	signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...)
+	ensureNewRound(newRoundCh, height+1, 0)
+
+	//height 3
+	height++
+	incrementHeight(vss...)
+	updateValidatorPubKey1 := css[nVals].privValidator.GetPubKey()
+	updatePubKey1ABCI := types.TM2PB.PubKey(updateValidatorPubKey1)
+	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
+	err = assertMempool(css[0].txNotifier).CheckTx(updateValidatorTx1, nil)
+	assert.Nil(t, err)
+	propBlock, _ = css[0].createProposalBlock() //changeProposer(t, cs1, vs2)
+	propBlockParts = propBlock.MakePartSet(partSize)
+	blockID = types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()}
+	proposal = types.NewProposal(vss[2].Height, round, -1, blockID)
+	if err := vss[2].SignProposal(config.ChainID(), proposal); err != nil {
+		t.Fatal("failed to sign bad proposal", err)
+	}
+
+	// set the proposal block
+	if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
+	ensureNewProposal(proposalCh, height, round)
+	rs = css[0].GetRoundState()
+	signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...)
+	ensureNewRound(newRoundCh, height+1, 0)
+
+	//height 4
+	height++
+	incrementHeight(vss...)
+	newValidatorPubKey2 := css[nVals+1].privValidator.GetPubKey()
+	newVal2ABCI := types.TM2PB.PubKey(newValidatorPubKey2)
+	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
+	err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx2, nil)
+	assert.Nil(t, err)
+	newValidatorPubKey3 := css[nVals+2].privValidator.GetPubKey()
+	newVal3ABCI := types.TM2PB.PubKey(newValidatorPubKey3)
+	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
+	err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx3, nil)
+	assert.Nil(t, err)
+	propBlock, _ = css[0].createProposalBlock() //changeProposer(t, cs1, vs2)
+	propBlockParts = propBlock.MakePartSet(partSize)
+	blockID = types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()}
+	newVss := make([]*validatorStub, nVals+1)
+	copy(newVss, vss[:nVals+1])
+	sort.Sort(ValidatorStubsByAddress(newVss))
+	selfIndex := 0
+	for i, vs := range newVss {
+		if vs.GetPubKey().Equals(css[0].privValidator.GetPubKey()) {
+			selfIndex = i
+			break
+		}
+	}
+
+	proposal = types.NewProposal(vss[3].Height, round, -1, blockID)
+	if err := vss[3].SignProposal(config.ChainID(), proposal); err != nil {
+		t.Fatal("failed to sign bad proposal", err)
+	}
+
+	// set the proposal block
+	if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
+	ensureNewProposal(proposalCh, height, round)
+
+	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
+	err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx2, nil)
+	assert.Nil(t, err)
+
+	rs = css[0].GetRoundState()
+	for i := 0; i < nVals+1; i++ {
+		if i == selfIndex {
+			continue
+		}
+		signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i])
+	}
+
+	ensureNewRound(newRoundCh, height+1, 0)
+
+	//height 5
+	height++
+	incrementHeight(vss...)
+	ensureNewProposal(proposalCh, height, round)
+	rs = css[0].GetRoundState()
+	for i := 0; i < nVals+1; i++ {
+		if i == selfIndex {
+			continue
+		}
+		signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i])
+	}
+	ensureNewRound(newRoundCh, height+1, 0)
+
+	//height 6
+	height++
+	incrementHeight(vss...)
+	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
+	err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx3, nil)
+	assert.Nil(t, err)
+	propBlock, _ = css[0].createProposalBlock() //changeProposer(t, cs1, vs2)
+	propBlockParts = propBlock.MakePartSet(partSize)
+	blockID = types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()}
+	newVss = make([]*validatorStub, nVals+3)
+	copy(newVss, vss[:nVals+3])
+	sort.Sort(ValidatorStubsByAddress(newVss))
+	for i, vs := range newVss {
+		if vs.GetPubKey().Equals(css[0].privValidator.GetPubKey()) {
+			selfIndex = i
+			break
+		}
+	}
+	proposal = types.NewProposal(vss[1].Height, round, -1, blockID)
+	if err := vss[1].SignProposal(config.ChainID(), proposal); err != nil {
+		t.Fatal("failed to sign bad proposal", err)
+	}
+
+	// set the proposal block
+	if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
+	ensureNewProposal(proposalCh, height, round)
+	rs = css[0].GetRoundState()
+	for i := 0; i < nVals+3; i++ {
+		if i == selfIndex {
+			continue
+		}
+		signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i])
+	}
+	ensureNewRound(newRoundCh, height+1, 0)
+
+	sim.Chain = make([]*types.Block, 0)
+	sim.Commits = make([]*types.Commit, 0)
+	for i := 1; i <= numBlocks; i++ {
+		sim.Chain = append(sim.Chain, css[0].blockStore.LoadBlock(int64(i)))
+		sim.Commits = append(sim.Commits, css[0].blockStore.LoadBlockCommit(int64(i)))
+	}
+}
+
 // Sync from scratch
 func TestHandshakeReplayAll(t *testing.T) {
 	for _, m := range modes {
-		testHandshakeReplay(t, 0, m)
+		testHandshakeReplay(t, config, 0, m, false)
+	}
+	for _, m := range modes {
+		testHandshakeReplay(t, config, 0, m, true)
 	}
 }
 
 // Sync many, not from scratch
 func TestHandshakeReplaySome(t *testing.T) {
 	for _, m := range modes {
-		testHandshakeReplay(t, 1, m)
+		testHandshakeReplay(t, config, 1, m, false)
+	}
+	for _, m := range modes {
+		testHandshakeReplay(t, config, 1, m, true)
 	}
 }
 
 // Sync from lagging by one
 func TestHandshakeReplayOne(t *testing.T) {
 	for _, m := range modes {
-		testHandshakeReplay(t, NUM_BLOCKS-1, m)
+		testHandshakeReplay(t, config, numBlocks-1, m, false)
+	}
+	for _, m := range modes {
+		testHandshakeReplay(t, config, numBlocks-1, m, true)
 	}
 }
 
 // Sync from caught up
 func TestHandshakeReplayNone(t *testing.T) {
 	for _, m := range modes {
-		testHandshakeReplay(t, NUM_BLOCKS, m)
+		testHandshakeReplay(t, config, numBlocks, m, false)
 	}
+	for _, m := range modes {
+		testHandshakeReplay(t, config, numBlocks, m, true)
+	}
+}
+
+// Test mockProxyApp should not panic when app return ABCIResponses with some empty ResponseDeliverTx
+func TestMockProxyApp(t *testing.T) {
+	sim.CleanupFunc() //clean the test env created in TestSimulateValidatorsChange
+	logger := log.TestingLogger()
+	var validTxs, invalidTxs = 0, 0
+	txIndex := 0
+
+	assert.NotPanics(t, func() {
+		abciResWithEmptyDeliverTx := new(sm.ABCIResponses)
+		abciResWithEmptyDeliverTx.DeliverTx = make([]*abci.ResponseDeliverTx, 0)
+		abciResWithEmptyDeliverTx.DeliverTx = append(abciResWithEmptyDeliverTx.DeliverTx, &abci.ResponseDeliverTx{})
+
+		// called when saveABCIResponses:
+		bytes := cdc.MustMarshalBinaryBare(abciResWithEmptyDeliverTx)
+		loadedAbciRes := new(sm.ABCIResponses)
+
+		// this also happens sm.LoadABCIResponses
+		err := cdc.UnmarshalBinaryBare(bytes, loadedAbciRes)
+		require.NoError(t, err)
+
+		mock := newMockProxyApp([]byte("mock_hash"), loadedAbciRes)
+
+		abciRes := new(sm.ABCIResponses)
+		abciRes.DeliverTx = make([]*abci.ResponseDeliverTx, len(loadedAbciRes.DeliverTx))
+		// Execute transactions and get hash.
+		proxyCb := func(req *abci.Request, res *abci.Response) {
+			switch r := res.Value.(type) {
+			case *abci.Response_DeliverTx:
+				// TODO: make use of res.Log
+				// TODO: make use of this info
+				// Blocks may include invalid txs.
+				txRes := r.DeliverTx
+				if txRes.Code == abci.CodeTypeOK {
+					validTxs++
+				} else {
+					logger.Debug("Invalid tx", "code", txRes.Code, "log", txRes.Log)
+					invalidTxs++
+				}
+				abciRes.DeliverTx[txIndex] = txRes
+				txIndex++
+			}
+		}
+		mock.SetResponseCallback(proxyCb)
+
+		someTx := []byte("tx")
+		mock.DeliverTxAsync(someTx)
+	})
+	assert.True(t, validTxs == 1)
+	assert.True(t, invalidTxs == 0)
 }
 
 func tempWALWithData(data []byte) string {
 	walFile, err := ioutil.TempFile("", "wal")
 	if err != nil {
-		panic(fmt.Errorf("failed to create temp WAL file: %v", err))
+		panic(fmt.Sprintf("failed to create temp WAL file: %v", err))
 	}
 	_, err = walFile.Write(data)
 	if err != nil {
-		panic(fmt.Errorf("failed to write to temp WAL file: %v", err))
+		panic(fmt.Sprintf("failed to write to temp WAL file: %v", err))
 	}
 	if err := walFile.Close(); err != nil {
-		panic(fmt.Errorf("failed to close temp WAL file: %v", err))
+		panic(fmt.Sprintf("failed to close temp WAL file: %v", err))
 	}
 	return walFile.Name()
 }
 
 // Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
-func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
-	config := ResetConfig("proxy_test_")
+func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uint, testValidatorsChange bool) {
+	var chain []*types.Block
+	var commits []*types.Commit
+	var store *mockBlockStore
+	var stateDB dbm.DB
+	var genisisState sm.State
+	if testValidatorsChange {
+		testConfig := ResetConfig(fmt.Sprintf("%s_%v_m", t.Name(), mode))
+		defer os.RemoveAll(testConfig.RootDir)
+		stateDB = dbm.NewMemDB()
+		genisisState = sim.GenesisState
+		config = sim.Config
+		chain = sim.Chain
+		commits = sim.Commits
+		store = newMockBlockStore(config, genisisState.ConsensusParams)
+	} else { //test single node
+		testConfig := ResetConfig(fmt.Sprintf("%s_%v_s", t.Name(), mode))
+		defer os.RemoveAll(testConfig.RootDir)
+		walBody, err := WALWithNBlocks(t, numBlocks)
+		require.NoError(t, err)
+		walFile := tempWALWithData(walBody)
+		config.Consensus.SetWalFile(walFile)
 
-	walBody, err := WALWithNBlocks(NUM_BLOCKS)
-	if err != nil {
-		t.Fatal(err)
+		privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+
+		wal, err := NewWAL(walFile)
+		require.NoError(t, err)
+		wal.SetLogger(log.TestingLogger())
+		err = wal.Start()
+		require.NoError(t, err)
+		defer wal.Stop()
+
+		chain, commits, err = makeBlockchainFromWAL(wal)
+		require.NoError(t, err)
+		stateDB, genisisState, store = stateAndStore(config, privVal.GetPubKey(), kvstore.ProtocolVersion)
 	}
-	walFile := tempWALWithData(walBody)
-	config.Consensus.SetWalFile(walFile)
-
-	privVal := privval.LoadFilePV(config.PrivValidatorFile())
-
-	wal, err := NewWAL(walFile)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wal.SetLogger(log.TestingLogger())
-	if err := wal.Start(); err != nil {
-		t.Fatal(err)
-	}
-	defer wal.Stop()
-
-	chain, commits, err := makeBlockchainFromWAL(wal)
-	if err != nil {
-		t.Fatalf(err.Error())
-	}
-
-	stateDB, state, store := stateAndStore(config, privVal.GetPubKey(), kvstore.ProtocolVersion)
 	store.chain = chain
 	store.commits = commits
 
+	state := genisisState.Copy()
 	// run the chain through state.ApplyBlock to build up the tendermint state
-	state = buildTMStateFromChain(config, stateDB, state, chain, mode)
+	state = buildTMStateFromChain(config, stateDB, state, chain, nBlocks, mode)
 	latestAppHash := state.AppHash
 
 	// make a new client creator
-	kvstoreApp := kvstore.NewPersistentKVStoreApplication(path.Join(config.DBDir(), "2"))
+	kvstoreApp := kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_a", nBlocks, mode)))
+
 	clientCreator2 := proxy.NewLocalClientCreator(kvstoreApp)
 	if nBlocks > 0 {
 		// run nBlocks against a new client to build up the app state.
 		// use a throwaway tendermint state
 		proxyApp := proxy.NewAppConns(clientCreator2)
-		stateDB, state, _ := stateAndStore(config, privVal.GetPubKey(), kvstore.ProtocolVersion)
-		buildAppStateFromChain(proxyApp, stateDB, state, chain, nBlocks, mode)
+		stateDB1 := dbm.NewMemDB()
+		sm.SaveState(stateDB1, genisisState)
+		buildAppStateFromChain(proxyApp, stateDB1, genisisState, chain, nBlocks, mode)
 	}
 
 	// now start the app using the handshake - it should sync
@@ -380,8 +667,8 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
 	}
 
-	expectedBlocksToSync := NUM_BLOCKS - nBlocks
-	if nBlocks == NUM_BLOCKS && mode > 0 {
+	expectedBlocksToSync := numBlocks - nBlocks
+	if nBlocks == numBlocks && mode > 0 {
 		expectedBlocksToSync++
 	} else if nBlocks > 0 && mode == 1 {
 		expectedBlocksToSync++
@@ -396,7 +683,7 @@ func applyBlock(stateDB dbm.DB, st sm.State, blk *types.Block, proxyApp proxy.Ap
 	testPartSize := types.BlockPartSizeBytes
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 
-	blkID := types.BlockID{blk.Hash(), blk.MakePartSet(testPartSize).Header()}
+	blkID := types.BlockID{Hash: blk.Hash(), PartsHeader: blk.MakePartSet(testPartSize).Header()}
 	newState, err := blockExec.ApplyBlock(st, blkID, blk)
 	if err != nil {
 		panic(err)
@@ -412,12 +699,14 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB,
 	}
 	defer proxyApp.Stop()
 
+	state.Version.Consensus.App = kvstore.ProtocolVersion //simulate handshake, receive app version
 	validators := types.TM2PB.ValidatorUpdates(state.Validators)
 	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{
 		Validators: validators,
 	}); err != nil {
 		panic(err)
 	}
+	sm.SaveState(stateDB, state) //save height 1's validatorsInfo
 
 	switch mode {
 	case 0:
@@ -440,21 +729,23 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB,
 
 }
 
-func buildTMStateFromChain(config *cfg.Config, stateDB dbm.DB, state sm.State, chain []*types.Block, mode uint) sm.State {
+func buildTMStateFromChain(config *cfg.Config, stateDB dbm.DB, state sm.State, chain []*types.Block, nBlocks int, mode uint) sm.State {
 	// run the whole chain against this client to build up the tendermint state
-	clientCreator := proxy.NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(path.Join(config.DBDir(), "1")))
+	clientCreator := proxy.NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode))))
 	proxyApp := proxy.NewAppConns(clientCreator)
 	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
 	defer proxyApp.Stop()
 
+	state.Version.Consensus.App = kvstore.ProtocolVersion //simulate handshake, receive app version
 	validators := types.TM2PB.ValidatorUpdates(state.Validators)
 	if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{
 		Validators: validators,
 	}); err != nil {
 		panic(err)
 	}
+	sm.SaveState(stateDB, state) //save height 1's validatorsInfo
 
 	switch mode {
 	case 0:
@@ -478,28 +769,162 @@ func buildTMStateFromChain(config *cfg.Config, stateDB dbm.DB, state sm.State, c
 	return state
 }
 
+func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
+	// 1. Initialize tendermint and commit 3 blocks with the following app hashes:
+	//		- 0x01
+	//		- 0x02
+	//		- 0x03
+	config := ResetConfig("handshake_test_")
+	defer os.RemoveAll(config.RootDir)
+	privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+	const appVersion = 0x0
+	stateDB, state, store := stateAndStore(config, privVal.GetPubKey(), appVersion)
+	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	state.LastValidators = state.Validators.Copy()
+	// mode = 0 for committing all the blocks
+	blocks := makeBlocks(3, &state, privVal)
+	store.chain = blocks
+
+	// 2. Tendermint must panic if app returns wrong hash for the first block
+	//		- RANDOM HASH
+	//		- 0x02
+	//		- 0x03
+	{
+		app := &badApp{numBlocks: 3, allHashesAreWrong: true}
+		clientCreator := proxy.NewLocalClientCreator(app)
+		proxyApp := proxy.NewAppConns(clientCreator)
+		err := proxyApp.Start()
+		require.NoError(t, err)
+		defer proxyApp.Stop()
+
+		assert.Panics(t, func() {
+			h := NewHandshaker(stateDB, state, store, genDoc)
+			h.Handshake(proxyApp)
+		})
+	}
+
+	// 3. Tendermint must panic if app returns wrong hash for the last block
+	//		- 0x01
+	//		- 0x02
+	//		- RANDOM HASH
+	{
+		app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true}
+		clientCreator := proxy.NewLocalClientCreator(app)
+		proxyApp := proxy.NewAppConns(clientCreator)
+		err := proxyApp.Start()
+		require.NoError(t, err)
+		defer proxyApp.Stop()
+
+		assert.Panics(t, func() {
+			h := NewHandshaker(stateDB, state, store, genDoc)
+			h.Handshake(proxyApp)
+		})
+	}
+}
+
+func makeBlocks(n int, state *sm.State, privVal types.PrivValidator) []*types.Block {
+	blocks := make([]*types.Block, 0)
+
+	var (
+		prevBlock     *types.Block
+		prevBlockMeta *types.BlockMeta
+	)
+
+	appHeight := byte(0x01)
+	for i := 0; i < n; i++ {
+		height := int64(i + 1)
+
+		block, parts := makeBlock(*state, prevBlock, prevBlockMeta, privVal, height)
+		blocks = append(blocks, block)
+
+		prevBlock = block
+		prevBlockMeta = types.NewBlockMeta(block, parts)
+
+		// update state
+		state.AppHash = []byte{appHeight}
+		appHeight++
+		state.LastBlockHeight = height
+	}
+
+	return blocks
+}
+
+func makeVote(header *types.Header, blockID types.BlockID, valset *types.ValidatorSet, privVal types.PrivValidator) *types.Vote {
+	addr := privVal.GetPubKey().Address()
+	idx, _ := valset.GetByAddress(addr)
+	vote := &types.Vote{
+		ValidatorAddress: addr,
+		ValidatorIndex:   idx,
+		Height:           header.Height,
+		Round:            1,
+		Timestamp:        tmtime.Now(),
+		Type:             types.PrecommitType,
+		BlockID:          blockID,
+	}
+
+	privVal.SignVote(header.ChainID, vote)
+
+	return vote
+}
+
+func makeBlock(state sm.State, lastBlock *types.Block, lastBlockMeta *types.BlockMeta,
+	privVal types.PrivValidator, height int64) (*types.Block, *types.PartSet) {
+
+	lastCommit := types.NewCommit(types.BlockID{}, nil)
+	if height > 1 {
+		vote := makeVote(&lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVal).CommitSig()
+		lastCommit = types.NewCommit(lastBlockMeta.BlockID, []*types.CommitSig{vote})
+	}
+
+	return state.MakeBlock(height, []types.Tx{}, lastCommit, nil, state.Validators.GetProposer().Address)
+}
+
+type badApp struct {
+	abci.BaseApplication
+	numBlocks           byte
+	height              byte
+	allHashesAreWrong   bool
+	onlyLastHashIsWrong bool
+}
+
+func (app *badApp) Commit() abci.ResponseCommit {
+	app.height++
+	if app.onlyLastHashIsWrong {
+		if app.height == app.numBlocks {
+			return abci.ResponseCommit{Data: cmn.RandBytes(8)}
+		}
+		return abci.ResponseCommit{Data: []byte{app.height}}
+	} else if app.allHashesAreWrong {
+		return abci.ResponseCommit{Data: cmn.RandBytes(8)}
+	}
+
+	panic("either allHashesAreWrong or onlyLastHashIsWrong must be set")
+}
+
 //--------------------------
 // utils for making blocks
 
 func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
+	var height int64
+
 	// Search for height marker
-	gr, found, err := wal.SearchForEndHeight(0, &WALSearchOptions{})
+	gr, found, err := wal.SearchForEndHeight(height, &WALSearchOptions{})
 	if err != nil {
 		return nil, nil, err
 	}
 	if !found {
-		return nil, nil, fmt.Errorf("WAL does not contain height %d.", 1)
+		return nil, nil, fmt.Errorf("WAL does not contain height %d", height)
 	}
 	defer gr.Close() // nolint: errcheck
 
 	// log.Notice("Build a blockchain by reading from the WAL")
 
-	var blocks []*types.Block
-	var commits []*types.Commit
-
-	var thisBlockParts *types.PartSet
-	var thisBlockCommit *types.Commit
-	var height int64
+	var (
+		blocks          []*types.Block
+		commits         []*types.Commit
+		thisBlockParts  *types.PartSet
+		thisBlockCommit *types.Commit
+	)
 
 	dec := NewWALDecoder(gr)
 	for {
@@ -544,10 +969,8 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 			}
 		case *types.Vote:
 			if p.Type == types.PrecommitType {
-				thisBlockCommit = &types.Commit{
-					BlockID:    p.BlockID,
-					Precommits: []*types.Vote{p},
-				}
+				commitSigs := []*types.CommitSig{p.CommitSig()}
+				thisBlockCommit = types.NewCommit(p.BlockID, commitSigs)
 			}
 		}
 	}
@@ -593,7 +1016,8 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey, appVersion version.
 	stateDB := dbm.NewMemDB()
 	state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile())
 	state.Version.Consensus.App = appVersion
-	store := NewMockBlockStore(config, state.ConsensusParams)
+	store := newMockBlockStore(config, state.ConsensusParams)
+	sm.SaveState(stateDB, state)
 	return stateDB, state, store
 }
 
@@ -608,7 +1032,7 @@ type mockBlockStore struct {
 }
 
 // TODO: NewBlockStore(db.NewMemDB) ...
-func NewMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore {
+func newMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore {
 	return &mockBlockStore{config, params, nil, nil}
 }
 
@@ -617,7 +1041,7 @@ func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain
 func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
 	block := bs.chain[height-1]
 	return &types.BlockMeta{
-		BlockID: types.BlockID{block.Hash(), block.MakePartSet(types.BlockPartSizeBytes).Header()},
+		BlockID: types.BlockID{Hash: block.Hash(), PartsHeader: block.MakePartSet(types.BlockPartSizeBytes).Header()},
 		Header:  block.Header,
 	}
 }
@@ -631,16 +1055,18 @@ func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit {
 	return bs.commits[height-1]
 }
 
-//----------------------------------------
+//---------------------------------------
+// Test handshake/init chain
 
-func TestInitChainUpdateValidators(t *testing.T) {
+func TestHandshakeUpdatesValidators(t *testing.T) {
 	val, _ := types.RandValidator(true, 10)
 	vals := types.NewValidatorSet([]*types.Validator{val})
 	app := &initChainApp{vals: types.TM2PB.ValidatorUpdates(vals)}
 	clientCreator := proxy.NewLocalClientCreator(app)
 
-	config := ResetConfig("proxy_test_")
-	privVal := privval.LoadFilePV(config.PrivValidatorFile())
+	config := ResetConfig("handshake_test_")
+	defer os.RemoveAll(config.RootDir)
+	privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
 	stateDB, state, store := stateAndStore(config, privVal.GetPubKey(), 0x0)
 
 	oldValAddr := state.Validators.Validators[0].Address
@@ -666,12 +1092,6 @@ func TestInitChainUpdateValidators(t *testing.T) {
 	assert.Equal(t, newValAddr, expectValAddr)
 }
 
-func newInitChainApp(vals []abci.ValidatorUpdate) *initChainApp {
-	return &initChainApp{
-		vals: vals,
-	}
-}
-
 // returns the vals on InitChain
 type initChainApp struct {
 	abci.BaseApplication

consensus/state.go

@@ -2,13 +2,14 @@ package consensus
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"reflect"
 	"runtime/debug"
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/fail"
 	"github.com/tendermint/tendermint/libs/log"
@@ -22,13 +23,6 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-//-----------------------------------------------------------------------------
-// Config
-
-const (
-	proposalHeartbeatIntervalSeconds = 2
-)
-
 //-----------------------------------------------------------------------------
 // Errors
 
@@ -63,6 +57,16 @@ func (ti *timeoutInfo) String() string {
 	return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step)
 }
 
+// interface to the mempool
+type txNotifier interface {
+	TxsAvailable() <-chan struct{}
+}
+
+// interface to the evidence pool
+type evidencePool interface {
+	AddEvidence(types.Evidence) error
+}
+
 // ConsensusState handles execution of the consensus algorithm.
 // It processes votes and proposals, and upon reaching agreement,
 // commits blocks to the chain and executes them against the application.
@@ -74,17 +78,23 @@ type ConsensusState struct {
 	config        *cfg.ConsensusConfig
 	privValidator types.PrivValidator // for signing votes
 
-	// services for creating and executing blocks
-	blockExec  *sm.BlockExecutor
+	// store blocks and commits
 	blockStore sm.BlockStore
-	mempool    sm.Mempool
-	evpool     sm.EvidencePool
+
+	// create and execute blocks
+	blockExec *sm.BlockExecutor
+
+	// notify us if txs are available
+	txNotifier txNotifier
+
+	// add evidence to the pool
+	// when it's detected
+	evpool evidencePool
 
 	// internal state
 	mtx sync.RWMutex
 	cstypes.RoundState
-	triggeredTimeoutPrecommit bool
-	state                     sm.State // State until height-1.
+	state sm.State // State until height-1.
 
 	// state changes may be triggered by: msgs from peers,
 	// msgs from ourself, or by timeouts
@@ -118,7 +128,7 @@ type ConsensusState struct {
 	done chan struct{}
 
 	// synchronous pubsub between consensus state and reactor.
-	// state only emits EventNewRoundStep, EventVote and EventProposalHeartbeat
+	// state only emits EventNewRoundStep and EventVote
 	evsw tmevents.EventSwitch
 
 	// for reporting metrics
@@ -134,15 +144,15 @@ func NewConsensusState(
 	state sm.State,
 	blockExec *sm.BlockExecutor,
 	blockStore sm.BlockStore,
-	mempool sm.Mempool,
-	evpool sm.EvidencePool,
+	txNotifier txNotifier,
+	evpool evidencePool,
 	options ...StateOption,
 ) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
 		blockExec:        blockExec,
 		blockStore:       blockStore,
-		mempool:          mempool,
+		txNotifier:       txNotifier,
 		peerMsgQueue:     make(chan msgInfo, msgQueueSize),
 		internalMsgQueue: make(chan msgInfo, msgQueueSize),
 		timeoutTicker:    NewTimeoutTicker(),
@@ -207,18 +217,16 @@ func (cs *ConsensusState) GetState() sm.State {
 // GetLastHeight returns the last height committed.
 // If there were no blocks, returns 0.
 func (cs *ConsensusState) GetLastHeight() int64 {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
-
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	return cs.RoundState.Height - 1
 }
 
 // GetRoundState returns a shallow copy of the internal consensus state.
 func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
 	cs.mtx.RLock()
-	defer cs.mtx.RUnlock()
-
 	rs := cs.RoundState // copy
+	cs.mtx.RUnlock()
 	return &rs
 }
 
@@ -226,7 +234,6 @@ func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
 func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) {
 	cs.mtx.RLock()
 	defer cs.mtx.RUnlock()
-
 	return cdc.MarshalJSON(cs.RoundState)
 }
 
@@ -234,7 +241,6 @@ func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) {
 func (cs *ConsensusState) GetRoundStateSimpleJSON() ([]byte, error) {
 	cs.mtx.RLock()
 	defer cs.mtx.RUnlock()
-
 	return cdc.MarshalJSON(cs.RoundState.RoundStateSimple())
 }
 
@@ -248,15 +254,15 @@ func (cs *ConsensusState) GetValidators() (int64, []*types.Validator) {
 // SetPrivValidator sets the private validator account for signing votes.
 func (cs *ConsensusState) SetPrivValidator(priv types.PrivValidator) {
 	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
 	cs.privValidator = priv
+	cs.mtx.Unlock()
 }
 
 // SetTimeoutTicker sets the local timer. It may be useful to overwrite for testing.
 func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) {
 	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
 	cs.timeoutTicker = timeoutTicker
+	cs.mtx.Unlock()
 }
 
 // LoadCommit loads the commit for a given height.
@@ -301,6 +307,23 @@ func (cs *ConsensusState) OnStart() error {
 	// reload from consensus log to catchup
 	if cs.doWALCatchup {
 		if err := cs.catchupReplay(cs.Height); err != nil {
+			// don't try to recover from data corruption error
+			if IsDataCorruptionError(err) {
+				cs.Logger.Error("Encountered corrupt WAL file", "err", err.Error())
+				cs.Logger.Error("Please repair the WAL file before restarting")
+				fmt.Println(`You can attempt to repair the WAL as follows:
+
+----
+WALFILE=~/.tendermint/data/cs.wal/wal
+cp $WALFILE ${WALFILE}.bak # backup the file
+go run scripts/wal2json/main.go $WALFILE > wal.json # this will panic, but can be ignored
+rm $WALFILE # remove the corrupt file
+go run scripts/json2wal/main.go wal.json $WALFILE # rebuild the file without corruption
+----`)
+
+				return err
+			}
+
 			cs.Logger.Error("Error on catchup replay. Proceeding to start ConsensusState anyway", "err", err.Error())
 			// NOTE: if we ever do return an error here,
 			// make sure to stop the timeoutTicker
@@ -328,10 +351,11 @@ func (cs *ConsensusState) startRoutines(maxSteps int) {
 	go cs.receiveRoutine(maxSteps)
 }
 
-// OnStop implements cmn.Service. It stops all routines and waits for the WAL to finish.
+// OnStop implements cmn.Service.
 func (cs *ConsensusState) OnStop() {
 	cs.evsw.Stop()
 	cs.timeoutTicker.Stop()
+	// WAL is stopped in receiveRoutine.
 }
 
 // Wait waits for the the main routine to return.
@@ -430,7 +454,7 @@ func (cs *ConsensusState) updateRoundStep(round int, step cstypes.RoundStepType)
 // enterNewRound(height, 0) at cs.StartTime.
 func (cs *ConsensusState) scheduleRound0(rs *cstypes.RoundState) {
 	//cs.Logger.Info("scheduleRound0", "now", tmtime.Now(), "startTime", cs.StartTime)
-	sleepDuration := rs.StartTime.Sub(tmtime.Now()) // nolint: gotype, gosimple
+	sleepDuration := rs.StartTime.Sub(tmtime.Now())
 	cs.scheduleTimeout(sleepDuration, rs.Height, 0, cstypes.RoundStepNewHeight)
 }
 
@@ -460,18 +484,9 @@ func (cs *ConsensusState) reconstructLastCommit(state sm.State) {
 		return
 	}
 	seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight)
-	lastPrecommits := types.NewVoteSet(state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.PrecommitType, state.LastValidators)
-	for _, precommit := range seenCommit.Precommits {
-		if precommit == nil {
-			continue
-		}
-		added, err := lastPrecommits.AddVote(precommit)
-		if !added || err != nil {
-			cmn.PanicCrisis(fmt.Sprintf("Failed to reconstruct LastCommit: %v", err))
-		}
-	}
+	lastPrecommits := types.CommitToVoteSet(state.ChainID, seenCommit, state.LastValidators)
 	if !lastPrecommits.HasTwoThirdsMajority() {
-		cmn.PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
+		panic("Failed to reconstruct LastCommit: Does not have +2/3 maj")
 	}
 	cs.LastCommit = lastPrecommits
 }
@@ -480,20 +495,20 @@ func (cs *ConsensusState) reconstructLastCommit(state sm.State) {
 // The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight.
 func (cs *ConsensusState) updateToState(state sm.State) {
 	if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
-		cmn.PanicSanity(fmt.Sprintf("updateToState() expected state height of %v but found %v",
+		panic(fmt.Sprintf("updateToState() expected state height of %v but found %v",
 			cs.Height, state.LastBlockHeight))
 	}
 	if !cs.state.IsEmpty() && cs.state.LastBlockHeight+1 != cs.Height {
 		// This might happen when someone else is mutating cs.state.
 		// Someone forgot to pass in state.Copy() somewhere?!
-		cmn.PanicSanity(fmt.Sprintf("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
+		panic(fmt.Sprintf("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
 			cs.state.LastBlockHeight+1, cs.Height))
 	}
 
 	// If state isn't further out than cs.state, just ignore.
 	// This happens when SwitchToConsensus() is called in the reactor.
 	// We don't want to reset e.g. the Votes, but we still want to
-	// signal the new round step, because other services (eg. mempool)
+	// signal the new round step, because other services (eg. txNotifier)
 	// depend on having an up-to-date peer state!
 	if !cs.state.IsEmpty() && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
 		cs.Logger.Info("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
@@ -506,7 +521,7 @@ func (cs *ConsensusState) updateToState(state sm.State) {
 	lastPrecommits := (*types.VoteSet)(nil)
 	if cs.CommitRound > -1 && cs.Votes != nil {
 		if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
-			cmn.PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
+			panic("updateToState(state) called but last Precommit round didn't have +2/3")
 		}
 		lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
 	}
@@ -542,6 +557,7 @@ func (cs *ConsensusState) updateToState(state sm.State) {
 	cs.CommitRound = -1
 	cs.LastCommit = lastPrecommits
 	cs.LastValidators = state.LastValidators
+	cs.TriggeredTimeoutPrecommit = false
 
 	cs.state = state
 
@@ -608,7 +624,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 		var mi msgInfo
 
 		select {
-		case <-cs.mempool.TxsAvailable():
+		case <-cs.txNotifier.TxsAvailable():
 			cs.handleTxsAvailable()
 		case mi = <-cs.peerMsgQueue:
 			cs.wal.Write(mi)
@@ -617,6 +633,15 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 			cs.handleMsg(mi)
 		case mi = <-cs.internalMsgQueue:
 			cs.wal.WriteSync(mi) // NOTE: fsync
+
+			if _, ok := mi.Msg.(*VoteMessage); ok {
+				// we actually want to simulate failing during
+				// the previous WriteSync, but this isn't easy to do.
+				// Equivalent would be to fail here and manually remove
+				// some bytes from the end of the wal.
+				fail.Fail() // XXX
+			}
+
 			// handles proposals, block parts, votes
 			cs.handleMsg(mi)
 		case ti := <-cs.timeoutTicker.Chan(): // tockChan:
@@ -636,7 +661,10 @@ func (cs *ConsensusState) handleMsg(mi msgInfo) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 
-	var err error
+	var (
+		added bool
+		err   error
+	)
 	msg, peerID := mi.Msg, mi.PeerID
 	switch msg := msg.(type) {
 	case *ProposalMessage:
@@ -645,7 +673,7 @@ func (cs *ConsensusState) handleMsg(mi msgInfo) {
 		err = cs.setProposal(msg.Proposal)
 	case *BlockPartMessage:
 		// if the proposal is complete, we'll enterPrevote or tryFinalizeCommit
-		added, err := cs.addProposalBlockPart(msg, peerID)
+		added, err = cs.addProposalBlockPart(msg, peerID)
 		if added {
 			cs.statsMsgQueue <- mi
 		}
@@ -657,7 +685,7 @@ func (cs *ConsensusState) handleMsg(mi msgInfo) {
 	case *VoteMessage:
 		// attempt to add the vote and dupeout the validator if its a duplicate signature
 		// if the vote gives us a 2/3-any or 2/3-one, we transition
-		added, err := cs.tryAddVote(msg.Vote, peerID)
+		added, err = cs.tryAddVote(msg.Vote, peerID)
 		if added {
 			cs.statsMsgQueue <- mi
 		}
@@ -677,10 +705,15 @@ func (cs *ConsensusState) handleMsg(mi msgInfo) {
 		// the peer is sending us CatchupCommit precommits.
 		// We could make note of this and help filter in broadcastHasVoteMessage().
 	default:
-		cs.Logger.Error("Unknown msg type", reflect.TypeOf(msg))
+		cs.Logger.Error("Unknown msg type", "type", reflect.TypeOf(msg))
+		return
 	}
+
 	if err != nil {
-		cs.Logger.Error("Error with msg", "height", cs.Height, "round", cs.Round, "type", reflect.TypeOf(msg), "peer", peerID, "err", err, "msg", msg)
+		// Causes TestReactorValidatorSetChanges to timeout
+		// https://github.com/tendermint/tendermint/issues/3406
+		// cs.Logger.Error("Error with msg", "height", cs.Height, "round", cs.Round,
+		// 	"peer", peerID, "err", err, "msg", msg)
 	}
 }
 
@@ -724,6 +757,7 @@ func (cs *ConsensusState) handleTxsAvailable() {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	// we only need to do this for round 0
+	cs.enterNewRound(cs.Height, 0)
 	cs.enterPropose(cs.Height, 0)
 }
 
@@ -755,7 +789,7 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) {
 	validators := cs.Validators
 	if cs.Round < round {
 		validators = validators.Copy()
-		validators.IncrementAccum(round - cs.Round)
+		validators.IncrementProposerPriority(round - cs.Round)
 	}
 
 	// Setup new round
@@ -774,9 +808,9 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) {
 		cs.ProposalBlockParts = nil
 	}
 	cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
-	cs.triggeredTimeoutPrecommit = false
+	cs.TriggeredTimeoutPrecommit = false
 
-	cs.eventBus.PublishEventNewRound(cs.RoundStateEvent())
+	cs.eventBus.PublishEventNewRound(cs.NewRoundEvent())
 	cs.metrics.Rounds.Set(float64(round))
 
 	// Wait for txs to be available in the mempool
@@ -788,7 +822,6 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) {
 			cs.scheduleTimeout(cs.config.CreateEmptyBlocksInterval, height, round,
 				cstypes.RoundStepNewRound)
 		}
-		go cs.proposalHeartbeat(height, round)
 	} else {
 		cs.enterPropose(height, round)
 	}
@@ -805,32 +838,6 @@ func (cs *ConsensusState) needProofBlock(height int64) bool {
 	return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash)
 }
 
-func (cs *ConsensusState) proposalHeartbeat(height int64, round int) {
-	counter := 0
-	addr := cs.privValidator.GetAddress()
-	valIndex, _ := cs.Validators.GetByAddress(addr)
-	chainID := cs.state.ChainID
-	for {
-		rs := cs.GetRoundState()
-		// if we've already moved on, no need to send more heartbeats
-		if rs.Step > cstypes.RoundStepNewRound || rs.Round > round || rs.Height > height {
-			return
-		}
-		heartbeat := &types.Heartbeat{
-			Height:           rs.Height,
-			Round:            rs.Round,
-			Sequence:         counter,
-			ValidatorAddress: addr,
-			ValidatorIndex:   valIndex,
-		}
-		cs.privValidator.SignHeartbeat(chainID, heartbeat)
-		cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat})
-		cs.evsw.FireEvent(types.EventProposalHeartbeat, heartbeat)
-		counter++
-		time.Sleep(proposalHeartbeatIntervalSeconds * time.Second)
-	}
-}
-
 // Enter (CreateEmptyBlocks): from enterNewRound(height,round)
 // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ): after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval
 // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool
@@ -866,13 +873,14 @@ func (cs *ConsensusState) enterPropose(height int64, round int) {
 	}
 
 	// if not a validator, we're done
-	if !cs.Validators.HasAddress(cs.privValidator.GetAddress()) {
-		logger.Debug("This node is not a validator", "addr", cs.privValidator.GetAddress(), "vals", cs.Validators)
+	address := cs.privValidator.GetPubKey().Address()
+	if !cs.Validators.HasAddress(address) {
+		logger.Debug("This node is not a validator", "addr", address, "vals", cs.Validators)
 		return
 	}
 	logger.Debug("This node is a validator")
 
-	if cs.isProposer() {
+	if cs.isProposer(address) {
 		logger.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator)
 		cs.decideProposal(height, round)
 	} else {
@@ -880,8 +888,8 @@ func (cs *ConsensusState) enterPropose(height int64, round int) {
 	}
 }
 
-func (cs *ConsensusState) isProposer() bool {
-	return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress())
+func (cs *ConsensusState) isProposer(address []byte) bool {
+	return bytes.Equal(cs.Validators.GetProposer().Address, address)
 }
 
 func (cs *ConsensusState) defaultDecideProposal(height int64, round int) {
@@ -900,8 +908,11 @@ func (cs *ConsensusState) defaultDecideProposal(height int64, round int) {
 		}
 	}
 
+	// Flush the WAL. Otherwise, we may not recompute the same proposal to sign, and the privValidator will refuse to sign anything.
+	cs.wal.FlushAndSync()
+
 	// Make proposal
-	propBlockId := types.BlockID{block.Hash(), blockParts.Header()}
+	propBlockId := types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()}
 	proposal := types.NewProposal(height, round, cs.ValidRound, propBlockId)
 	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal); err == nil {
 
@@ -946,7 +957,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 	if cs.Height == 1 {
 		// We're creating a proposal for the first block.
 		// The commit is empty, but not nil.
-		commit = &types.Commit{}
+		commit = types.NewCommit(types.BlockID{}, nil)
 	} else if cs.LastCommit.HasTwoThirdsMajority() {
 		// Make the commit from LastCommit
 		commit = cs.LastCommit.MakeCommit()
@@ -956,20 +967,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 		return
 	}
 
-	maxBytes := cs.state.ConsensusParams.BlockSize.MaxBytes
-	maxGas := cs.state.ConsensusParams.BlockSize.MaxGas
-	// bound evidence to 1/10th of the block
-	evidence := cs.evpool.PendingEvidence(types.MaxEvidenceBytesPerBlock(maxBytes))
-	// Mempool validated transactions
-	txs := cs.mempool.ReapMaxBytesMaxGas(types.MaxDataBytes(
-		maxBytes,
-		cs.state.Validators.Size(),
-		len(evidence),
-	), maxGas)
-	proposerAddr := cs.privValidator.GetAddress()
-	block, parts := cs.state.MakeBlock(cs.Height, txs, commit, evidence, proposerAddr)
-
-	return block, parts
+	proposerAddr := cs.privValidator.GetPubKey().Address()
+	return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr)
 }
 
 // Enter: `timeoutPropose` after entering Propose.
@@ -1039,7 +1038,7 @@ func (cs *ConsensusState) enterPrevoteWait(height int64, round int) {
 		return
 	}
 	if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
-		cmn.PanicSanity(fmt.Sprintf("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
+		panic(fmt.Sprintf("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
 	}
 	logger.Info(fmt.Sprintf("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
@@ -1095,7 +1094,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) {
 	// the latest POLRound should be this round.
 	polRound, _ := cs.Votes.POLInfo()
 	if polRound < round {
-		cmn.PanicSanity(fmt.Sprintf("This POLRound should be %v but got %v", round, polRound))
+		panic(fmt.Sprintf("This POLRound should be %v but got %v", round, polRound))
 	}
 
 	// +2/3 prevoted nil. Unlock and precommit nil.
@@ -1129,7 +1128,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) {
 		logger.Info("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash)
 		// Validate the block.
 		if err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock); err != nil {
-			cmn.PanicConsensus(fmt.Sprintf("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
+			panic(fmt.Sprintf("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
 		}
 		cs.LockedRound = round
 		cs.LockedBlock = cs.ProposalBlock
@@ -1158,22 +1157,22 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) {
 func (cs *ConsensusState) enterPrecommitWait(height int64, round int) {
 	logger := cs.Logger.With("height", height, "round", round)
 
-	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.triggeredTimeoutPrecommit) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) {
 		logger.Debug(
 			fmt.Sprintf(
 				"enterPrecommitWait(%v/%v): Invalid args. "+
-					"Current state is Height/Round: %v/%v/, triggeredTimeoutPrecommit:%v",
-				height, round, cs.Height, cs.Round, cs.triggeredTimeoutPrecommit))
+					"Current state is Height/Round: %v/%v/, TriggeredTimeoutPrecommit:%v",
+				height, round, cs.Height, cs.Round, cs.TriggeredTimeoutPrecommit))
 		return
 	}
 	if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
-		cmn.PanicSanity(fmt.Sprintf("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
+		panic(fmt.Sprintf("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
 	}
 	logger.Info(fmt.Sprintf("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
 		// Done enterPrecommitWait:
-		cs.triggeredTimeoutPrecommit = true
+		cs.TriggeredTimeoutPrecommit = true
 		cs.newStep()
 	}()
 
@@ -1206,7 +1205,7 @@ func (cs *ConsensusState) enterCommit(height int64, commitRound int) {
 
 	blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
 	if !ok {
-		cmn.PanicSanity("RunActionCommit() expects +2/3 precommits")
+		panic("RunActionCommit() expects +2/3 precommits")
 	}
 
 	// The Locked* fields no longer matter.
@@ -1239,7 +1238,7 @@ func (cs *ConsensusState) tryFinalizeCommit(height int64) {
 	logger := cs.Logger.With("height", height)
 
 	if cs.Height != height {
-		cmn.PanicSanity(fmt.Sprintf("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
+		panic(fmt.Sprintf("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
 	}
 
 	blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
@@ -1269,16 +1268,16 @@ func (cs *ConsensusState) finalizeCommit(height int64) {
 	block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
 
 	if !ok {
-		cmn.PanicSanity(fmt.Sprintf("Cannot finalizeCommit, commit does not have two thirds majority"))
+		panic(fmt.Sprintf("Cannot finalizeCommit, commit does not have two thirds majority"))
 	}
 	if !blockParts.HasHeader(blockID.PartsHeader) {
-		cmn.PanicSanity(fmt.Sprintf("Expected ProposalBlockParts header to be commit header"))
+		panic(fmt.Sprintf("Expected ProposalBlockParts header to be commit header"))
 	}
 	if !block.HashesTo(blockID.Hash) {
-		cmn.PanicSanity(fmt.Sprintf("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
+		panic(fmt.Sprintf("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
 	}
 	if err := cs.blockExec.ValidateBlock(cs.state, block); err != nil {
-		cmn.PanicConsensus(fmt.Sprintf("+2/3 committed an invalid block: %v", err))
+		panic(fmt.Sprintf("+2/3 committed an invalid block: %v", err))
 	}
 
 	cs.Logger.Info(fmt.Sprintf("Finalizing commit of block with %d txs", block.NumTxs),
@@ -1324,7 +1323,7 @@ func (cs *ConsensusState) finalizeCommit(height int64) {
 	// Execute and commit the block, update and save the state, and update the mempool.
 	// NOTE The block.AppHash wont reflect these txs until the next block.
 	var err error
-	stateCopy, err = cs.blockExec.ApplyBlock(stateCopy, types.BlockID{block.Hash(), blockParts.Header()}, block)
+	stateCopy, err = cs.blockExec.ApplyBlock(stateCopy, types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()}, block)
 	if err != nil {
 		cs.Logger.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "err", err)
 		err := cmn.Kill()
@@ -1360,7 +1359,7 @@ func (cs *ConsensusState) recordMetrics(height int64, block *types.Block) {
 	missingValidators := 0
 	missingValidatorsPower := int64(0)
 	for i, val := range cs.Validators.Validators {
-		var vote *types.Vote
+		var vote *types.CommitSig
 		if i < len(block.LastCommit.Precommits) {
 			vote = block.LastCommit.Precommits[i]
 		}
@@ -1408,9 +1407,9 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
 		return nil
 	}
 
-	// Verify POLRound, which must be -1 or between 0 and proposal.Round exclusive.
-	if proposal.POLRound != -1 &&
-		(proposal.POLRound < 0 || proposal.Round <= proposal.POLRound) {
+	// Verify POLRound, which must be -1 or in range [0, proposal.Round).
+	if proposal.POLRound < -1 ||
+		(proposal.POLRound >= 0 && proposal.POLRound >= proposal.Round) {
 		return ErrInvalidProposalPOLRound
 	}
 
@@ -1459,14 +1458,14 @@ func (cs *ConsensusState) addProposalBlockPart(msg *BlockPartMessage, peerID p2p
 		_, err = cdc.UnmarshalBinaryLengthPrefixedReader(
 			cs.ProposalBlockParts.GetReader(),
 			&cs.ProposalBlock,
-			int64(cs.state.ConsensusParams.BlockSize.MaxBytes),
+			int64(cs.state.ConsensusParams.Block.MaxBytes),
 		)
 		if err != nil {
 			return added, err
 		}
 		// NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
 		cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
-		cs.eventBus.PublishEventCompleteProposal(cs.RoundStateEvent())
+		cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent())
 
 		// Update Valid* if we can.
 		prevotes := cs.Votes.Prevotes(cs.Round)
@@ -1511,7 +1510,8 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerID p2p.ID) (bool, err
 		if err == ErrVoteHeightMismatch {
 			return added, err
 		} else if voteErr, ok := err.(*types.ErrVoteConflictingVotes); ok {
-			if bytes.Equal(vote.ValidatorAddress, cs.privValidator.GetAddress()) {
+			addr := cs.privValidator.GetPubKey().Address()
+			if bytes.Equal(vote.ValidatorAddress, addr) {
 				cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
 				return added, err
 			}
@@ -1546,7 +1546,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool,
 		}
 
 		cs.Logger.Info(fmt.Sprintf("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
-		cs.eventBus.PublishEventVote(types.EventDataVote{vote})
+		cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote})
 		cs.evsw.FireEvent(types.EventVote, vote)
 
 		// if we can skip timeoutCommit and have all the votes now,
@@ -1563,7 +1563,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool,
 	// Not necessarily a bad peer, but not favourable behaviour.
 	if vote.Height != cs.Height {
 		err = ErrVoteHeightMismatch
-		cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "err", err)
+		cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "peerID", peerID)
 		return
 	}
 
@@ -1574,7 +1574,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool,
 		return
 	}
 
-	cs.eventBus.PublishEventVote(types.EventDataVote{vote})
+	cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote})
 	cs.evsw.FireEvent(types.EventVote, vote)
 
 	switch vote.Type {
@@ -1676,7 +1676,10 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool,
 }
 
 func (cs *ConsensusState) signVote(type_ types.SignedMsgType, hash []byte, header types.PartSetHeader) (*types.Vote, error) {
-	addr := cs.privValidator.GetAddress()
+	// Flush the WAL. Otherwise, we may not recompute the same vote to sign, and the privValidator will refuse to sign anything.
+	cs.wal.FlushAndSync()
+
+	addr := cs.privValidator.GetPubKey().Address()
 	valIndex, _ := cs.Validators.GetByAddress(addr)
 
 	vote := &types.Vote{
@@ -1686,7 +1689,7 @@ func (cs *ConsensusState) signVote(type_ types.SignedMsgType, hash []byte, heade
 		Round:            cs.Round,
 		Timestamp:        cs.voteTime(),
 		Type:             type_,
-		BlockID:          types.BlockID{hash, header},
+		BlockID:          types.BlockID{Hash: hash, PartsHeader: header},
 	}
 	err := cs.privValidator.SignVote(cs.state.ChainID, vote)
 	return vote, err
@@ -1697,10 +1700,12 @@ func (cs *ConsensusState) voteTime() time.Time {
 	minVoteTime := now
 	// TODO: We should remove next line in case we don't vote for v in case cs.ProposalBlock == nil,
 	// even if cs.LockedBlock != nil. See https://github.com/tendermint/spec.
+	timeIotaMs := time.Duration(cs.state.ConsensusParams.Block.TimeIotaMs) * time.Millisecond
 	if cs.LockedBlock != nil {
-		minVoteTime = cs.config.MinValidVoteTime(cs.LockedBlock.Time)
+		// See the BFT time spec https://tendermint.com/docs/spec/consensus/bft-time.html
+		minVoteTime = cs.LockedBlock.Time.Add(timeIotaMs)
 	} else if cs.ProposalBlock != nil {
-		minVoteTime = cs.config.MinValidVoteTime(cs.ProposalBlock.Time)
+		minVoteTime = cs.ProposalBlock.Time.Add(timeIotaMs)
 	}
 
 	if now.After(minVoteTime) {
@@ -1712,7 +1717,7 @@ func (cs *ConsensusState) voteTime() time.Time {
 // sign the vote and publish on internalMsgQueue
 func (cs *ConsensusState) signAddVote(type_ types.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote {
 	// if we don't have a key or we're not in the validator set, do nothing
-	if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) {
+	if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetPubKey().Address()) {
 		return nil
 	}
 	vote, err := cs.signVote(type_, hash, header)

consensus/state_test.go

@@ -14,18 +14,10 @@ import (
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
-	p2pdummy "github.com/tendermint/tendermint/p2p/dummy"
+	p2pmock "github.com/tendermint/tendermint/p2p/mock"
 	"github.com/tendermint/tendermint/types"
 )
 
-func init() {
-	config = ResetConfig("consensus_state_test")
-}
-
-func ensureProposeTimeout(timeoutPropose time.Duration) time.Duration {
-	return time.Duration(timeoutPropose.Nanoseconds()*2) * time.Nanosecond
-}
-
 /*
 
 ProposeSuite
@@ -73,7 +65,8 @@ func TestStateProposerSelection0(t *testing.T) {
 
 	// Commit a block and ensure proposer for the next height is correct.
 	prop := cs1.GetRoundState().Validators.GetProposer()
-	if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) {
+	address := cs1.privValidator.GetPubKey().Address()
+	if !bytes.Equal(prop.Address, address) {
 		t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address)
 	}
 
@@ -87,7 +80,8 @@ func TestStateProposerSelection0(t *testing.T) {
 	ensureNewRound(newRoundCh, height+1, 0)
 
 	prop = cs1.GetRoundState().Validators.GetProposer()
-	if !bytes.Equal(prop.Address, vss[1].GetAddress()) {
+	addr := vss[1].GetPubKey().Address()
+	if !bytes.Equal(prop.Address, addr) {
 		panic(fmt.Sprintf("expected proposer to be validator %d. Got %X", 1, prop.Address))
 	}
 }
@@ -110,7 +104,8 @@ func TestStateProposerSelection2(t *testing.T) {
 	// everyone just votes nil. we get a new proposer each round
 	for i := 0; i < len(vss); i++ {
 		prop := cs1.GetRoundState().Validators.GetProposer()
-		correctProposer := vss[(i+round)%len(vss)].GetAddress()
+		addr := vss[(i+round)%len(vss)].GetPubKey().Address()
+		correctProposer := addr
 		if !bytes.Equal(prop.Address, correctProposer) {
 			panic(fmt.Sprintf("expected RoundState.Validators.GetProposer() to be validator %d. Got %X", (i+2)%len(vss), prop.Address))
 		}
@@ -197,9 +192,8 @@ func TestStateBadProposal(t *testing.T) {
 	stateHash[0] = byte((stateHash[0] + 1) % 255)
 	propBlock.AppHash = stateHash
 	propBlockParts := propBlock.MakePartSet(partSize)
-	proposal := types.NewProposal(
-		vs2.Height, round, -1,
-		types.BlockID{propBlock.Hash(), propBlockParts.Header()})
+	blockID := types.BlockID{propBlock.Hash(), propBlockParts.Header()}
+	proposal := types.NewProposal(vs2.Height, round, -1, blockID)
 	if err := vs2.SignProposal(config.ChainID(), proposal); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
@@ -213,7 +207,7 @@ func TestStateBadProposal(t *testing.T) {
 	startTestRound(cs1, height, round)
 
 	// wait for proposal
-	ensureNewProposal(proposalCh, height, round)
+	ensureProposal(proposalCh, height, round, blockID)
 
 	// wait for prevote
 	ensurePrevote(voteCh, height, round)
@@ -245,7 +239,7 @@ func TestStateFullRound1(t *testing.T) {
 	cs.SetEventBus(eventBus)
 	eventBus.Start()
 
-	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
+	voteCh := subscribeUnBuffered(cs.eventBus, types.EventQueryVote)
 	propCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal)
 	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
 
@@ -273,7 +267,7 @@ func TestStateFullRoundNil(t *testing.T) {
 	cs, vss := randConsensusState(1)
 	height, round := cs.Height, cs.Round
 
-	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
+	voteCh := subscribeUnBuffered(cs.eventBus, types.EventQueryVote)
 
 	cs.enterPrevote(height, round)
 	cs.startRoutines(4)
@@ -292,7 +286,7 @@ func TestStateFullRound2(t *testing.T) {
 	vs2 := vss[1]
 	height, round := cs1.Height, cs1.Round
 
-	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	voteCh := subscribeUnBuffered(cs1.eventBus, types.EventQueryVote)
 	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
 
 	// start round and wait for propose and prevote
@@ -336,7 +330,7 @@ func TestStateLockNoPOL(t *testing.T) {
 
 	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
-	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
+	voteCh := subscribeUnBuffered(cs1.eventBus, types.EventQueryVote)
 	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 
@@ -376,7 +370,7 @@ func TestStateLockNoPOL(t *testing.T) {
 
 	// (note we're entering precommit for a second time this round)
 	// but with invalid args. then we enterPrecommitWait, and the timeout to new round
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	///
 
@@ -390,7 +384,7 @@ func TestStateLockNoPOL(t *testing.T) {
 	incrementRound(vs2)
 
 	// now we're on a new round and not the proposer, so wait for timeout
-	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds())
+	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.Propose(round).Nanoseconds())
 
 	rs := cs1.GetRoundState()
 
@@ -409,7 +403,7 @@ func TestStateLockNoPOL(t *testing.T) {
 
 	// now we're going to enter prevote again, but with invalid args
 	// and then prevote wait, which should timeout. then wait for precommit
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Prevote(round).Nanoseconds())
 
 	ensurePrecommit(voteCh, height, round) // precommit
 	// the proposed block should still be locked and our precommit added
@@ -422,7 +416,7 @@ func TestStateLockNoPOL(t *testing.T) {
 
 	// (note we're entering precommit for a second time this round, but with invalid args
 	// then we enterPrecommitWait and timeout into NewRound
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	round = round + 1 // entering new round
 	ensureNewRound(newRoundCh, height, round)
@@ -447,7 +441,7 @@ func TestStateLockNoPOL(t *testing.T) {
 	signAddVotes(cs1, types.PrevoteType, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2)
 	ensurePrevote(voteCh, height, round)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Prevote(round).Nanoseconds())
 	ensurePrecommit(voteCh, height, round) // precommit
 
 	validatePrecommit(t, cs1, round, 0, vss[0], nil, theBlockHash) // precommit nil but be locked on proposal
@@ -455,7 +449,7 @@ func TestStateLockNoPOL(t *testing.T) {
 	signAddVotes(cs1, types.PrecommitType, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height
 	ensurePrecommit(voteCh, height, round)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	cs2, _ := randConsensusState(2) // needed so generated block is different than locked block
 	// before we time out into new round, set next proposal block
@@ -488,7 +482,7 @@ func TestStateLockNoPOL(t *testing.T) {
 	signAddVotes(cs1, types.PrevoteType, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2)
 	ensurePrevote(voteCh, height, round)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Prevote(round).Nanoseconds())
 	ensurePrecommit(voteCh, height, round)
 	validatePrecommit(t, cs1, round, 0, vss[0], nil, theBlockHash) // precommit nil but locked on proposal
 
@@ -506,7 +500,8 @@ func TestStateLockPOLRelock(t *testing.T) {
 
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader)
 
@@ -547,7 +542,7 @@ func TestStateLockPOLRelock(t *testing.T) {
 	incrementRound(vs2, vs3, vs4)
 
 	// timeout to new round
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	round = round + 1 // moving to the next round
 	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
@@ -597,7 +592,8 @@ func TestStateLockPOLUnlock(t *testing.T) {
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// everything done from perspective of cs1
 
@@ -636,7 +632,7 @@ func TestStateLockPOLUnlock(t *testing.T) {
 	propBlockParts := propBlock.MakePartSet(partSize)
 
 	// timeout to new round
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 	rs = cs1.GetRoundState()
 	lockedBlockHash := rs.LockedBlock.Hash()
 
@@ -690,7 +686,8 @@ func TestStateLockPOLSafety1(t *testing.T) {
 	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, round)
@@ -713,7 +710,7 @@ func TestStateLockPOLSafety1(t *testing.T) {
 
 	// cs1 precommit nil
 	ensurePrecommit(voteCh, height, round)
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	t.Log("### ONTO ROUND 1")
 
@@ -757,7 +754,7 @@ func TestStateLockPOLSafety1(t *testing.T) {
 
 	signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	incrementRound(vs2, vs3, vs4)
 	round = round + 1 // moving to the next round
@@ -770,7 +767,7 @@ func TestStateLockPOLSafety1(t *testing.T) {
 	*/
 
 	// timeout of propose
-	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds())
+	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.Propose(round).Nanoseconds())
 
 	// finish prevote
 	ensurePrevote(voteCh, height, round)
@@ -806,7 +803,8 @@ func TestStateLockPOLSafety2(t *testing.T) {
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// the block for R0: gets polkad but we miss it
 	// (even though we signed it, shhh)
@@ -852,7 +850,7 @@ func TestStateLockPOLSafety2(t *testing.T) {
 	incrementRound(vs2, vs3, vs4)
 
 	// timeout of precommit wait to new round
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	round = round + 1 // moving to the next round
 	// in round 2 we see the polkad block from round 0
@@ -897,7 +895,8 @@ func TestProposeValidBlock(t *testing.T) {
 	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, round)
@@ -920,7 +919,7 @@ func TestProposeValidBlock(t *testing.T) {
 
 	signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	incrementRound(vs2, vs3, vs4)
 	round = round + 1 // moving to the next round
@@ -930,7 +929,7 @@ func TestProposeValidBlock(t *testing.T) {
 	t.Log("### ONTO ROUND 2")
 
 	// timeout of propose
-	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds())
+	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.Propose(round).Nanoseconds())
 
 	ensurePrevote(voteCh, height, round)
 	validatePrevote(t, cs1, round, vss[0], propBlockHash)
@@ -953,7 +952,7 @@ func TestProposeValidBlock(t *testing.T) {
 	ensureNewRound(newRoundCh, height, round)
 	t.Log("### ONTO ROUND 3")
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	round = round + 1 // moving to the next round
 
@@ -983,7 +982,8 @@ func TestSetValidBlockOnDelayedPrevote(t *testing.T) {
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, cs1.Height, round)
@@ -1004,7 +1004,7 @@ func TestSetValidBlockOnDelayedPrevote(t *testing.T) {
 	// vs3 send prevote nil
 	signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs3)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Prevote(round).Nanoseconds())
 
 	ensurePrecommit(voteCh, height, round)
 	// we should have precommitted
@@ -1042,7 +1042,8 @@ func TestSetValidBlockOnDelayedProposal(t *testing.T) {
 	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
 
 	round = round + 1 // move to round in which P0 is not proposer
@@ -1051,7 +1052,7 @@ func TestSetValidBlockOnDelayedProposal(t *testing.T) {
 	startTestRound(cs1, cs1.Height, round)
 	ensureNewRound(newRoundCh, height, round)
 
-	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds())
+	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.Propose(round).Nanoseconds())
 
 	ensurePrevote(voteCh, height, round)
 	validatePrevote(t, cs1, round, vss[0], nil)
@@ -1064,7 +1065,7 @@ func TestSetValidBlockOnDelayedProposal(t *testing.T) {
 	signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4)
 	ensureNewValidBlock(validBlockCh, height, round)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Prevote(round).Nanoseconds())
 
 	ensurePrecommit(voteCh, height, round)
 	validatePrecommit(t, cs1, round, -1, vss[0], nil, nil)
@@ -1098,7 +1099,7 @@ func TestWaitingTimeoutOnNilPolka(t *testing.T) {
 
 	signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 	ensureNewRound(newRoundCh, height, round+1)
 }
 
@@ -1112,7 +1113,8 @@ func TestWaitingTimeoutProposeOnNewRound(t *testing.T) {
 
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round
 	startTestRound(cs1, height, round)
@@ -1129,7 +1131,7 @@ func TestWaitingTimeoutProposeOnNewRound(t *testing.T) {
 	rs := cs1.GetRoundState()
 	assert.True(t, rs.Step == cstypes.RoundStepPropose) // P0 does not prevote before timeoutPropose expires
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPropose.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Propose(round).Nanoseconds())
 
 	ensurePrevote(voteCh, height, round)
 	validatePrevote(t, cs1, round, vss[0], nil)
@@ -1145,7 +1147,8 @@ func TestRoundSkipOnNilPolkaFromHigherRound(t *testing.T) {
 
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round
 	startTestRound(cs1, height, round)
@@ -1162,7 +1165,7 @@ func TestRoundSkipOnNilPolkaFromHigherRound(t *testing.T) {
 	ensurePrecommit(voteCh, height, round)
 	validatePrecommit(t, cs1, round, -1, vss[0], nil, nil)
 
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	round = round + 1 // moving to the next round
 	ensureNewRound(newRoundCh, height, round)
@@ -1178,7 +1181,8 @@ func TestWaitTimeoutProposeOnNilPolkaForTheCurrentRound(t *testing.T) {
 
 	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round in which PO is not proposer
 	startTestRound(cs1, height, round)
@@ -1187,7 +1191,7 @@ func TestWaitTimeoutProposeOnNilPolkaForTheCurrentRound(t *testing.T) {
 	incrementRound(vss[1:]...)
 	signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs2, vs3, vs4)
 
-	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds())
+	ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.Propose(round).Nanoseconds())
 
 	ensurePrevote(voteCh, height, round)
 	validatePrevote(t, cs1, round, vss[0], nil)
@@ -1267,6 +1271,126 @@ func TestCommitFromPreviousRound(t *testing.T) {
 	ensureNewRound(newRoundCh, height+1, 0)
 }
 
+type fakeTxNotifier struct {
+	ch chan struct{}
+}
+
+func (n *fakeTxNotifier) TxsAvailable() <-chan struct{} {
+	return n.ch
+}
+
+func (n *fakeTxNotifier) Notify() {
+	n.ch <- struct{}{}
+}
+
+func TestStartNextHeightCorrectly(t *testing.T) {
+	config.Consensus.SkipTimeoutCommit = false
+	cs1, vss := randConsensusState(4)
+	cs1.txNotifier = &fakeTxNotifier{ch: make(chan struct{})}
+
+	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
+	height, round := cs1.Height, cs1.Round
+
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
+
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader)
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
+
+	// start round and wait for propose and prevote
+	startTestRound(cs1, height, round)
+	ensureNewRound(newRoundCh, height, round)
+
+	ensureNewProposal(proposalCh, height, round)
+	rs := cs1.GetRoundState()
+	theBlockHash := rs.ProposalBlock.Hash()
+	theBlockParts := rs.ProposalBlockParts.Header()
+
+	ensurePrevote(voteCh, height, round)
+	validatePrevote(t, cs1, round, vss[0], theBlockHash)
+
+	signAddVotes(cs1, types.PrevoteType, theBlockHash, theBlockParts, vs2, vs3, vs4)
+
+	ensurePrecommit(voteCh, height, round)
+	// the proposed block should now be locked and our precommit added
+	validatePrecommit(t, cs1, round, round, vss[0], theBlockHash, theBlockHash)
+
+	rs = cs1.GetRoundState()
+
+	// add precommits
+	signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2)
+	signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs3)
+	time.Sleep(5 * time.Millisecond)
+	signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs4)
+
+	rs = cs1.GetRoundState()
+	assert.True(t, rs.TriggeredTimeoutPrecommit)
+
+	ensureNewBlockHeader(newBlockHeader, height, theBlockHash)
+
+	cs1.txNotifier.(*fakeTxNotifier).Notify()
+
+	ensureNewTimeout(timeoutProposeCh, height+1, round, cs1.config.Propose(round).Nanoseconds())
+	rs = cs1.GetRoundState()
+	assert.False(t, rs.TriggeredTimeoutPrecommit, "triggeredTimeoutPrecommit should be false at the beginning of each round")
+}
+
+func TestResetTimeoutPrecommitUponNewHeight(t *testing.T) {
+	config.Consensus.SkipTimeoutCommit = false
+	cs1, vss := randConsensusState(4)
+
+	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
+	height, round := cs1.Height, cs1.Round
+
+	partSize := types.BlockPartSizeBytes
+
+	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
+
+	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
+	newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader)
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
+
+	// start round and wait for propose and prevote
+	startTestRound(cs1, height, round)
+	ensureNewRound(newRoundCh, height, round)
+
+	ensureNewProposal(proposalCh, height, round)
+	rs := cs1.GetRoundState()
+	theBlockHash := rs.ProposalBlock.Hash()
+	theBlockParts := rs.ProposalBlockParts.Header()
+
+	ensurePrevote(voteCh, height, round)
+	validatePrevote(t, cs1, round, vss[0], theBlockHash)
+
+	signAddVotes(cs1, types.PrevoteType, theBlockHash, theBlockParts, vs2, vs3, vs4)
+
+	ensurePrecommit(voteCh, height, round)
+	validatePrecommit(t, cs1, round, round, vss[0], theBlockHash, theBlockHash)
+
+	rs = cs1.GetRoundState()
+
+	// add precommits
+	signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2)
+	signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs3)
+	signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs4)
+
+	ensureNewBlockHeader(newBlockHeader, height, theBlockHash)
+
+	prop, propBlock := decideProposal(cs1, vs2, height+1, 0)
+	propBlockParts := propBlock.MakePartSet(partSize)
+
+	if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil {
+		t.Fatal(err)
+	}
+	ensureNewProposal(proposalCh, height+1, 0)
+
+	rs = cs1.GetRoundState()
+	assert.False(t, rs.TriggeredTimeoutPrecommit, "triggeredTimeoutPrecommit should be false at the beginning of each height")
+}
+
 //------------------------------------------------------------------------------------------
 // SlashingSuite
 // TODO: Slashing
@@ -1362,7 +1486,8 @@ func TestStateHalt1(t *testing.T) {
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
-	voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress())
+	addr := cs1.privValidator.GetPubKey().Address()
+	voteCh := subscribeToVoter(cs1, addr)
 
 	// start round and wait for propose and prevote
 	startTestRound(cs1, height, round)
@@ -1390,7 +1515,7 @@ func TestStateHalt1(t *testing.T) {
 	incrementRound(vs2, vs3, vs4)
 
 	// timeout to new round
-	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds())
+	ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.Precommit(round).Nanoseconds())
 
 	round = round + 1 // moving to the next round
 
@@ -1419,7 +1544,7 @@ func TestStateHalt1(t *testing.T) {
 func TestStateOutputsBlockPartsStats(t *testing.T) {
 	// create dummy peer
 	cs, _ := randConsensusState(1)
-	peer := p2pdummy.NewPeer()
+	peer := p2pmock.NewPeer(nil)
 
 	// 1) new block part
 	parts := types.NewPartSetFromData(cmn.RandBytes(100), 10)
@@ -1462,7 +1587,7 @@ func TestStateOutputsBlockPartsStats(t *testing.T) {
 func TestStateOutputVoteStats(t *testing.T) {
 	cs, vss := randConsensusState(2)
 	// create dummy peer
-	peer := p2pdummy.NewPeer()
+	peer := p2pmock.NewPeer(nil)
 
 	vote := signVote(vss[1], types.PrecommitType, []byte("test"), types.PartSetHeader{})
 
@@ -1491,11 +1616,19 @@ func TestStateOutputVoteStats(t *testing.T) {
 }
 
 // subscribe subscribes test client to the given query and returns a channel with cap = 1.
-func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan interface{} {
-	out := make(chan interface{}, 1)
-	err := eventBus.Subscribe(context.Background(), testSubscriber, q, out)
+func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan tmpubsub.Message {
+	sub, err := eventBus.Subscribe(context.Background(), testSubscriber, q)
 	if err != nil {
 		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, q))
 	}
-	return out
+	return sub.Out()
+}
+
+// subscribe subscribes test client to the given query and returns a channel with cap = 0.
+func subscribeUnBuffered(eventBus *types.EventBus, q tmpubsub.Query) <-chan tmpubsub.Message {
+	sub, err := eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, q)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, q))
+	}
+	return sub.Out()
 }

consensus/types/height_vote_set.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 	"sync"
 
-	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
 )
@@ -83,7 +82,7 @@ func (hvs *HeightVoteSet) SetRound(round int) {
 	hvs.mtx.Lock()
 	defer hvs.mtx.Unlock()
 	if hvs.round != 0 && (round < hvs.round+1) {
-		cmn.PanicSanity("SetRound() must increment hvs.round")
+		panic("SetRound() must increment hvs.round")
 	}
 	for r := hvs.round + 1; r <= round; r++ {
 		if _, ok := hvs.roundVoteSets[r]; ok {
@@ -96,7 +95,7 @@ func (hvs *HeightVoteSet) SetRound(round int) {
 
 func (hvs *HeightVoteSet) addRound(round int) {
 	if _, ok := hvs.roundVoteSets[round]; ok {
-		cmn.PanicSanity("addRound() for an existing round")
+		panic("addRound() for an existing round")
 	}
 	// log.Debug("addRound(round)", "round", round)
 	prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, types.PrevoteType, hvs.valSet)
@@ -169,8 +168,7 @@ func (hvs *HeightVoteSet) getVoteSet(round int, type_ types.SignedMsgType) *type
 	case types.PrecommitType:
 		return rvs.Precommits
 	default:
-		cmn.PanicSanity(fmt.Sprintf("Unexpected vote type %X", type_))
-		return nil
+		panic(fmt.Sprintf("Unexpected vote type %X", type_))
 	}
 }
 

consensus/types/height_vote_set_test.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"fmt"
+	"os"
 	"testing"
 
 	cfg "github.com/tendermint/tendermint/config"
@@ -11,8 +12,11 @@ import (
 
 var config *cfg.Config // NOTE: must be reset for each _test.go file
 
-func init() {
+func TestMain(m *testing.M) {
 	config = cfg.ResetTestRoot("consensus_height_vote_set_test")
+	code := m.Run()
+	os.RemoveAll(config.RootDir)
+	os.Exit(code)
 }
 
 func TestPeerCatchupRounds(t *testing.T) {
@@ -50,8 +54,9 @@ func TestPeerCatchupRounds(t *testing.T) {
 
 func makeVoteHR(t *testing.T, height int64, round int, privVals []types.PrivValidator, valIndex int) *types.Vote {
 	privVal := privVals[valIndex]
+	addr := privVal.GetPubKey().Address()
 	vote := &types.Vote{
-		ValidatorAddress: privVal.GetAddress(),
+		ValidatorAddress: addr,
 		ValidatorIndex:   valIndex,
 		Height:           height,
 		Round:            round,
@@ -63,7 +68,6 @@ func makeVoteHR(t *testing.T, height int64, round int, privVals []types.PrivVali
 	err := privVal.SignVote(chainID, vote)
 	if err != nil {
 		panic(fmt.Sprintf("Error signing vote: %v", err))
-		return nil
 	}
 	return vote
 }

consensus/types/peer_round_state.go

@@ -55,3 +55,31 @@ func (prs PeerRoundState) StringIndented(indent string) string {
 		indent, prs.CatchupCommit, prs.CatchupCommitRound,
 		indent)
 }
+
+//-----------------------------------------------------------
+// These methods are for Protobuf Compatibility
+
+// Size returns the size of the amino encoding, in bytes.
+func (ps *PeerRoundState) Size() int {
+	bs, _ := ps.Marshal()
+	return len(bs)
+}
+
+// Marshal returns the amino encoding.
+func (ps *PeerRoundState) Marshal() ([]byte, error) {
+	return cdc.MarshalBinaryBare(ps)
+}
+
+// MarshalTo calls Marshal and copies to the given buffer.
+func (ps *PeerRoundState) MarshalTo(data []byte) (int, error) {
+	bs, err := ps.Marshal()
+	if err != nil {
+		return -1, err
+	}
+	return copy(data, bs), nil
+}
+
+// Unmarshal deserializes from amino encoded form.
+func (ps *PeerRoundState) Unmarshal(bs []byte) error {
+	return cdc.UnmarshalBinaryBare(bs, ps)
+}

consensus/types/round_state.go

@@ -65,25 +65,26 @@ func (rs RoundStepType) String() string {
 // NOTE: Not thread safe. Should only be manipulated by functions downstream
 // of the cs.receiveRoutine
 type RoundState struct {
-	Height             int64               `json:"height"` // Height we are working on
-	Round              int                 `json:"round"`
-	Step               RoundStepType       `json:"step"`
-	StartTime          time.Time           `json:"start_time"`
-	CommitTime         time.Time           `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found
-	Validators         *types.ValidatorSet `json:"validators"`
-	Proposal           *types.Proposal     `json:"proposal"`
-	ProposalBlock      *types.Block        `json:"proposal_block"`
-	ProposalBlockParts *types.PartSet      `json:"proposal_block_parts"`
-	LockedRound        int                 `json:"locked_round"`
-	LockedBlock        *types.Block        `json:"locked_block"`
-	LockedBlockParts   *types.PartSet      `json:"locked_block_parts"`
-	ValidRound         int                 `json:"valid_round"`       // Last known round with POL for non-nil valid block.
-	ValidBlock         *types.Block        `json:"valid_block"`       // Last known block of POL mentioned above.
-	ValidBlockParts    *types.PartSet      `json:"valid_block_parts"` // Last known block parts of POL metnioned above.
-	Votes              *HeightVoteSet      `json:"votes"`
-	CommitRound        int                 `json:"commit_round"` //
-	LastCommit         *types.VoteSet      `json:"last_commit"`  // Last precommits at Height-1
-	LastValidators     *types.ValidatorSet `json:"last_validators"`
+	Height                    int64               `json:"height"` // Height we are working on
+	Round                     int                 `json:"round"`
+	Step                      RoundStepType       `json:"step"`
+	StartTime                 time.Time           `json:"start_time"`
+	CommitTime                time.Time           `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found
+	Validators                *types.ValidatorSet `json:"validators"`
+	Proposal                  *types.Proposal     `json:"proposal"`
+	ProposalBlock             *types.Block        `json:"proposal_block"`
+	ProposalBlockParts        *types.PartSet      `json:"proposal_block_parts"`
+	LockedRound               int                 `json:"locked_round"`
+	LockedBlock               *types.Block        `json:"locked_block"`
+	LockedBlockParts          *types.PartSet      `json:"locked_block_parts"`
+	ValidRound                int                 `json:"valid_round"`       // Last known round with POL for non-nil valid block.
+	ValidBlock                *types.Block        `json:"valid_block"`       // Last known block of POL mentioned above.
+	ValidBlockParts           *types.PartSet      `json:"valid_block_parts"` // Last known block parts of POL metnioned above.
+	Votes                     *HeightVoteSet      `json:"votes"`
+	CommitRound               int                 `json:"commit_round"` //
+	LastCommit                *types.VoteSet      `json:"last_commit"`  // Last precommits at Height-1
+	LastValidators            *types.ValidatorSet `json:"last_validators"`
+	TriggeredTimeoutPrecommit bool                `json:"triggered_timeout_precommit"`
 }
 
 // Compressed version of the RoundState for use in RPC
@@ -112,18 +113,46 @@ func (rs *RoundState) RoundStateSimple() RoundStateSimple {
 	}
 }
 
+// NewRoundEvent returns the RoundState with proposer information as an event.
+func (rs *RoundState) NewRoundEvent() types.EventDataNewRound {
+	addr := rs.Validators.GetProposer().Address
+	idx, _ := rs.Validators.GetByAddress(addr)
+
+	return types.EventDataNewRound{
+		Height: rs.Height,
+		Round:  rs.Round,
+		Step:   rs.Step.String(),
+		Proposer: types.ValidatorInfo{
+			Address: addr,
+			Index:   idx,
+		},
+	}
+}
+
+// CompleteProposalEvent returns information about a proposed block as an event.
+func (rs *RoundState) CompleteProposalEvent() types.EventDataCompleteProposal {
+	// We must construct BlockID from ProposalBlock and ProposalBlockParts
+	// cs.Proposal is not guaranteed to be set when this function is called
+	blockId := types.BlockID{
+		Hash:        rs.ProposalBlock.Hash(),
+		PartsHeader: rs.ProposalBlockParts.Header(),
+	}
+
+	return types.EventDataCompleteProposal{
+		Height:  rs.Height,
+		Round:   rs.Round,
+		Step:    rs.Step.String(),
+		BlockID: blockId,
+	}
+}
+
 // RoundStateEvent returns the H/R/S of the RoundState as an event.
 func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
-	// copy the RoundState.
-	// TODO: if we want to avoid this, we may need synchronous events after all
-	rsCopy := *rs
-	edrs := types.EventDataRoundState{
-		Height:     rs.Height,
-		Round:      rs.Round,
-		Step:       rs.Step.String(),
-		RoundState: &rsCopy,
+	return types.EventDataRoundState{
+		Height: rs.Height,
+		Round:  rs.Round,
+		Step:   rs.Step.String(),
 	}
-	return edrs
 }
 
 // String returns a string
@@ -169,3 +198,31 @@ func (rs *RoundState) StringShort() string {
 	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
 		rs.Height, rs.Round, rs.Step, rs.StartTime)
 }
+
+//-----------------------------------------------------------
+// These methods are for Protobuf Compatibility
+
+// Size returns the size of the amino encoding, in bytes.
+func (rs *RoundStateSimple) Size() int {
+	bs, _ := rs.Marshal()
+	return len(bs)
+}
+
+// Marshal returns the amino encoding.
+func (rs *RoundStateSimple) Marshal() ([]byte, error) {
+	return cdc.MarshalBinaryBare(rs)
+}
+
+// MarshalTo calls Marshal and copies to the given buffer.
+func (rs *RoundStateSimple) MarshalTo(data []byte) (int, error) {
+	bs, err := rs.Marshal()
+	if err != nil {
+		return -1, err
+	}
+	return copy(data, bs), nil
+}
+
+// Unmarshal deserializes from amino encoded form.
+func (rs *RoundStateSimple) Unmarshal(bs []byte) error {
+	return cdc.UnmarshalBinaryBare(bs, rs)
+}

consensus/types/round_state_test.go

@@ -3,7 +3,7 @@ package types
 import (
 	"testing"
 
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/types"
@@ -16,7 +16,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 	// Random validators
 	nval, ntxs := 100, 100
 	vset, _ := types.RandValidatorSet(nval, 1)
-	precommits := make([]*types.Vote, nval)
+	precommits := make([]*types.CommitSig, nval)
 	blockID := types.BlockID{
 		Hash: cmn.RandBytes(20),
 		PartsHeader: types.PartSetHeader{
@@ -25,12 +25,12 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 	}
 	sig := make([]byte, ed25519.SignatureSize)
 	for i := 0; i < nval; i++ {
-		precommits[i] = &types.Vote{
+		precommits[i] = (&types.Vote{
 			ValidatorAddress: types.Address(cmn.RandBytes(20)),
 			Timestamp:        tmtime.Now(),
 			BlockID:          blockID,
 			Signature:        sig,
-		}
+		}).CommitSig()
 	}
 	txs := make([]types.Tx, ntxs)
 	for i := 0; i < ntxs; i++ {
@@ -53,11 +53,8 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 		Data: types.Data{
 			Txs: txs,
 		},
-		Evidence: types.EvidenceData{},
-		LastCommit: &types.Commit{
-			BlockID:    blockID,
-			Precommits: precommits,
-		},
+		Evidence:   types.EvidenceData{},
+		LastCommit: types.NewCommit(blockID, precommits),
 	}
 	parts := block.MakePartSet(4096)
 	// Random Proposal

consensus/wal.go

@@ -13,6 +13,7 @@ import (
 	amino "github.com/tendermint/go-amino"
 	auto "github.com/tendermint/tendermint/libs/autofile"
 	cmn "github.com/tendermint/tendermint/libs/common"
+	"github.com/tendermint/tendermint/libs/log"
 	"github.com/tendermint/tendermint/types"
 	tmtime "github.com/tendermint/tendermint/types/time"
 )
@@ -20,6 +21,9 @@ import (
 const (
 	// must be greater than types.BlockPartSizeBytes + a few bytes
 	maxMsgSizeBytes = 1024 * 1024 // 1MB
+
+	// how often the WAL should be sync'd during period sync'ing
+	walDefaultFlushInterval = 2 * time.Second
 )
 
 //--------------------------------------------------------
@@ -53,26 +57,36 @@ func RegisterWALMessages(cdc *amino.Codec) {
 type WAL interface {
 	Write(WALMessage)
 	WriteSync(WALMessage)
-	Group() *auto.Group
-	SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error)
+	FlushAndSync() error
 
+	SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error)
+
+	// service methods
 	Start() error
 	Stop() error
 	Wait()
 }
 
 // Write ahead logger writes msgs to disk before they are processed.
-// Can be used for crash-recovery and deterministic replay
-// TODO: currently the wal is overwritten during replay catchup
-//   give it a mode so it's either reading or appending - must read to end to start appending again
+// Can be used for crash-recovery and deterministic replay.
+// TODO: currently the wal is overwritten during replay catchup, give it a mode
+// so it's either reading or appending - must read to end to start appending
+// again.
 type baseWAL struct {
 	cmn.BaseService
 
 	group *auto.Group
 
 	enc *WALEncoder
+
+	flushTicker   *time.Ticker
+	flushInterval time.Duration
 }
 
+var _ WAL = &baseWAL{}
+
+// NewWAL returns a new write-ahead logger based on `baseWAL`, which implements
+// WAL. It's flushed and synced to disk every 2s and once when stopped.
 func NewWAL(walFile string, groupOptions ...func(*auto.Group)) (*baseWAL, error) {
 	err := cmn.EnsureDir(filepath.Dir(walFile), 0700)
 	if err != nil {
@@ -84,17 +98,28 @@ func NewWAL(walFile string, groupOptions ...func(*auto.Group)) (*baseWAL, error)
 		return nil, err
 	}
 	wal := &baseWAL{
-		group: group,
-		enc:   NewWALEncoder(group),
+		group:         group,
+		enc:           NewWALEncoder(group),
+		flushInterval: walDefaultFlushInterval,
 	}
 	wal.BaseService = *cmn.NewBaseService(nil, "baseWAL", wal)
 	return wal, nil
 }
 
+// SetFlushInterval allows us to override the periodic flush interval for the WAL.
+func (wal *baseWAL) SetFlushInterval(i time.Duration) {
+	wal.flushInterval = i
+}
+
 func (wal *baseWAL) Group() *auto.Group {
 	return wal.group
 }
 
+func (wal *baseWAL) SetLogger(l log.Logger) {
+	wal.BaseService.Logger = l
+	wal.group.SetLogger(l)
+}
+
 func (wal *baseWAL) OnStart() error {
 	size, err := wal.group.Head.Size()
 	if err != nil {
@@ -103,14 +128,49 @@ func (wal *baseWAL) OnStart() error {
 		wal.WriteSync(EndHeightMessage{0})
 	}
 	err = wal.group.Start()
-	return err
+	if err != nil {
+		return err
+	}
+	wal.flushTicker = time.NewTicker(wal.flushInterval)
+	go wal.processFlushTicks()
+	return nil
 }
 
+func (wal *baseWAL) processFlushTicks() {
+	for {
+		select {
+		case <-wal.flushTicker.C:
+			if err := wal.FlushAndSync(); err != nil {
+				wal.Logger.Error("Periodic WAL flush failed", "err", err)
+			}
+		case <-wal.Quit():
+			return
+		}
+	}
+}
+
+// FlushAndSync flushes and fsync's the underlying group's data to disk.
+// See auto#FlushAndSync
+func (wal *baseWAL) FlushAndSync() error {
+	return wal.group.FlushAndSync()
+}
+
+// Stop the underlying autofile group.
+// Use Wait() to ensure it's finished shutting down
+// before cleaning up files.
 func (wal *baseWAL) OnStop() {
+	wal.flushTicker.Stop()
+	wal.FlushAndSync()
 	wal.group.Stop()
 	wal.group.Close()
 }
 
+// Wait for the underlying autofile group to finish shutting down
+// so it's safe to cleanup files.
+func (wal *baseWAL) Wait() {
+	wal.group.Wait()
+}
+
 // Write is called in newStep and for each receive on the
 // peerMsgQueue and the timeoutTicker.
 // NOTE: does not call fsync()
@@ -134,7 +194,7 @@ func (wal *baseWAL) WriteSync(msg WALMessage) {
 	}
 
 	wal.Write(msg)
-	if err := wal.group.Flush(); err != nil {
+	if err := wal.FlushAndSync(); err != nil {
 		panic(fmt.Sprintf("Error flushing consensus wal buf to file. Error: %v \n", err))
 	}
 }
@@ -150,14 +210,17 @@ type WALSearchOptions struct {
 // Group reader will be nil if found equals false.
 //
 // CONTRACT: caller must close group reader.
-func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
-	var msg *TimedWALMessage
+func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	var (
+		msg *TimedWALMessage
+		gr  *auto.GroupReader
+	)
 	lastHeightFound := int64(-1)
 
 	// NOTE: starting from the last file in the group because we're usually
 	// searching for the last height. See replay.go
 	min, max := wal.group.MinIndex(), wal.group.MaxIndex()
-	wal.Logger.Debug("Searching for height", "height", height, "min", min, "max", max)
+	wal.Logger.Info("Searching for height", "height", height, "min", min, "max", max)
 	for index := max; index >= min; index-- {
 		gr, err = wal.group.NewReader(index)
 		if err != nil {
@@ -177,7 +240,7 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions)
 				break
 			}
 			if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) {
-				wal.Logger.Debug("Corrupted entry. Skipping...", "err", err)
+				wal.Logger.Error("Corrupted entry. Skipping...", "err", err)
 				// do nothing
 				continue
 			} else if err != nil {
@@ -188,7 +251,7 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions)
 			if m, ok := msg.Msg.(EndHeightMessage); ok {
 				lastHeightFound = m.Height
 				if m.Height == height { // found
-					wal.Logger.Debug("Found", "height", height, "index", index)
+					wal.Logger.Info("Found", "height", height, "index", index)
 					return gr, true, nil
 				}
 			}
@@ -213,12 +276,17 @@ func NewWALEncoder(wr io.Writer) *WALEncoder {
 	return &WALEncoder{wr}
 }
 
-// Encode writes the custom encoding of v to the stream.
+// Encode writes the custom encoding of v to the stream. It returns an error if
+// the amino-encoded size of v is greater than 1MB. Any error encountered
+// during the write is also returned.
 func (enc *WALEncoder) Encode(v *TimedWALMessage) error {
 	data := cdc.MustMarshalBinaryBare(v)
 
 	crc := crc32.Checksum(data, crc32c)
 	length := uint32(len(data))
+	if length > maxMsgSizeBytes {
+		return fmt.Errorf("Msg is too big: %d bytes, max: %d bytes", length, maxMsgSizeBytes)
+	}
 	totalLength := 8 + int(length)
 
 	msg := make([]byte, totalLength)
@@ -275,31 +343,31 @@ func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
 		return nil, err
 	}
 	if err != nil {
-		return nil, fmt.Errorf("failed to read checksum: %v", err)
+		return nil, DataCorruptionError{fmt.Errorf("failed to read checksum: %v", err)}
 	}
 	crc := binary.BigEndian.Uint32(b)
 
 	b = make([]byte, 4)
 	_, err = dec.rd.Read(b)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read length: %v", err)
+		return nil, DataCorruptionError{fmt.Errorf("failed to read length: %v", err)}
 	}
 	length := binary.BigEndian.Uint32(b)
 
 	if length > maxMsgSizeBytes {
-		return nil, fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes)
+		return nil, DataCorruptionError{fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes)}
 	}
 
 	data := make([]byte, length)
-	_, err = dec.rd.Read(data)
+	n, err := dec.rd.Read(data)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read data: %v", err)
+		return nil, DataCorruptionError{fmt.Errorf("failed to read data: %v (read: %d, wanted: %d)", err, n, length)}
 	}
 
 	// check checksum before decoding data
 	actualCRC := crc32.Checksum(data, crc32c)
 	if actualCRC != crc {
-		return nil, DataCorruptionError{fmt.Errorf("checksums do not match: (read: %v, actual: %v)", crc, actualCRC)}
+		return nil, DataCorruptionError{fmt.Errorf("checksums do not match: read: %v, actual: %v", crc, actualCRC)}
 	}
 
 	var res = new(TimedWALMessage) // nolint: gosimple
@@ -313,10 +381,12 @@ func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
 
 type nilWAL struct{}
 
+var _ WAL = nilWAL{}
+
 func (nilWAL) Write(m WALMessage)     {}
 func (nilWAL) WriteSync(m WALMessage) {}
-func (nilWAL) Group() *auto.Group     { return nil }
-func (nilWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+func (nilWAL) FlushAndSync() error    { return nil }
+func (nilWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
 	return nil, false, nil
 }
 func (nilWAL) Start() error { return nil }

consensus/wal_generator.go

@@ -5,31 +5,31 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
-	"strings"
+	"testing"
 	"time"
 
 	"github.com/pkg/errors"
+
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
-	auto "github.com/tendermint/tendermint/libs/autofile"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/mock"
 	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
-// WALGenerateNBlocks generates a consensus WAL. It does this by spining up a
+// WALGenerateNBlocks generates a consensus WAL. It does this by spinning up a
 // stripped down version of node (proxy app, event bus, consensus state) with a
 // persistent kvstore application and special consensus wal instance
 // (byteBufferWAL) and waits until numBlocks are created. If the node fails to produce given numBlocks, it returns an error.
-func WALGenerateNBlocks(wr io.Writer, numBlocks int) (err error) {
-	config := getConfig()
+func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
+	config := getConfig(t)
 
 	app := kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), "wal_generator"))
 
@@ -40,19 +40,21 @@ func WALGenerateNBlocks(wr io.Writer, numBlocks int) (err error) {
 	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
 	// NOTE: we can't import node package because of circular dependency.
 	// NOTE: we don't do handshake so need to set state.Version.Consensus.App directly.
-	privValidatorFile := config.PrivValidatorFile()
-	privValidator := privval.LoadOrGenFilePV(privValidatorFile)
+	privValidatorKeyFile := config.PrivValidatorKeyFile()
+	privValidatorStateFile := config.PrivValidatorStateFile()
+	privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
 	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
 	if err != nil {
 		return errors.Wrap(err, "failed to read genesis file")
 	}
-	stateDB := db.NewMemDB()
 	blockStoreDB := db.NewMemDB()
+	stateDB := blockStoreDB
 	state, err := sm.MakeGenesisState(genDoc)
 	if err != nil {
 		return errors.Wrap(err, "failed to make genesis state")
 	}
 	state.Version.Consensus.App = kvstore.ProtocolVersion
+	sm.SaveState(stateDB, state)
 	blockStore := bc.NewBlockStore(blockStoreDB)
 	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app))
 	proxyApp.SetLogger(logger.With("module", "proxy"))
@@ -67,7 +69,7 @@ func WALGenerateNBlocks(wr io.Writer, numBlocks int) (err error) {
 		return errors.Wrap(err, "failed to start event bus")
 	}
 	defer eventBus.Stop()
-	mempool := sm.MockMempool{}
+	mempool := mock.Mempool{}
 	evpool := sm.MockEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 	consensusState := NewConsensusState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
@@ -101,11 +103,11 @@ func WALGenerateNBlocks(wr io.Writer, numBlocks int) (err error) {
 }
 
 //WALWithNBlocks returns a WAL content with numBlocks.
-func WALWithNBlocks(numBlocks int) (data []byte, err error) {
+func WALWithNBlocks(t *testing.T, numBlocks int) (data []byte, err error) {
 	var b bytes.Buffer
 	wr := bufio.NewWriter(&b)
 
-	if err := WALGenerateNBlocks(wr, numBlocks); err != nil {
+	if err := WALGenerateNBlocks(t, wr, numBlocks); err != nil {
 		return []byte{}, err
 	}
 
@@ -113,18 +115,6 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) {
 	return b.Bytes(), nil
 }
 
-// f**ing long, but unique for each test
-func makePathname() string {
-	// get path
-	p, err := os.Getwd()
-	if err != nil {
-		panic(err)
-	}
-	// fmt.Println(p)
-	sep := string(filepath.Separator)
-	return strings.Replace(p, sep, "_", -1)
-}
-
 func randPort() int {
 	// returns between base and base + spread
 	base, spread := 20000, 20000
@@ -139,9 +129,8 @@ func makeAddrs() (string, string, string) {
 }
 
 // getConfig returns a config for test cases
-func getConfig() *cfg.Config {
-	pathname := makePathname()
-	c := cfg.ResetTestRoot(fmt.Sprintf("%s_%d", pathname, cmn.RandInt()))
+func getConfig(t *testing.T) *cfg.Config {
+	c := cfg.ResetTestRoot(t.Name())
 
 	// and we use random ports to run in parallel
 	tm, rpc, grpc := makeAddrs()
@@ -205,10 +194,9 @@ func (w *byteBufferWAL) WriteSync(m WALMessage) {
 	w.Write(m)
 }
 
-func (w *byteBufferWAL) Group() *auto.Group {
-	panic("not implemented")
-}
-func (w *byteBufferWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
+func (w *byteBufferWAL) FlushAndSync() error { return nil }
+
+func (w *byteBufferWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
 	return nil, false, nil
 }
 

consensus/wal_test.go

@@ -3,7 +3,6 @@ package consensus
 import (
 	"bytes"
 	"crypto/rand"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -14,6 +13,7 @@ import (
 
 	"github.com/tendermint/tendermint/consensus/types"
 	"github.com/tendermint/tendermint/libs/autofile"
+	"github.com/tendermint/tendermint/libs/log"
 	tmtypes "github.com/tendermint/tendermint/types"
 	tmtime "github.com/tendermint/tendermint/types/time"
 
@@ -21,41 +21,51 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const (
+	walTestFlushInterval = time.Duration(100) * time.Millisecond
+)
+
 func TestWALTruncate(t *testing.T) {
 	walDir, err := ioutil.TempDir("", "wal")
-	if err != nil {
-		panic(fmt.Errorf("failed to create temp WAL file: %v", err))
-	}
+	require.NoError(t, err)
 	defer os.RemoveAll(walDir)
 
 	walFile := filepath.Join(walDir, "wal")
 
-	//this magic number 4K can truncate the content when RotateFile. defaultHeadSizeLimit(10M) is hard to simulate.
-	//this magic number 1 * time.Millisecond make RotateFile check frequently. defaultGroupCheckDuration(5s) is hard to simulate.
-	wal, err := NewWAL(walFile, autofile.GroupHeadSizeLimit(4096), autofile.GroupCheckDuration(1*time.Millisecond))
-	if err != nil {
-		t.Fatal(err)
-	}
+	// this magic number 4K can truncate the content when RotateFile.
+	// defaultHeadSizeLimit(10M) is hard to simulate.
+	// this magic number 1 * time.Millisecond make RotateFile check frequently.
+	// defaultGroupCheckDuration(5s) is hard to simulate.
+	wal, err := NewWAL(walFile,
+		autofile.GroupHeadSizeLimit(4096),
+		autofile.GroupCheckDuration(1*time.Millisecond),
+	)
+	require.NoError(t, err)
+	wal.SetLogger(log.TestingLogger())
+	err = wal.Start()
+	require.NoError(t, err)
+	defer func() {
+		wal.Stop()
+		// wait for the wal to finish shutting down so we
+		// can safely remove the directory
+		wal.Wait()
+	}()
 
-	wal.Start()
-	defer wal.Stop()
-
-	//60 block's size nearly 70K, greater than group's headBuf size(4096 * 10), when headBuf is full, truncate content will Flush to the file.
-	//at this time, RotateFile is called, truncate content exist in each file.
-	err = WALGenerateNBlocks(wal.Group(), 60)
-	if err != nil {
-		t.Fatal(err)
-	}
+	// 60 block's size nearly 70K, greater than group's headBuf size(4096 * 10),
+	// when headBuf is full, truncate content will Flush to the file. at this
+	// time, RotateFile is called, truncate content exist in each file.
+	err = WALGenerateNBlocks(t, wal.Group(), 60)
+	require.NoError(t, err)
 
 	time.Sleep(1 * time.Millisecond) //wait groupCheckDuration, make sure RotateFile run
 
-	wal.Group().Flush()
+	wal.FlushAndSync()
 
 	h := int64(50)
 	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
-	assert.NoError(t, err, fmt.Sprintf("expected not to err on height %d", h))
-	assert.True(t, found, fmt.Sprintf("expected to find end height for %d", h))
-	assert.NotNil(t, gr, "expected group not to be nil")
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
 	defer gr.Close()
 
 	dec := NewWALDecoder(gr)
@@ -63,14 +73,14 @@ func TestWALTruncate(t *testing.T) {
 	assert.NoError(t, err, "expected to decode a message")
 	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
 	assert.True(t, ok, "expected message of type EventDataRoundState")
-	assert.Equal(t, rs.Height, h+1, fmt.Sprintf("wrong height"))
+	assert.Equal(t, rs.Height, h+1, "wrong height")
 }
 
 func TestWALEncoderDecoder(t *testing.T) {
 	now := tmtime.Now()
 	msgs := []TimedWALMessage{
-		TimedWALMessage{Time: now, Msg: EndHeightMessage{0}},
-		TimedWALMessage{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
+		{Time: now, Msg: EndHeightMessage{0}},
+		{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
 	}
 
 	b := new(bytes.Buffer)
@@ -91,23 +101,42 @@ func TestWALEncoderDecoder(t *testing.T) {
 	}
 }
 
+func TestWALWritePanicsIfMsgIsTooBig(t *testing.T) {
+	walDir, err := ioutil.TempDir("", "wal")
+	require.NoError(t, err)
+	defer os.RemoveAll(walDir)
+	walFile := filepath.Join(walDir, "wal")
+
+	wal, err := NewWAL(walFile)
+	require.NoError(t, err)
+	err = wal.Start()
+	require.NoError(t, err)
+	defer func() {
+		wal.Stop()
+		// wait for the wal to finish shutting down so we
+		// can safely remove the directory
+		wal.Wait()
+	}()
+
+	assert.Panics(t, func() { wal.Write(make([]byte, maxMsgSizeBytes+1)) })
+}
+
 func TestWALSearchForEndHeight(t *testing.T) {
-	walBody, err := WALWithNBlocks(6)
+	walBody, err := WALWithNBlocks(t, 6)
 	if err != nil {
 		t.Fatal(err)
 	}
 	walFile := tempWALWithData(walBody)
 
 	wal, err := NewWAL(walFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
+	wal.SetLogger(log.TestingLogger())
 
 	h := int64(3)
 	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
-	assert.NoError(t, err, fmt.Sprintf("expected not to err on height %d", h))
-	assert.True(t, found, fmt.Sprintf("expected to find end height for %d", h))
-	assert.NotNil(t, gr, "expected group not to be nil")
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
 	defer gr.Close()
 
 	dec := NewWALDecoder(gr)
@@ -115,7 +144,47 @@ func TestWALSearchForEndHeight(t *testing.T) {
 	assert.NoError(t, err, "expected to decode a message")
 	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
 	assert.True(t, ok, "expected message of type EventDataRoundState")
-	assert.Equal(t, rs.Height, h+1, fmt.Sprintf("wrong height"))
+	assert.Equal(t, rs.Height, h+1, "wrong height")
+}
+
+func TestWALPeriodicSync(t *testing.T) {
+	walDir, err := ioutil.TempDir("", "wal")
+	require.NoError(t, err)
+	defer os.RemoveAll(walDir)
+
+	walFile := filepath.Join(walDir, "wal")
+	wal, err := NewWAL(walFile, autofile.GroupCheckDuration(1*time.Millisecond))
+	require.NoError(t, err)
+
+	wal.SetFlushInterval(walTestFlushInterval)
+	wal.SetLogger(log.TestingLogger())
+
+	// Generate some data
+	err = WALGenerateNBlocks(t, wal.Group(), 5)
+	require.NoError(t, err)
+
+	// We should have data in the buffer now
+	assert.NotZero(t, wal.Group().Buffered())
+
+	require.NoError(t, wal.Start())
+	defer func() {
+		wal.Stop()
+		wal.Wait()
+	}()
+
+	time.Sleep(walTestFlushInterval + (10 * time.Millisecond))
+
+	// The data should have been flushed by the periodic sync
+	assert.Zero(t, wal.Group().Buffered())
+
+	h := int64(4)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	if gr != nil {
+		gr.Close()
+	}
 }
 
 /*

consensus/wire.go

@@ -1,7 +1,7 @@
 package consensus
 
 import (
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 	"github.com/tendermint/tendermint/types"
 )
 

crypto/armor/armor.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 
-	"golang.org/x/crypto/openpgp/armor" // forked to github.com/tendermint/crypto
+	"golang.org/x/crypto/openpgp/armor"
 )
 
 func EncodeArmor(blockType string, headers map[string]string, data []byte) string {

crypto/doc.go

@@ -37,9 +37,6 @@
 //     sum := crypto.Sha256([]byte("This is Tendermint"))
 //     fmt.Printf("%x\n", sum)
 
-// Ripemd160
-//     sum := crypto.Ripemd160([]byte("This is consensus"))
-//     fmt.Printf("%x\n", sum)
 package crypto
 
 // TODO: Add more docs in here

crypto/ed25519/ed25519.go

@@ -7,7 +7,7 @@ import (
 	"io"
 
 	amino "github.com/tendermint/go-amino"
-	"golang.org/x/crypto/ed25519" // forked to github.com/tendermint/crypto
+	"golang.org/x/crypto/ed25519"
 
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/tmhash"
@@ -18,8 +18,8 @@ import (
 var _ crypto.PrivKey = PrivKeyEd25519{}
 
 const (
-	PrivKeyAminoRoute = "tendermint/PrivKeyEd25519"
-	PubKeyAminoRoute  = "tendermint/PubKeyEd25519"
+	PrivKeyAminoName = "tendermint/PrivKeyEd25519"
+	PubKeyAminoName  = "tendermint/PubKeyEd25519"
 	// Size of an Edwards25519 signature. Namely the size of a compressed
 	// Edwards25519 point, and a field element. Both of which are 32 bytes.
 	SignatureSize = 64
@@ -30,11 +30,11 @@ var cdc = amino.NewCodec()
 func init() {
 	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
 	cdc.RegisterConcrete(PubKeyEd25519{},
-		PubKeyAminoRoute, nil)
+		PubKeyAminoName, nil)
 
 	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
 	cdc.RegisterConcrete(PrivKeyEd25519{},
-		PrivKeyAminoRoute, nil)
+		PrivKeyAminoName, nil)
 }
 
 // PrivKeyEd25519 implements crypto.PrivKey.

crypto/encoding/amino/amino.go

@@ -12,11 +12,11 @@ import (
 
 var cdc = amino.NewCodec()
 
-// routeTable is used to map public key concrete types back
-// to their amino routes. This should eventually be handled
+// nameTable is used to map public key concrete types back
+// to their registered amino names. This should eventually be handled
 // by amino. Example usage:
-// routeTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoRoute
-var routeTable = make(map[reflect.Type]string, 3)
+// nameTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoName
+var nameTable = make(map[reflect.Type]string, 3)
 
 func init() {
 	// NOTE: It's important that there be no conflicts here,
@@ -29,16 +29,16 @@ func init() {
 
 	// TODO: Have amino provide a way to go from concrete struct to route directly.
 	// Its currently a private API
-	routeTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoRoute
-	routeTable[reflect.TypeOf(secp256k1.PubKeySecp256k1{})] = secp256k1.PubKeyAminoRoute
-	routeTable[reflect.TypeOf(&multisig.PubKeyMultisigThreshold{})] = multisig.PubKeyMultisigThresholdAminoRoute
+	nameTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoName
+	nameTable[reflect.TypeOf(secp256k1.PubKeySecp256k1{})] = secp256k1.PubKeyAminoName
+	nameTable[reflect.TypeOf(multisig.PubKeyMultisigThreshold{})] = multisig.PubKeyMultisigThresholdAminoRoute
 }
 
-// PubkeyAminoRoute returns the amino route of a pubkey
+// PubkeyAminoName returns the amino route of a pubkey
 // cdc is currently passed in, as eventually this will not be using
 // a package level codec.
-func PubkeyAminoRoute(cdc *amino.Codec, key crypto.PubKey) (string, bool) {
-	route, found := routeTable[reflect.TypeOf(key)]
+func PubkeyAminoName(cdc *amino.Codec, key crypto.PubKey) (string, bool) {
+	route, found := nameTable[reflect.TypeOf(key)]
 	return route, found
 }
 
@@ -47,17 +47,17 @@ func RegisterAmino(cdc *amino.Codec) {
 	// These are all written here instead of
 	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
 	cdc.RegisterConcrete(ed25519.PubKeyEd25519{},
-		ed25519.PubKeyAminoRoute, nil)
+		ed25519.PubKeyAminoName, nil)
 	cdc.RegisterConcrete(secp256k1.PubKeySecp256k1{},
-		secp256k1.PubKeyAminoRoute, nil)
+		secp256k1.PubKeyAminoName, nil)
 	cdc.RegisterConcrete(multisig.PubKeyMultisigThreshold{},
 		multisig.PubKeyMultisigThresholdAminoRoute, nil)
 
 	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
 	cdc.RegisterConcrete(ed25519.PrivKeyEd25519{},
-		ed25519.PrivKeyAminoRoute, nil)
+		ed25519.PrivKeyAminoName, nil)
 	cdc.RegisterConcrete(secp256k1.PrivKeySecp256k1{},
-		secp256k1.PrivKeyAminoRoute, nil)
+		secp256k1.PrivKeyAminoName, nil)
 }
 
 func PrivKeyFromBytes(privKeyBytes []byte) (privKey crypto.PrivKey, err error) {

crypto/encoding/amino/encode_test.go

@@ -25,9 +25,8 @@ func checkAminoBinary(t *testing.T, src, dst interface{}, size int) {
 		assert.Equal(t, byterSrc.Bytes(), bz, "Amino binary vs Bytes() mismatch")
 	}
 	// Make sure we have the expected length.
-	if size != -1 {
-		assert.Equal(t, size, len(bz), "Amino binary size mismatch")
-	}
+	assert.Equal(t, size, len(bz), "Amino binary size mismatch")
+
 	// Unmarshal.
 	err = cdc.UnmarshalBinaryBare(bz, dst)
 	require.Nil(t, err, "%+v", err)
@@ -48,6 +47,8 @@ func checkAminoJSON(t *testing.T, src interface{}, dst interface{}, isNil bool)
 	require.Nil(t, err, "%+v", err)
 }
 
+// ExamplePrintRegisteredTypes refers to unknown identifier: PrintRegisteredTypes
+//nolint:govet
 func ExamplePrintRegisteredTypes() {
 	cdc.PrintTypes(os.Stdout)
 	// Output: | Type | Name | Prefix | Length | Notes |
@@ -128,18 +129,18 @@ func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) {
 	require.Nil(t, pk)
 }
 
-func TestPubkeyAminoRoute(t *testing.T) {
+func TestPubkeyAminoName(t *testing.T) {
 	tests := []struct {
 		key   crypto.PubKey
 		want  string
 		found bool
 	}{
-		{ed25519.PubKeyEd25519{}, ed25519.PubKeyAminoRoute, true},
-		{secp256k1.PubKeySecp256k1{}, secp256k1.PubKeyAminoRoute, true},
-		{&multisig.PubKeyMultisigThreshold{}, multisig.PubKeyMultisigThresholdAminoRoute, true},
+		{ed25519.PubKeyEd25519{}, ed25519.PubKeyAminoName, true},
+		{secp256k1.PubKeySecp256k1{}, secp256k1.PubKeyAminoName, true},
+		{multisig.PubKeyMultisigThreshold{}, multisig.PubKeyMultisigThresholdAminoRoute, true},
 	}
 	for i, tc := range tests {
-		got, found := PubkeyAminoRoute(cdc, tc.key)
+		got, found := PubkeyAminoName(cdc, tc.key)
 		require.Equal(t, tc.found, found, "not equal on tc %d", i)
 		if tc.found {
 			require.Equal(t, tc.want, got, "not equal on tc %d", i)

crypto/example_test.go

@@ -26,10 +26,3 @@ func ExampleSha256() {
 	// Output:
 	// f91afb642f3d1c87c17eb01aae5cb65c242dfdbe7cf1066cc260f4ce5d33b94e
 }
-
-func ExampleRipemd160() {
-	sum := crypto.Ripemd160([]byte("This is Tendermint"))
-	fmt.Printf("%x\n", sum)
-	// Output:
-	// 051e22663e8f0fd2f2302f1210f954adff009005
-}

crypto/hash.go

@@ -2,8 +2,6 @@ package crypto
 
 import (
 	"crypto/sha256"
-
-	"golang.org/x/crypto/ripemd160" // forked to github.com/tendermint/crypto
 )
 
 func Sha256(bytes []byte) []byte {
@@ -11,9 +9,3 @@ func Sha256(bytes []byte) []byte {
 	hasher.Write(bytes)
 	return hasher.Sum(nil)
 }
-
-func Ripemd160(bytes []byte) []byte {
-	hasher := ripemd160.New()
-	hasher.Write(bytes)
-	return hasher.Sum(nil)
-}

crypto/merkle/hash.go

@@ -0,0 +1,21 @@
+package merkle
+
+import (
+	"github.com/tendermint/tendermint/crypto/tmhash"
+)
+
+// TODO: make these have a large predefined capacity
+var (
+	leafPrefix  = []byte{0}
+	innerPrefix = []byte{1}
+)
+
+// returns tmhash(0x00 || leaf)
+func leafHash(leaf []byte) []byte {
+	return tmhash.Sum(append(leafPrefix, leaf...))
+}
+
+// returns tmhash(0x01 || left || right)
+func innerHash(left []byte, right []byte) []byte {
+	return tmhash.Sum(append(innerPrefix, append(left, right...)...))
+}

crypto/merkle/merkle.pb.go

@@ -39,7 +39,7 @@ func (m *ProofOp) Reset()         { *m = ProofOp{} }
 func (m *ProofOp) String() string { return proto.CompactTextString(m) }
 func (*ProofOp) ProtoMessage()    {}
 func (*ProofOp) Descriptor() ([]byte, []int) {
-	return fileDescriptor_merkle_5d3f6051907285da, []int{0}
+	return fileDescriptor_merkle_24be8bc4e405ac66, []int{0}
 }
 func (m *ProofOp) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -101,7 +101,7 @@ func (m *Proof) Reset()         { *m = Proof{} }
 func (m *Proof) String() string { return proto.CompactTextString(m) }
 func (*Proof) ProtoMessage()    {}
 func (*Proof) Descriptor() ([]byte, []int) {
-	return fileDescriptor_merkle_5d3f6051907285da, []int{1}
+	return fileDescriptor_merkle_24be8bc4e405ac66, []int{1}
 }
 func (m *Proof) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -395,6 +395,9 @@ func encodeVarintPopulateMerkle(dAtA []byte, v uint64) []byte {
 	return dAtA
 }
 func (m *ProofOp) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Type)
@@ -416,6 +419,9 @@ func (m *ProofOp) Size() (n int) {
 }
 
 func (m *Proof) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	if len(m.Ops) > 0 {
@@ -772,9 +778,9 @@ var (
 	ErrIntOverflowMerkle   = fmt.Errorf("proto: integer overflow")
 )
 
-func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_merkle_5d3f6051907285da) }
+func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_merkle_24be8bc4e405ac66) }
 
-var fileDescriptor_merkle_5d3f6051907285da = []byte{
+var fileDescriptor_merkle_24be8bc4e405ac66 = []byte{
 	// 200 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c,
 	0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25,

crypto/merkle/proof.go

@@ -43,6 +43,9 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er
 	for i, op := range poz {
 		key := op.GetKey()
 		if len(key) != 0 {
+			if len(keys) == 0 {
+				return cmn.NewError("Key path has insufficient # of parts: expected no more keys but got %+v", string(key))
+			}
 			lastKey := keys[len(keys)-1]
 			if !bytes.Equal(lastKey, key) {
 				return cmn.NewError("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key))
@@ -95,7 +98,7 @@ func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) {
 }
 
 func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) {
-	var poz ProofOperators
+	poz := make(ProofOperators, 0, len(proof.Ops))
 	for _, pop := range proof.Ops {
 		operator, err := prt.Decode(pop)
 		if err != nil {

crypto/merkle/proof_key_path_test.go

@@ -1,6 +1,8 @@
 package merkle
 
 import (
+	// it is ok to use math/rand here: we do not need a cryptographically secure random
+	// number generator here and we can run the tests a bit faster
 	"math/rand"
 	"testing"
 
@@ -24,7 +26,7 @@ func TestKeyPath(t *testing.T) {
 					keys[i][j] = alphanum[rand.Intn(len(alphanum))]
 				}
 			case KeyEncodingHex:
-				rand.Read(keys[i])
+				rand.Read(keys[i]) //nolint: gosec
 			default:
 				panic("Unexpected encoding")
 			}

crypto/merkle/proof_simple_value.go

@@ -71,11 +71,11 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) {
 	hasher.Write(value) // does not error
 	vhash := hasher.Sum(nil)
 
+	bz := new(bytes.Buffer)
 	// Wrap <op.Key, vhash> to hash the KVPair.
-	hasher = tmhash.New()
-	encodeByteSlice(hasher, []byte(op.key)) // does not error
-	encodeByteSlice(hasher, []byte(vhash))  // does not error
-	kvhash := hasher.Sum(nil)
+	encodeByteSlice(bz, []byte(op.key)) // does not error
+	encodeByteSlice(bz, []byte(vhash))  // does not error
+	kvhash := leafHash(bz.Bytes())
 
 	if !bytes.Equal(kvhash, op.Proof.LeafHash) {
 		return nil, cmn.NewError("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash)

crypto/merkle/proof_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
@@ -26,6 +26,7 @@ func NewDominoOp(key, input, output string) DominoOp {
 	}
 }
 
+//nolint:unused
 func DominoOpDecoder(pop ProofOp) (ProofOperator, error) {
 	if pop.Type != ProofOpDomino {
 		panic("unexpected proof op type")
@@ -107,6 +108,10 @@ func TestProofOperators(t *testing.T) {
 	err = popz.Verify(bz("OUTPUT4"), "//KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")})
 	assert.NotNil(t, err)
 
+	// BAD KEY 5
+	err = popz.Verify(bz("OUTPUT4"), "/KEY2/KEY1", [][]byte{bz("INPUT1")})
+	assert.NotNil(t, err)
+
 	// BAD OUTPUT 1
 	err = popz.Verify(bz("OUTPUT4_WRONG"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")})
 	assert.NotNil(t, err)

crypto/merkle/rfc6962_test.go

@@ -0,0 +1,97 @@
+package merkle
+
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// These tests were taken from https://github.com/google/trillian/blob/master/merkle/rfc6962/rfc6962_test.go,
+// and consequently fall under the above license.
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	"github.com/tendermint/tendermint/crypto/tmhash"
+)
+
+func TestRFC6962Hasher(t *testing.T) {
+	_, leafHashTrail := trailsFromByteSlices([][]byte{[]byte("L123456")})
+	leafHash := leafHashTrail.Hash
+	_, leafHashTrail = trailsFromByteSlices([][]byte{{}})
+	emptyLeafHash := leafHashTrail.Hash
+	for _, tc := range []struct {
+		desc string
+		got  []byte
+		want string
+	}{
+		// Since creating a merkle tree of no leaves is unsupported here, we skip
+		// the corresponding trillian test vector.
+
+		// Check that the empty hash is not the same as the hash of an empty leaf.
+		// echo -n 00 | xxd -r -p | sha256sum
+		{
+			desc: "RFC6962 Empty Leaf",
+			want: "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"[:tmhash.Size*2],
+			got:  emptyLeafHash,
+		},
+		// echo -n 004C313233343536 | xxd -r -p | sha256sum
+		{
+			desc: "RFC6962 Leaf",
+			want: "395aa064aa4c29f7010acfe3f25db9485bbd4b91897b6ad7ad547639252b4d56"[:tmhash.Size*2],
+			got:  leafHash,
+		},
+		// echo -n 014E3132334E343536 | xxd -r -p | sha256sum
+		{
+			desc: "RFC6962 Node",
+			want: "aa217fe888e47007fa15edab33c2b492a722cb106c64667fc2b044444de66bbb"[:tmhash.Size*2],
+			got:  innerHash([]byte("N123"), []byte("N456")),
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			wantBytes, err := hex.DecodeString(tc.want)
+			if err != nil {
+				t.Fatalf("hex.DecodeString(%x): %v", tc.want, err)
+			}
+			if got, want := tc.got, wantBytes; !bytes.Equal(got, want) {
+				t.Errorf("got %x, want %x", got, want)
+			}
+		})
+	}
+}
+
+func TestRFC6962HasherCollisions(t *testing.T) {
+	// Check that different leaves have different hashes.
+	leaf1, leaf2 := []byte("Hello"), []byte("World")
+	_, leafHashTrail := trailsFromByteSlices([][]byte{leaf1})
+	hash1 := leafHashTrail.Hash
+	_, leafHashTrail = trailsFromByteSlices([][]byte{leaf2})
+	hash2 := leafHashTrail.Hash
+	if bytes.Equal(hash1, hash2) {
+		t.Errorf("Leaf hashes should differ, but both are %x", hash1)
+	}
+	// Compute an intermediate subtree hash.
+	_, subHash1Trail := trailsFromByteSlices([][]byte{hash1, hash2})
+	subHash1 := subHash1Trail.Hash
+	// Check that this is not the same as a leaf hash of their concatenation.
+	preimage := append(hash1, hash2...)
+	_, forgedHashTrail := trailsFromByteSlices([][]byte{preimage})
+	forgedHash := forgedHashTrail.Hash
+	if bytes.Equal(subHash1, forgedHash) {
+		t.Errorf("Hasher is not second-preimage resistant")
+	}
+	// Swap the order of nodes and check that the hash is different.
+	_, subHash2Trail := trailsFromByteSlices([][]byte{hash2, hash1})
+	subHash2 := subHash2Trail.Hash
+	if bytes.Equal(subHash1, subHash2) {
+		t.Errorf("Subtree hash does not depend on the order of leaves")
+	}
+}

crypto/merkle/simple_map_test.go

@@ -13,14 +13,14 @@ func TestSimpleMap(t *testing.T) {
 		values []string // each string gets converted to []byte in test
 		want   string
 	}{
-		{[]string{"key1"}, []string{"value1"}, "321d150de16dceb51c72981b432b115045383259b1a550adf8dc80f927508967"},
-		{[]string{"key1"}, []string{"value2"}, "2a9e4baf321eac99f6eecc3406603c14bc5e85bb7b80483cbfc75b3382d24a2f"},
+		{[]string{"key1"}, []string{"value1"}, "a44d3cc7daba1a4600b00a2434b30f8b970652169810d6dfa9fb1793a2189324"},
+		{[]string{"key1"}, []string{"value2"}, "0638e99b3445caec9d95c05e1a3fc1487b4ddec6a952ff337080360b0dcc078c"},
 		// swap order with 2 keys
-		{[]string{"key1", "key2"}, []string{"value1", "value2"}, "c4d8913ab543ba26aa970646d4c99a150fd641298e3367cf68ca45fb45a49881"},
-		{[]string{"key2", "key1"}, []string{"value2", "value1"}, "c4d8913ab543ba26aa970646d4c99a150fd641298e3367cf68ca45fb45a49881"},
+		{[]string{"key1", "key2"}, []string{"value1", "value2"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"},
+		{[]string{"key2", "key1"}, []string{"value2", "value1"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"},
 		// swap order with 3 keys
-		{[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b23cef00eda5af4548a213a43793f2752d8d9013b3f2b64bc0523a4791196268"},
-		{[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b23cef00eda5af4548a213a43793f2752d8d9013b3f2b64bc0523a4791196268"},
+		{[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"},
+		{[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"},
 	}
 	for i, tc := range tests {
 		db := newSimpleMap()

crypto/merkle/simple_proof.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/tendermint/tendermint/crypto/tmhash"
 	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
@@ -67,7 +66,8 @@ func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[strin
 
 // Verify that the SimpleProof proves the root hash.
 // Check sp.Index/sp.Total manually if needed
-func (sp *SimpleProof) Verify(rootHash []byte, leafHash []byte) error {
+func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error {
+	leafHash := leafHash(leaf)
 	if sp.Total < 0 {
 		return errors.New("Proof total must be positive")
 	}
@@ -128,19 +128,19 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][
 		if len(innerHashes) == 0 {
 			return nil
 		}
-		numLeft := (total + 1) / 2
+		numLeft := getSplitPoint(total)
 		if index < numLeft {
 			leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1])
 			if leftHash == nil {
 				return nil
 			}
-			return simpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1])
+			return innerHash(leftHash, innerHashes[len(innerHashes)-1])
 		}
 		rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1])
 		if rightHash == nil {
 			return nil
 		}
-		return simpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash)
+		return innerHash(innerHashes[len(innerHashes)-1], rightHash)
 	}
 }
 
@@ -182,12 +182,13 @@ func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *Simp
 	case 0:
 		return nil, nil
 	case 1:
-		trail := &SimpleProofNode{tmhash.Sum(items[0]), nil, nil, nil}
+		trail := &SimpleProofNode{leafHash(items[0]), nil, nil, nil}
 		return []*SimpleProofNode{trail}, trail
 	default:
-		lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2])
-		rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:])
-		rootHash := simpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash)
+		k := getSplitPoint(len(items))
+		lefts, leftRoot := trailsFromByteSlices(items[:k])
+		rights, rightRoot := trailsFromByteSlices(items[k:])
+		rootHash := innerHash(leftRoot.Hash, rightRoot.Hash)
 		root := &SimpleProofNode{rootHash, nil, nil, nil}
 		leftRoot.Parent = root
 		leftRoot.Right = rightRoot

crypto/merkle/simple_tree.go

@@ -1,23 +1,9 @@
 package merkle
 
 import (
-	"github.com/tendermint/tendermint/crypto/tmhash"
+	"math/bits"
 )
 
-// simpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right).
-func simpleHashFromTwoHashes(left, right []byte) []byte {
-	var hasher = tmhash.New()
-	err := encodeByteSlice(hasher, left)
-	if err != nil {
-		panic(err)
-	}
-	err = encodeByteSlice(hasher, right)
-	if err != nil {
-		panic(err)
-	}
-	return hasher.Sum(nil)
-}
-
 // SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice,
 // in the provided order.
 func SimpleHashFromByteSlices(items [][]byte) []byte {
@@ -25,11 +11,83 @@ func SimpleHashFromByteSlices(items [][]byte) []byte {
 	case 0:
 		return nil
 	case 1:
-		return tmhash.Sum(items[0])
+		return leafHash(items[0])
 	default:
-		left := SimpleHashFromByteSlices(items[:(len(items)+1)/2])
-		right := SimpleHashFromByteSlices(items[(len(items)+1)/2:])
-		return simpleHashFromTwoHashes(left, right)
+		k := getSplitPoint(len(items))
+		left := SimpleHashFromByteSlices(items[:k])
+		right := SimpleHashFromByteSlices(items[k:])
+		return innerHash(left, right)
+	}
+}
+
+// SimpleHashFromByteSliceIterative is an iterative alternative to
+// SimpleHashFromByteSlice motivated by potential performance improvements.
+// (#2611) had suggested that an iterative version of
+// SimpleHashFromByteSlice would be faster, presumably because
+// we can envision some overhead accumulating from stack
+// frames and function calls. Additionally, a recursive algorithm risks
+// hitting the stack limit and causing a stack overflow should the tree
+// be too large.
+//
+// Provided here is an iterative alternative, a simple test to assert
+// correctness and a benchmark. On the performance side, there appears to
+// be no overall difference:
+//
+// BenchmarkSimpleHashAlternatives/recursive-4                20000 77677 ns/op
+// BenchmarkSimpleHashAlternatives/iterative-4                20000 76802 ns/op
+//
+// On the surface it might seem that the additional overhead is due to
+// the different allocation patterns of the implementations. The recursive
+// version uses a single [][]byte slices which it then re-slices at each level of the tree.
+// The iterative version reproduces [][]byte once within the function and
+// then rewrites sub-slices of that array at each level of the tree.
+//
+// Experimenting by modifying the code to simply calculate the
+// hash and not store the result show little to no difference in performance.
+//
+// These preliminary results suggest:
+//
+// 1. The performance of the SimpleHashFromByteSlice is pretty good
+// 2. Go has low overhead for recursive functions
+// 3. The performance of the SimpleHashFromByteSlice routine is dominated
+//    by the actual hashing of data
+//
+// Although this work is in no way exhaustive, point #3 suggests that
+// optimization of this routine would need to take an alternative
+// approach to make significant improvements on the current performance.
+//
+// Finally, considering that the recursive implementation is easier to
+// read, it might not be worthwhile to switch to a less intuitive
+// implementation for so little benefit.
+func SimpleHashFromByteSlicesIterative(input [][]byte) []byte {
+	items := make([][]byte, len(input))
+
+	for i, leaf := range input {
+		items[i] = leafHash(leaf)
+	}
+
+	size := len(items)
+	for {
+		switch size {
+		case 0:
+			return nil
+		case 1:
+			return items[0]
+		default:
+			rp := 0 // read position
+			wp := 0 // write position
+			for rp < size {
+				if rp+1 < size {
+					items[wp] = innerHash(items[rp], items[rp+1])
+					rp += 2
+				} else {
+					items[wp] = items[rp]
+					rp += 1
+				}
+				wp += 1
+			}
+			size = wp
+		}
 	}
 }
 
@@ -44,3 +102,17 @@ func SimpleHashFromMap(m map[string][]byte) []byte {
 	}
 	return sm.Hash()
 }
+
+// getSplitPoint returns the largest power of 2 less than length
+func getSplitPoint(length int) int {
+	if length < 1 {
+		panic("Trying to split a tree with size < 1")
+	}
+	uLength := uint(length)
+	bitlen := bits.Len(uLength)
+	k := 1 << uint(bitlen-1)
+	if k == length {
+		k >>= 1
+	}
+	return k
+}

crypto/merkle/simple_tree_test.go

@@ -34,7 +34,6 @@ func TestSimpleProof(t *testing.T) {
 
 	// For each item, check the trail.
 	for i, item := range items {
-		itemHash := tmhash.Sum(item)
 		proof := proofs[i]
 
 		// Check total/index
@@ -43,30 +42,89 @@ func TestSimpleProof(t *testing.T) {
 		require.Equal(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total)
 
 		// Verify success
-		err := proof.Verify(rootHash, itemHash)
-		require.NoError(t, err, "Verificatior failed: %v.", err)
+		err := proof.Verify(rootHash, item)
+		require.NoError(t, err, "Verification failed: %v.", err)
 
 		// Trail too long should make it fail
 		origAunts := proof.Aunts
 		proof.Aunts = append(proof.Aunts, cmn.RandBytes(32))
-		err = proof.Verify(rootHash, itemHash)
+		err = proof.Verify(rootHash, item)
 		require.Error(t, err, "Expected verification to fail for wrong trail length")
 
 		proof.Aunts = origAunts
 
 		// Trail too short should make it fail
 		proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1]
-		err = proof.Verify(rootHash, itemHash)
+		err = proof.Verify(rootHash, item)
 		require.Error(t, err, "Expected verification to fail for wrong trail length")
 
 		proof.Aunts = origAunts
 
 		// Mutating the itemHash should make it fail.
-		err = proof.Verify(rootHash, MutateByteSlice(itemHash))
+		err = proof.Verify(rootHash, MutateByteSlice(item))
 		require.Error(t, err, "Expected verification to fail for mutated leaf hash")
 
 		// Mutating the rootHash should make it fail.
-		err = proof.Verify(MutateByteSlice(rootHash), itemHash)
+		err = proof.Verify(MutateByteSlice(rootHash), item)
 		require.Error(t, err, "Expected verification to fail for mutated root hash")
 	}
 }
+
+func TestSimpleHashAlternatives(t *testing.T) {
+
+	total := 100
+
+	items := make([][]byte, total)
+	for i := 0; i < total; i++ {
+		items[i] = testItem(cmn.RandBytes(tmhash.Size))
+	}
+
+	rootHash1 := SimpleHashFromByteSlicesIterative(items)
+	rootHash2 := SimpleHashFromByteSlices(items)
+	require.Equal(t, rootHash1, rootHash2, "Unmatched root hashes: %X vs %X", rootHash1, rootHash2)
+}
+
+func BenchmarkSimpleHashAlternatives(b *testing.B) {
+	total := 100
+
+	items := make([][]byte, total)
+	for i := 0; i < total; i++ {
+		items[i] = testItem(cmn.RandBytes(tmhash.Size))
+	}
+
+	b.ResetTimer()
+	b.Run("recursive", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_ = SimpleHashFromByteSlices(items)
+		}
+	})
+
+	b.Run("iterative", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_ = SimpleHashFromByteSlicesIterative(items)
+		}
+	})
+}
+
+func Test_getSplitPoint(t *testing.T) {
+	tests := []struct {
+		length int
+		want   int
+	}{
+		{1, 0},
+		{2, 1},
+		{3, 2},
+		{4, 2},
+		{5, 4},
+		{10, 8},
+		{20, 16},
+		{100, 64},
+		{255, 128},
+		{256, 128},
+		{257, 256},
+	}
+	for _, tt := range tests {
+		got := getSplitPoint(tt.length)
+		require.Equal(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want)
+	}
+}

crypto/merkle/wire.go

@@ -1,7 +1,7 @@
 package merkle
 
 import (
-	"github.com/tendermint/go-amino"
+	amino "github.com/tendermint/go-amino"
 )
 
 var cdc *amino.Codec

crypto/multisig/multisignature.go

@@ -1,7 +1,8 @@
 package multisig
 
 import (
-	"errors"
+	"fmt"
+	"strings"
 
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/multisig/bitarray"
@@ -53,13 +54,19 @@ func (mSig *Multisignature) AddSignature(sig []byte, index int) {
 	mSig.Sigs[newSigIndex] = sig
 }
 
-// AddSignatureFromPubKey adds a signature to the multisig,
-// at the index in keys corresponding to the provided pubkey.
+// AddSignatureFromPubKey adds a signature to the multisig, at the index in
+// keys corresponding to the provided pubkey.
 func (mSig *Multisignature) AddSignatureFromPubKey(sig []byte, pubkey crypto.PubKey, keys []crypto.PubKey) error {
 	index := getIndex(pubkey, keys)
 	if index == -1 {
-		return errors.New("provided key didn't exist in pubkeys")
+		keysStr := make([]string, len(keys))
+		for i, k := range keys {
+			keysStr[i] = fmt.Sprintf("%X", k.Bytes())
+		}
+
+		return fmt.Errorf("provided key %X doesn't exist in pubkeys: \n%s", pubkey.Bytes(), strings.Join(keysStr, "\n"))
 	}
+
 	mSig.AddSignature(sig, index)
 	return nil
 }

crypto/multisig/threshold_pubkey.go

@@ -2,7 +2,6 @@ package multisig
 
 import (
 	"github.com/tendermint/tendermint/crypto"
-	"github.com/tendermint/tendermint/crypto/tmhash"
 )
 
 // PubKeyMultisigThreshold implements a K of N threshold multisig.
@@ -11,7 +10,7 @@ type PubKeyMultisigThreshold struct {
 	PubKeys []crypto.PubKey `json:"pubkeys"`
 }
 
-var _ crypto.PubKey = &PubKeyMultisigThreshold{}
+var _ crypto.PubKey = PubKeyMultisigThreshold{}
 
 // NewPubKeyMultisigThreshold returns a new PubKeyMultisigThreshold.
 // Panics if len(pubkeys) < k or 0 >= k.
@@ -22,7 +21,7 @@ func NewPubKeyMultisigThreshold(k int, pubkeys []crypto.PubKey) crypto.PubKey {
 	if len(pubkeys) < k {
 		panic("threshold k of n multisignature: len(pubkeys) < k")
 	}
-	return &PubKeyMultisigThreshold{uint(k), pubkeys}
+	return PubKeyMultisigThreshold{uint(k), pubkeys}
 }
 
 // VerifyBytes expects sig to be an amino encoded version of a MultiSignature.
@@ -31,8 +30,8 @@ func NewPubKeyMultisigThreshold(k int, pubkeys []crypto.PubKey) crypto.PubKey {
 // and all signatures are valid. (Not just k of the signatures)
 // The multisig uses a bitarray, so multiple signatures for the same key is not
 // a concern.
-func (pk *PubKeyMultisigThreshold) VerifyBytes(msg []byte, marshalledSig []byte) bool {
-	var sig *Multisignature
+func (pk PubKeyMultisigThreshold) VerifyBytes(msg []byte, marshalledSig []byte) bool {
+	var sig Multisignature
 	err := cdc.UnmarshalBinaryBare(marshalledSig, &sig)
 	if err != nil {
 		return false
@@ -64,19 +63,19 @@ func (pk *PubKeyMultisigThreshold) VerifyBytes(msg []byte, marshalledSig []byte)
 }
 
 // Bytes returns the amino encoded version of the PubKeyMultisigThreshold
-func (pk *PubKeyMultisigThreshold) Bytes() []byte {
+func (pk PubKeyMultisigThreshold) Bytes() []byte {
 	return cdc.MustMarshalBinaryBare(pk)
 }
 
 // Address returns tmhash(PubKeyMultisigThreshold.Bytes())
-func (pk *PubKeyMultisigThreshold) Address() crypto.Address {
-	return crypto.Address(tmhash.Sum(pk.Bytes()))
+func (pk PubKeyMultisigThreshold) Address() crypto.Address {
+	return crypto.AddressHash(pk.Bytes())
 }
 
 // Equals returns true iff pk and other both have the same number of keys, and
 // all constituent keys are the same, and in the same order.
-func (pk *PubKeyMultisigThreshold) Equals(other crypto.PubKey) bool {
-	otherKey, sameType := other.(*PubKeyMultisigThreshold)
+func (pk PubKeyMultisigThreshold) Equals(other crypto.PubKey) bool {
+	otherKey, sameType := other.(PubKeyMultisigThreshold)
 	if !sameType {
 		return false
 	}

crypto/multisig/threshold_pubkey_test.go

@@ -36,30 +36,68 @@ func TestThresholdMultisigValidCases(t *testing.T) {
 	for tcIndex, tc := range cases {
 		multisigKey := NewPubKeyMultisigThreshold(tc.k, tc.pubkeys)
 		multisignature := NewMultisig(len(tc.pubkeys))
+
 		for i := 0; i < tc.k-1; i++ {
 			signingIndex := tc.signingIndices[i]
-			multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys)
-			require.False(t, multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
-				"multisig passed when i < k, tc %d, i %d", tcIndex, i)
-			multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys)
-			require.Equal(t, i+1, len(multisignature.Sigs),
-				"adding a signature for the same pubkey twice increased signature count by 2, tc %d", tcIndex)
+			require.NoError(
+				t,
+				multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys),
+			)
+			require.False(
+				t,
+				multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
+				"multisig passed when i < k, tc %d, i %d", tcIndex, i,
+			)
+			require.NoError(
+				t,
+				multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys),
+			)
+			require.Equal(
+				t,
+				i+1,
+				len(multisignature.Sigs),
+				"adding a signature for the same pubkey twice increased signature count by 2, tc %d", tcIndex,
+			)
 		}
-		require.False(t, multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
-			"multisig passed with k - 1 sigs, tc %d", tcIndex)
-		multisignature.AddSignatureFromPubKey(tc.signatures[tc.signingIndices[tc.k]], tc.pubkeys[tc.signingIndices[tc.k]], tc.pubkeys)
-		require.True(t, multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
-			"multisig failed after k good signatures, tc %d", tcIndex)
+
+		require.False(
+			t,
+			multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
+			"multisig passed with k - 1 sigs, tc %d", tcIndex,
+		)
+		require.NoError(
+			t,
+			multisignature.AddSignatureFromPubKey(tc.signatures[tc.signingIndices[tc.k]], tc.pubkeys[tc.signingIndices[tc.k]], tc.pubkeys),
+		)
+		require.True(
+			t,
+			multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
+			"multisig failed after k good signatures, tc %d", tcIndex,
+		)
+
 		for i := tc.k + 1; i < len(tc.signingIndices); i++ {
 			signingIndex := tc.signingIndices[i]
-			multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys)
-			require.Equal(t, tc.passAfterKSignatures[i-tc.k-1],
-				multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
-				"multisig didn't verify as expected after k sigs, tc %d, i %d", tcIndex, i)
 
-			multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys)
-			require.Equal(t, i+1, len(multisignature.Sigs),
-				"adding a signature for the same pubkey twice increased signature count by 2, tc %d", tcIndex)
+			require.NoError(
+				t,
+				multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys),
+			)
+			require.Equal(
+				t,
+				tc.passAfterKSignatures[i-tc.k-1],
+				multisigKey.VerifyBytes(tc.msg, multisignature.Marshal()),
+				"multisig didn't verify as expected after k sigs, tc %d, i %d", tcIndex, i,
+			)
+			require.NoError(
+				t,
+				multisignature.AddSignatureFromPubKey(tc.signatures[signingIndex], tc.pubkeys[signingIndex], tc.pubkeys),
+			)
+			require.Equal(
+				t,
+				i+1,
+				len(multisignature.Sigs),
+				"adding a signature for the same pubkey twice increased signature count by 2, tc %d", tcIndex,
+			)
 		}
 	}
 }
@@ -82,7 +120,7 @@ func TestMultiSigPubKeyEquality(t *testing.T) {
 	msg := []byte{1, 2, 3, 4}
 	pubkeys, _ := generatePubKeysAndSignatures(5, msg)
 	multisigKey := NewPubKeyMultisigThreshold(2, pubkeys)
-	var unmarshalledMultisig *PubKeyMultisigThreshold
+	var unmarshalledMultisig PubKeyMultisigThreshold
 	cdc.MustUnmarshalBinaryBare(multisigKey.Bytes(), &unmarshalledMultisig)
 	require.True(t, multisigKey.Equals(unmarshalledMultisig))
 
@@ -95,6 +133,29 @@ func TestMultiSigPubKeyEquality(t *testing.T) {
 	require.False(t, multisigKey.Equals(multisigKey2))
 }
 
+func TestAddress(t *testing.T) {
+	msg := []byte{1, 2, 3, 4}
+	pubkeys, _ := generatePubKeysAndSignatures(5, msg)
+	multisigKey := NewPubKeyMultisigThreshold(2, pubkeys)
+	require.Len(t, multisigKey.Address().Bytes(), 20)
+}
+
+func TestPubKeyMultisigThresholdAminoToIface(t *testing.T) {
+	msg := []byte{1, 2, 3, 4}
+	pubkeys, _ := generatePubKeysAndSignatures(5, msg)
+	multisigKey := NewPubKeyMultisigThreshold(2, pubkeys)
+
+	ab, err := cdc.MarshalBinaryLengthPrefixed(multisigKey)
+	require.NoError(t, err)
+	// like other crypto.Pubkey implementations (e.g. ed25519.PubKeyEd25519),
+	// PubKeyMultisigThreshold should be deserializable into a crypto.PubKey:
+	var pubKey crypto.PubKey
+	err = cdc.UnmarshalBinaryLengthPrefixed(ab, &pubKey)
+	require.NoError(t, err)
+
+	require.Equal(t, multisigKey, pubKey)
+}
+
 func generatePubKeysAndSignatures(n int, msg []byte) (pubkeys []crypto.PubKey, signatures [][]byte) {
 	pubkeys = make([]crypto.PubKey, n)
 	signatures = make([][]byte, n)

crypto/multisig/wire.go

@@ -20,7 +20,7 @@ func init() {
 	cdc.RegisterConcrete(PubKeyMultisigThreshold{},
 		PubKeyMultisigThresholdAminoRoute, nil)
 	cdc.RegisterConcrete(ed25519.PubKeyEd25519{},
-		ed25519.PubKeyAminoRoute, nil)
+		ed25519.PubKeyAminoName, nil)
 	cdc.RegisterConcrete(secp256k1.PubKeySecp256k1{},
-		secp256k1.PubKeyAminoRoute, nil)
+		secp256k1.PubKeyAminoName, nil)
 }

crypto/random.go

@@ -1,42 +1,11 @@
 package crypto
 
 import (
-	"crypto/cipher"
 	crand "crypto/rand"
-	"crypto/sha256"
 	"encoding/hex"
 	"io"
-	"sync"
-
-	"golang.org/x/crypto/chacha20poly1305"
 )
 
-// NOTE: This is ignored for now until we have time
-// to properly review the MixEntropy function - https://github.com/tendermint/tendermint/issues/2099.
-//
-// The randomness here is derived from xoring a chacha20 keystream with
-// output from crypto/rand's OS Entropy Reader. (Due to fears of the OS'
-// entropy being backdoored)
-//
-// For forward secrecy of produced randomness, the internal chacha key is hashed
-// and thereby rotated after each call.
-var gRandInfo *randInfo
-
-func init() {
-	gRandInfo = &randInfo{}
-
-	// TODO: uncomment after reviewing MixEntropy -
-	// https://github.com/tendermint/tendermint/issues/2099
-	// gRandInfo.MixEntropy(randBytes(32)) // Init
-}
-
-// WARNING: This function needs review - https://github.com/tendermint/tendermint/issues/2099.
-// Mix additional bytes of randomness, e.g. from hardware, user-input, etc.
-// It is OK to call it multiple times.  It does not diminish security.
-func MixEntropy(seedBytes []byte) {
-	gRandInfo.MixEntropy(seedBytes)
-}
-
 // This only uses the OS's randomness
 func randBytes(numBytes int) []byte {
 	b := make([]byte, numBytes)
@@ -52,19 +21,6 @@ func CRandBytes(numBytes int) []byte {
 	return randBytes(numBytes)
 }
 
-/* TODO: uncomment after reviewing MixEntropy - https://github.com/tendermint/tendermint/issues/2099
-// This uses the OS and the Seed(s).
-func CRandBytes(numBytes int) []byte {
-	return randBytes(numBytes)
-		b := make([]byte, numBytes)
-		_, err := gRandInfo.Read(b)
-		if err != nil {
-			panic(err)
-		}
-		return b
-}
-*/
-
 // CRandHex returns a hex encoded string that's floor(numDigits/2) * 2 long.
 //
 // Note: CRandHex(24) gives 96 bits of randomness that
@@ -77,63 +33,3 @@ func CRandHex(numDigits int) string {
 func CReader() io.Reader {
 	return crand.Reader
 }
-
-/* TODO: uncomment after reviewing MixEntropy - https://github.com/tendermint/tendermint/issues/2099
-// Returns a crand.Reader mixed with user-supplied entropy
-func CReader() io.Reader {
-	return gRandInfo
-}
-*/
-
-//--------------------------------------------------------------------------------
-
-type randInfo struct {
-	mtx       sync.Mutex
-	seedBytes [chacha20poly1305.KeySize]byte
-	chacha    cipher.AEAD
-	reader    io.Reader
-}
-
-// You can call this as many times as you'd like.
-// XXX/TODO: review - https://github.com/tendermint/tendermint/issues/2099
-func (ri *randInfo) MixEntropy(seedBytes []byte) {
-	ri.mtx.Lock()
-	defer ri.mtx.Unlock()
-	// Make new ri.seedBytes using passed seedBytes and current ri.seedBytes:
-	// ri.seedBytes = sha256( seedBytes || ri.seedBytes )
-	h := sha256.New()
-	h.Write(seedBytes)
-	h.Write(ri.seedBytes[:])
-	hashBytes := h.Sum(nil)
-	copy(ri.seedBytes[:], hashBytes)
-	chacha, err := chacha20poly1305.New(ri.seedBytes[:])
-	if err != nil {
-		panic("Initializing chacha20 failed")
-	}
-	ri.chacha = chacha
-	// Create new reader
-	ri.reader = &cipher.StreamReader{S: ri, R: crand.Reader}
-}
-
-func (ri *randInfo) XORKeyStream(dst, src []byte) {
-	// nonce being 0 is safe due to never re-using a key.
-	emptyNonce := make([]byte, 12)
-	tmpDst := ri.chacha.Seal([]byte{}, emptyNonce, src, []byte{0})
-	// this removes the poly1305 tag as well, since chacha is a stream cipher
-	// and we truncate at input length.
-	copy(dst, tmpDst[:len(src)])
-	// hash seedBytes for forward secrecy, and initialize new chacha instance
-	newSeed := sha256.Sum256(ri.seedBytes[:])
-	chacha, err := chacha20poly1305.New(newSeed[:])
-	if err != nil {
-		panic("Initializing chacha20 failed")
-	}
-	ri.chacha = chacha
-}
-
-func (ri *randInfo) Read(b []byte) (n int, err error) {
-	ri.mtx.Lock()
-	n, err = ri.reader.Read(b)
-	ri.mtx.Unlock()
-	return
-}

crypto/secp256k1/internal/secp256k1/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+
+*~

crypto/secp256k1/internal/secp256k1/LICENSE

@@ -0,0 +1,31 @@
+Copyright (c) 2010 The Go Authors. All rights reserved.
+Copyright (c) 2011 ThePiachu. All rights reserved.
+Copyright (c) 2015 Jeffrey Wilcke. All rights reserved.
+Copyright (c) 2015 Felix Lange. All rights reserved.
+Copyright (c) 2015 Gustav Simonsson. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of the copyright holder. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

crypto/secp256k1/internal/secp256k1/README.md

@@ -0,0 +1,3 @@
+This package is copied from https://github.com/ethereum/go-ethereum/tree/729bf365b5f17325be9107b63b233da54100eec6/crypto/secp256k1
+
+Unlike the rest of go-ethereum it is MIT licensed so compatible with our Apache2.0 license. We opt to copy in here rather than depend on go-ethereum to avoid issues with vendoring of the GPL parts of that repository by downstream.