Merge pull request #2152 from tendermint/release/v0.23.0

Release/v0.23.0

.circleci/config.yml

@@ -16,7 +16,7 @@ jobs:
       - checkout
       - restore_cache:
           keys:
-            - v2-pkg-cache
+            - v3-pkg-cache
       - run:
           name: tools
           command: |
@@ -38,11 +38,11 @@ jobs:
             - bin
             - profiles
       - save_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
           paths:
             - /go/pkg
       - save_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
           paths:
             - /go/src/github.com/tendermint/tendermint
 
@@ -52,9 +52,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: slate docs
           command: |
@@ -68,15 +68,21 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: metalinter
           command: |
             set -ex
             export PATH="$GOBIN:$PATH"
             make metalinter
+      - run:
+          name: check_dep
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make check_dep
 
   test_abci_apps:
     <<: *defaults
@@ -84,9 +90,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run abci apps tests
           command: |
@@ -101,9 +107,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run abci-cli tests
           command: |
@@ -116,9 +122,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils
       - run:
           name: Run tests
@@ -131,17 +137,17 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run: mkdir -p /tmp/logs
       - run:
           name: Run tests
           command: |
-            for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
+            for pkg in $(go list github.com/tendermint/tendermint/... | circleci tests split --split-by=timings); do
               id=$(basename "$pkg")
 
-              GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
+              GOCACHE=off go test -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
             done
       - persist_to_workspace:
           root: /tmp/workspace
@@ -156,9 +162,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run tests
           command: bash test/persist/test_failure_indices.sh
@@ -181,7 +187,7 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: gather
           command: |

.github/ISSUE_TEMPLATE

@@ -1,42 +0,0 @@
-<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.-->
-
-**Is this a BUG REPORT or FEATURE REQUEST?** (choose one):
-
-<!--
-If this is a BUG REPORT, please:
-  - Fill in as much of the template below as you can.
-
-If this is a FEATURE REQUEST, please:
-  - Describe *in detail* the feature/behavior/change you'd like to see.
-
-In both cases, be ready for followup questions, and please respond in a timely
-manner. We might ask you to provide additional logs and data (tendermint & app)
-in a case of bug.
--->
-
-**Tendermint version** (use `tendermint version` or `git rev-parse --verify HEAD` if installed from source):
-
-
-**ABCI app** (name for built-in, URL for self-written if it's publicly available):
-
-**Environment**:
-- **OS** (e.g. from /etc/os-release):
-- **Install tools**:
-- **Others**:
-
-
-**What happened**:
-
-
-**What you expected to happen**:
-
-
-**How to reproduce it** (as minimally and precisely as possible):
-
-**Logs (you can paste a small part showing an error or link a pastebin, gist, etc. containing more of the log file)**:
-
-**Config (you can paste only the changes you've made)**:
-
-**`/dump_consensus_state` output for consensus bugs**
-
-**Anything else do we need to know**:

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,42 @@
+---
+name: Bug Report
+about: Create a report to help us squash bugs!
+
+---
+<!--
+Please fill in as much of the template below as you can.
+
+Be ready for followup questions, and please respond in a timely
+manner. We might ask you to provide additional logs and data (tendermint & app).
+-->
+
+**Tendermint version** (use `tendermint version` or `git rev-parse --verify HEAD` if installed from source):
+
+
+**ABCI app** (name for built-in, URL for self-written if it's publicly available):
+
+**Environment**:
+- **OS** (e.g. from /etc/os-release):
+- **Install tools**:
+- **Others**:
+
+
+**What happened**:
+
+
+**What you expected to happen**:
+
+
+**Have you tried the latest version**: yes/no
+
+**How to reproduce it** (as minimally and precisely as possible):
+
+**Logs (paste a small part showing an error (< 10 lines) or link a pastebin, gist, etc. containing more of the log file)**:
+
+**Config (you can paste only the changes you've made)**:
+
+**node command runtime flags**:
+
+**`/dump_consensus_state` output for consensus bugs**
+
+**Anything else we need to know**:

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,13 @@
+---
+name: Feature Request
+about: Create a proposal to request a feature
+
+---
+<!--
+Please describe *in detail* the feature/behavior/change you'd like to see.
+
+Be ready for followup questions, and please respond in a timely
+manner. 
+
+Word of caution: poorly thought out proposals may be rejected without deliberation 
+-->

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,4 +3,4 @@
 * [ ] Updated all relevant documentation in docs
 * [ ] Updated all code comments where relevant
 * [ ] Wrote tests
-* [ ] Updated CHANGELOG.md
+* [ ] Updated CHANGELOG_PENDING.md

.gitignore

@@ -16,8 +16,8 @@ coverage.txt
 docs/_build
 *.log
 abci-cli
-abci/types/types.pb.go
 docs/node_modules/
+index.html.md
 
 scripts/wal2json/wal2json
 scripts/cutWALUntil/cutWALUntil
@@ -27,3 +27,10 @@ scripts/cutWALUntil/cutWALUntil
 
 libs/pubsub/query/fuzz_test/output
 shunit2
+
+*/vendor
+*/.glide
+.terraform
+terraform.tfstate
+terraform.tfstate.backup
+terraform.tfstate.d

CHANGELOG.md

@@ -1,5 +1,134 @@
 # Changelog
 
+## 0.23.0
+
+*August 5th, 2018*
+
+This release includes breaking upgrades in our P2P encryption,
+some ABCI messages, and how we encode time and signatures.
+
+A few more changes are still coming to the Header, ABCI,
+and validator set handling to better support light clients, BFT time, and
+upgrades. Most notably, validator set changes will be delayed by one block (see
+[#1815][i1815]).
+
+We also removed `make ensure_deps` in favour of `make get_vendor_deps`.
+
+BREAKING CHANGES:
+- [abci] Changed time format from int64 to google.protobuf.Timestamp
+- [abci] Changed Validators to LastCommitInfo in RequestBeginBlock
+- [abci] Removed Fee from ResponseDeliverTx and ResponseCheckTx
+- [crypto] Switch crypto.Signature from interface to []byte for space efficiency
+  [#2128](https://github.com/tendermint/tendermint/pull/2128)
+    - NOTE: this means signatures no longer have the prefix bytes in Amino
+      binary nor the `type` field in Amino JSON. They're just bytes.
+- [p2p] Remove salsa and ripemd primitives, in favor of using chacha as a stream cipher, and hkdf [#2054](https://github.com/tendermint/tendermint/pull/2054)
+- [tools] Removed `make ensure_deps` in favor of `make get_vendor_deps`
+- [types] CanonicalTime uses nanoseconds instead of clipping to ms
+    - breaks serialization/signing of all messages with a timestamp
+
+FEATURES:
+- [tools] Added `make check_dep`
+    - ensures gopkg.lock is synced with gopkg.toml
+    - ensures no branches are used in the gopkg.toml
+
+IMPROVEMENTS:
+- [blockchain] Improve fast-sync logic
+  [#1805](https://github.com/tendermint/tendermint/pull/1805)
+    - tweak params
+    - only process one block at a time to avoid starving
+- [common] bit array functions which take in another parameter are now thread safe
+- [crypto] Switch hkdfchachapoly1305 to xchachapoly1305
+- [p2p] begin connecting to peers as soon a seed node provides them to you ([#2093](https://github.com/tendermint/tendermint/issues/2093))
+
+BUG FIXES:
+- [common] Safely handle cases where atomic write files already exist [#2109](https://github.com/tendermint/tendermint/issues/2109)
+- [privval] fix a deadline for accepting new connections in socket private
+  validator.
+- [p2p] Allow startup if a configured seed node's IP can't be resolved ([#1716](https://github.com/tendermint/tendermint/issues/1716))
+- [node] Fully exit when CTRL-C is pressed even if consensus state panics [#2072](https://github.com/tendermint/tendermint/issues/2072)
+
+[i1815]: https://github.com/tendermint/tendermint/pull/1815
+
+## 0.22.8
+
+*July 26th, 2018*
+
+BUG FIXES
+
+- [consensus, blockchain] Fix 0.22.7 below.
+
+## 0.22.7
+
+*July 26th, 2018*
+
+BUG FIXES
+
+- [consensus, blockchain] Register the Evidence interface so it can be
+  marshalled/unmarshalled by the blockchain and consensus reactors
+
+## 0.22.6
+
+*July 24th, 2018*
+
+BUG FIXES
+
+- [rpc] Fix `/blockchain` endpoint
+    - (#2049) Fix OOM attack by returning error on negative input
+    - Fix result length to have max 20 (instead of 21) block metas
+- [rpc] Validate height is non-negative in `/abci_query`
+- [consensus] (#2050) Include evidence in proposal block parts (previously evidence was
+  not being included in blocks!)
+- [p2p] (#2046) Close rejected inbound connections so file descriptor doesn't
+  leak
+- [Gopkg] (#2053) Fix versions in the toml
+
+## 0.22.5
+
+*July 23th, 2018*
+
+BREAKING CHANGES:
+- [crypto] Refactor `tendermint/crypto` into many subpackages
+- [libs/common] remove exponentially distributed random numbers
+
+IMPROVEMENTS:
+- [abci, libs/common] Generated gogoproto static marshaller methods
+- [config] Increase default send/recv rates to 5 mB/s
+- [p2p] reject addresses coming from private peers
+- [p2p] allow persistent peers to be private
+
+BUG FIXES:
+- [mempool] fixed a race condition when `create_empty_blocks=false` where a
+  transaction is published at an old height.
+- [p2p] dial external IP setup by `persistent_peers`, not internal NAT IP
+- [rpc] make `/status` RPC endpoint resistant to consensus halt
+
+## 0.22.4
+
+*July 14th, 2018*
+
+BREAKING CHANGES:
+- [genesis] removed deprecated `app_options` field.
+- [types] Genesis.AppStateJSON -> Genesis.AppState
+
+FEATURES:
+- [tools] Merged in from github.com/tendermint/tools
+
+BUG FIXES:
+- [tools/tm-bench] Various fixes
+- [consensus] Wait for WAL to stop on shutdown
+- [abci] Fix #1891, pending requests cannot hang when abci server dies.
+  Previously a crash in BeginBlock could leave tendermint in broken state.
+
+## 0.22.3
+
+*July 10th, 2018*
+
+IMPROVEMENTS
+- Update dependencies
+    * pin all values in Gopkg.toml to version or commit
+    * update golang/protobuf to v1.1.0
+
 ## 0.22.2
 
 *July 10th, 2018*
@@ -32,6 +161,7 @@ BUG FIXES
   already in the validator set.
 * [consensus] Shut down WAL properly.
 
+
 ## 0.22.0
 
 *July 2nd, 2018*

Gopkg.lock

@@ -3,50 +3,63 @@
 
 [[projects]]
   branch = "master"
+  digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d"
   name = "github.com/beorn7/perks"
   packages = ["quantile"]
+  pruneopts = "UT"
   revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
 
 [[projects]]
   branch = "master"
+  digest = "1:2c00f064ba355903866cbfbf3f7f4c0fe64af6638cc7d1b8bdcf3181bc67f1d8"
   name = "github.com/btcsuite/btcd"
   packages = ["btcec"]
-  revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64"
+  pruneopts = "UT"
+  revision = "f5e261fc9ec3437697fb31d8b38453c293204b29"
 
 [[projects]]
-  branch = "master"
+  digest = "1:1d8e1cb71c33a9470bbbae09bfec09db43c6bf358dfcae13cd8807c4e2a9a2bf"
   name = "github.com/btcsuite/btcutil"
   packages = [
     "base58",
-    "bech32"
+    "bech32",
   ]
+  pruneopts = "UT"
   revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4"
 
 [[projects]]
+  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
   name = "github.com/davecgh/go-spew"
   packages = ["spew"]
+  pruneopts = "UT"
   revision = "346938d642f2ec3594ed81d874461961cd0faa76"
   version = "v1.1.0"
 
 [[projects]]
-  branch = "master"
+  digest = "1:c7644c73a3d23741fdba8a99b1464e021a224b7e205be497271a8003a15ca41b"
   name = "github.com/ebuchman/fail-test"
   packages = ["."]
+  pruneopts = "UT"
   revision = "95f809107225be108efcf10a3509e4ea6ceef3c4"
 
 [[projects]]
-  branch = "master"
+  digest = "1:544229a3ca0fb2dd5ebc2896d3d2ff7ce096d9751635301e44e37e761349ee70"
   name = "github.com/fortytw2/leaktest"
   packages = ["."]
-  revision = "b008db64ef8daabb22ff6daa557f33b41d8f6ccd"
+  pruneopts = "UT"
+  revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e"
+  version = "v1.2.0"
 
 [[projects]]
+  digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd"
   name = "github.com/fsnotify/fsnotify"
   packages = ["."]
+  pruneopts = "UT"
   revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
   version = "v1.4.7"
 
 [[projects]]
+  digest = "1:fdf5169073fb0ad6dc12a70c249145e30f4058647bea25f0abd48b6d9f228a11"
   name = "github.com/go-kit/kit"
   packages = [
     "log",
@@ -55,24 +68,30 @@
     "metrics",
     "metrics/discard",
     "metrics/internal/lv",
-    "metrics/prometheus"
+    "metrics/prometheus",
   ]
+  pruneopts = "UT"
   revision = "4dc7be5d2d12881735283bcab7352178e190fc71"
   version = "v0.6.0"
 
 [[projects]]
+  digest = "1:31a18dae27a29aa074515e43a443abfd2ba6deb6d69309d8d7ce789c45f34659"
   name = "github.com/go-logfmt/logfmt"
   packages = ["."]
+  pruneopts = "UT"
   revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
   version = "v0.3.0"
 
 [[projects]]
+  digest = "1:c4a2528ccbcabf90f9f3c464a5fc9e302d592861bbfd0b7135a7de8a943d0406"
   name = "github.com/go-stack/stack"
   packages = ["."]
+  pruneopts = "UT"
   revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc"
   version = "v1.7.0"
 
 [[projects]]
+  digest = "1:35621fe20f140f05a0c4ef662c26c0ab4ee50bca78aa30fe87d33120bd28165e"
   name = "github.com/gogo/protobuf"
   packages = [
     "gogoproto",
@@ -80,37 +99,45 @@
     "proto",
     "protoc-gen-gogo/descriptor",
     "sortkeys",
-    "types"
+    "types",
   ]
-  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
-  version = "v1.0.0"
+  pruneopts = "UT"
+  revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
+  version = "v1.1.1"
 
 [[projects]]
+  digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260"
   name = "github.com/golang/protobuf"
   packages = [
     "proto",
     "ptypes",
     "ptypes/any",
     "ptypes/duration",
-    "ptypes/timestamp"
+    "ptypes/timestamp",
   ]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
+  pruneopts = "UT"
+  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
+  digest = "1:4a0c6bb4805508a6287675fac876be2ac1182539ca8a32468d8128882e9d5009"
   name = "github.com/golang/snappy"
   packages = ["."]
+  pruneopts = "UT"
   revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
 
 [[projects]]
+  digest = "1:43dd08a10854b2056e615d1b1d22ac94559d822e1f8b6fcc92c1a1057e85188e"
   name = "github.com/gorilla/websocket"
   packages = ["."]
+  pruneopts = "UT"
   revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
   version = "v1.2.0"
 
 [[projects]]
   branch = "master"
+  digest = "1:12247a2e99a060cc692f6680e5272c8adf0b8f572e6bce0d7095e624c958a240"
   name = "github.com/hashicorp/hcl"
   packages = [
     ".",
@@ -121,156 +148,197 @@
     "hcl/token",
     "json/parser",
     "json/scanner",
-    "json/token"
+    "json/token",
   ]
+  pruneopts = "UT"
   revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168"
 
 [[projects]]
+  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
   name = "github.com/inconshreveable/mousetrap"
   packages = ["."]
+  pruneopts = "UT"
   revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
   version = "v1.0"
 
 [[projects]]
-  branch = "master"
+  digest = "1:39b27d1381a30421f9813967a5866fba35dc1d4df43a6eefe3b7a5444cb07214"
   name = "github.com/jmhodges/levigo"
   packages = ["."]
+  pruneopts = "UT"
   revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
 
 [[projects]]
   branch = "master"
+  digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72"
   name = "github.com/kr/logfmt"
   packages = ["."]
+  pruneopts = "UT"
   revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
 
 [[projects]]
+  digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7"
   name = "github.com/magiconair/properties"
   packages = ["."]
+  pruneopts = "UT"
   revision = "c2353362d570a7bfa228149c62842019201cfb71"
   version = "v1.8.0"
 
 [[projects]]
+  digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc"
   name = "github.com/matttproud/golang_protobuf_extensions"
   packages = ["pbutil"]
+  pruneopts = "UT"
   revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
   version = "v1.0.1"
 
 [[projects]]
   branch = "master"
+  digest = "1:5ab79470a1d0fb19b041a624415612f8236b3c06070161a910562f2b2d064355"
   name = "github.com/mitchellh/mapstructure"
   packages = ["."]
-  revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b"
+  pruneopts = "UT"
+  revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac"
 
 [[projects]]
+  digest = "1:95741de3af260a92cc5c7f3f3061e85273f5a81b5db20d4bd68da74bd521675e"
   name = "github.com/pelletier/go-toml"
   packages = ["."]
+  pruneopts = "UT"
   revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
   version = "v1.2.0"
 
 [[projects]]
+  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
   name = "github.com/pkg/errors"
   packages = ["."]
+  pruneopts = "UT"
   revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
   version = "v0.8.0"
 
 [[projects]]
+  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
   name = "github.com/pmezard/go-difflib"
   packages = ["difflib"]
+  pruneopts = "UT"
   revision = "792786c7400a136282c1664665ae0a8db921c6c2"
   version = "v1.0.0"
 
 [[projects]]
-  branch = "master"
+  digest = "1:c1a04665f9613e082e1209cf288bf64f4068dcd6c87a64bf1c4ff006ad422ba0"
   name = "github.com/prometheus/client_golang"
   packages = [
     "prometheus",
-    "prometheus/promhttp"
+    "prometheus/promhttp",
   ]
-  revision = "d6a9817c4afc94d51115e4a30d449056a3fbf547"
+  pruneopts = "UT"
+  revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632"
 
 [[projects]]
   branch = "master"
+  digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
   name = "github.com/prometheus/client_model"
   packages = ["go"]
-  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
+  pruneopts = "UT"
+  revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
 
 [[projects]]
   branch = "master"
+  digest = "1:63b68062b8968092eb86bedc4e68894bd096ea6b24920faca8b9dcf451f54bb5"
   name = "github.com/prometheus/common"
   packages = [
     "expfmt",
     "internal/bitbucket.org/ww/goautoneg",
-    "model"
+    "model",
   ]
-  revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
+  pruneopts = "UT"
+  revision = "c7de2306084e37d54b8be01f3541a8464345e9a5"
 
 [[projects]]
   branch = "master"
+  digest = "1:8c49953a1414305f2ff5465147ee576dd705487c35b15918fcd4efdc0cb7a290"
   name = "github.com/prometheus/procfs"
   packages = [
     ".",
     "internal/util",
     "nfs",
-    "xfs"
+    "xfs",
   ]
-  revision = "40f013a808ec4fa79def444a1a56de4d1727efcb"
+  pruneopts = "UT"
+  revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92"
 
 [[projects]]
-  branch = "master"
+  digest = "1:c4556a44e350b50a490544d9b06e9fba9c286c21d6c0e47f54f3a9214597298c"
   name = "github.com/rcrowley/go-metrics"
   packages = ["."]
+  pruneopts = "UT"
   revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
 
 [[projects]]
+  digest = "1:bd1ae00087d17c5a748660b8e89e1043e1e5479d0fea743352cda2f8dd8c4f84"
   name = "github.com/spf13/afero"
   packages = [
     ".",
-    "mem"
+    "mem",
   ]
+  pruneopts = "UT"
   revision = "787d034dfe70e44075ccc060d346146ef53270ad"
   version = "v1.1.1"
 
 [[projects]]
+  digest = "1:516e71bed754268937f57d4ecb190e01958452336fa73dbac880894164e91c1f"
   name = "github.com/spf13/cast"
   packages = ["."]
+  pruneopts = "UT"
   revision = "8965335b8c7107321228e3e3702cab9832751bac"
   version = "v1.2.0"
 
 [[projects]]
+  digest = "1:7ffc0983035bc7e297da3688d9fe19d60a420e9c38bef23f845c53788ed6a05e"
   name = "github.com/spf13/cobra"
   packages = ["."]
+  pruneopts = "UT"
   revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
   version = "v0.0.1"
 
 [[projects]]
   branch = "master"
+  digest = "1:080e5f630945ad754f4b920e60b4d3095ba0237ebf88dc462eb28002932e3805"
   name = "github.com/spf13/jwalterweatherman"
   packages = ["."]
+  pruneopts = "UT"
   revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
 
 [[projects]]
+  digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7"
   name = "github.com/spf13/pflag"
   packages = ["."]
+  pruneopts = "UT"
   revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
   version = "v1.0.1"
 
 [[projects]]
+  digest = "1:f8e1a678a2571e265f4bf91a3e5e32aa6b1474a55cb0ea849750cc177b664d96"
   name = "github.com/spf13/viper"
   packages = ["."]
+  pruneopts = "UT"
   revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
   version = "v1.0.0"
 
 [[projects]]
+  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
   name = "github.com/stretchr/testify"
   packages = [
     "assert",
-    "require"
+    "require",
   ]
-  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
-  version = "v1.2.2"
+  pruneopts = "UT"
+  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
+  version = "v1.2.1"
 
 [[projects]]
   branch = "master"
+  digest = "1:b3cfb8d82b1601a846417c3f31c03a7961862cb2c98dcf0959c473843e6d9a2b"
   name = "github.com/syndtr/goleveldb"
   packages = [
     "leveldb",
@@ -284,28 +352,34 @@
     "leveldb/opt",
     "leveldb/storage",
     "leveldb/table",
-    "leveldb/util"
+    "leveldb/util",
   ]
-  revision = "e2150783cd35f5b607daca48afd8c57ec54cc995"
+  pruneopts = "UT"
+  revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445"
 
 [[projects]]
   branch = "master"
+  digest = "1:087aaa7920e5d0bf79586feb57ce01c35c830396ab4392798112e8aae8c47722"
   name = "github.com/tendermint/ed25519"
   packages = [
     ".",
     "edwards25519",
-    "extra25519"
+    "extra25519",
   ]
+  pruneopts = "UT"
   revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057"
 
 [[projects]]
+  digest = "1:e9113641c839c21d8eaeb2c907c7276af1eddeed988df8322168c56b7e06e0e1"
   name = "github.com/tendermint/go-amino"
   packages = ["."]
+  pruneopts = "UT"
   revision = "2106ca61d91029c931fd54968c2bb02dc96b1412"
   version = "0.10.1"
 
 [[projects]]
   branch = "master"
+  digest = "1:c31a37cafc12315b8bd745c8ad6a006ac25350472488162a821e557b3e739d67"
   name = "golang.org/x/crypto"
   packages = [
     "bcrypt",
@@ -321,12 +395,13 @@
     "openpgp/errors",
     "poly1305",
     "ripemd160",
-    "salsa20/salsa"
+    "salsa20/salsa",
   ]
-  revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
+  pruneopts = "UT"
+  revision = "c126467f60eb25f8f27e5a981f32a87e3965053f"
 
 [[projects]]
-  branch = "master"
+  digest = "1:d36f55a999540d29b6ea3c2ea29d71c76b1d9853fdcd3e5c5cb4836f2ba118f1"
   name = "golang.org/x/net"
   packages = [
     "context",
@@ -336,20 +411,24 @@
     "idna",
     "internal/timeseries",
     "netutil",
-    "trace"
+    "trace",
   ]
-  revision = "4cb1c02c05b0e749b0365f61ae859a8e0cfceed9"
+  pruneopts = "UT"
+  revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f"
 
 [[projects]]
   branch = "master"
+  digest = "1:bb0fe59917bdd5b89f49b9a8b26e5f465e325d9223b3a8e32254314bdf51e0f1"
   name = "golang.org/x/sys"
   packages = [
     "cpu",
-    "unix"
+    "unix",
   ]
-  revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56"
+  pruneopts = "UT"
+  revision = "3dc4335d56c789b04b0ba99b7a37249d9b614314"
 
 [[projects]]
+  digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
   name = "golang.org/x/text"
   packages = [
     "collate",
@@ -365,17 +444,22 @@
     "unicode/bidi",
     "unicode/cldr",
     "unicode/norm",
-    "unicode/rangetable"
+    "unicode/rangetable",
   ]
+  pruneopts = "UT"
   revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
   version = "v0.3.0"
 
 [[projects]]
+  branch = "master"
+  digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c"
   name = "google.golang.org/genproto"
   packages = ["googleapis/rpc/status"]
-  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
+  pruneopts = "UT"
+  revision = "daca94659cb50e9f37c1b834680f2e46358f10b0"
 
 [[projects]]
+  digest = "1:2dab32a43451e320e49608ff4542fdfc653c95dcc35d0065ec9c6c3dd540ed74"
   name = "google.golang.org/grpc"
   packages = [
     ".",
@@ -387,9 +471,11 @@
     "credentials",
     "encoding",
     "encoding/proto",
-    "grpclb/grpc_lb_v1/messages",
     "grpclog",
     "internal",
+    "internal/backoff",
+    "internal/channelz",
+    "internal/grpcrand",
     "keepalive",
     "metadata",
     "naming",
@@ -400,20 +486,71 @@
     "stats",
     "status",
     "tap",
-    "transport"
+    "transport",
   ]
-  revision = "d11072e7ca9811b1100b80ca0269ac831f06d024"
-  version = "v1.11.3"
+  pruneopts = "UT"
+  revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
+  version = "v1.13.0"
 
 [[projects]]
+  digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202"
   name = "gopkg.in/yaml.v2"
   packages = ["."]
+  pruneopts = "UT"
   revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
   version = "v2.2.1"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "6e854634d6c203278ce83bef7725cecbcf90023b0d0e440fb3374acedacbd5ad"
+  input-imports = [
+    "github.com/btcsuite/btcd/btcec",
+    "github.com/btcsuite/btcutil/base58",
+    "github.com/btcsuite/btcutil/bech32",
+    "github.com/ebuchman/fail-test",
+    "github.com/fortytw2/leaktest",
+    "github.com/go-kit/kit/log",
+    "github.com/go-kit/kit/log/level",
+    "github.com/go-kit/kit/log/term",
+    "github.com/go-kit/kit/metrics",
+    "github.com/go-kit/kit/metrics/discard",
+    "github.com/go-kit/kit/metrics/prometheus",
+    "github.com/go-logfmt/logfmt",
+    "github.com/gogo/protobuf/gogoproto",
+    "github.com/gogo/protobuf/jsonpb",
+    "github.com/gogo/protobuf/proto",
+    "github.com/gogo/protobuf/types",
+    "github.com/golang/protobuf/proto",
+    "github.com/golang/protobuf/ptypes/timestamp",
+    "github.com/gorilla/websocket",
+    "github.com/jmhodges/levigo",
+    "github.com/pkg/errors",
+    "github.com/prometheus/client_golang/prometheus",
+    "github.com/prometheus/client_golang/prometheus/promhttp",
+    "github.com/rcrowley/go-metrics",
+    "github.com/spf13/cobra",
+    "github.com/spf13/viper",
+    "github.com/stretchr/testify/assert",
+    "github.com/stretchr/testify/require",
+    "github.com/syndtr/goleveldb/leveldb",
+    "github.com/syndtr/goleveldb/leveldb/errors",
+    "github.com/syndtr/goleveldb/leveldb/iterator",
+    "github.com/syndtr/goleveldb/leveldb/opt",
+    "github.com/tendermint/ed25519",
+    "github.com/tendermint/ed25519/extra25519",
+    "github.com/tendermint/go-amino",
+    "golang.org/x/crypto/bcrypt",
+    "golang.org/x/crypto/chacha20poly1305",
+    "golang.org/x/crypto/curve25519",
+    "golang.org/x/crypto/hkdf",
+    "golang.org/x/crypto/nacl/box",
+    "golang.org/x/crypto/nacl/secretbox",
+    "golang.org/x/crypto/openpgp/armor",
+    "golang.org/x/crypto/ripemd160",
+    "golang.org/x/net/context",
+    "golang.org/x/net/netutil",
+    "google.golang.org/grpc",
+    "google.golang.org/grpc/credentials",
+  ]
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -10,11 +10,6 @@
 #   name = "github.com/user/project"
 #   version = "1.0.0"
 #
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
 # [[override]]
 #   name = "github.com/x/y"
 #   version = "2.4.0"
@@ -23,40 +18,32 @@
 #   non-go = false
 #   go-tests = true
 #   unused-packages = true
+#
+###########################################################
+# NOTE: All packages should be pinned to specific versions.
+# Packages without releases must pin to a commit.
 
 
-[[constraint]]
-  name = "github.com/ebuchman/fail-test"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/fortytw2/leaktest"
-  branch = "master"
-
 [[constraint]]
   name = "github.com/go-kit/kit"
   version = "=0.6.0"
 
 [[constraint]]
   name = "github.com/gogo/protobuf"
-  version = "=1.0.0"
+  version = "=1.1.1"
 
 [[constraint]]
   name = "github.com/golang/protobuf"
-  version = "=1.0.0"
+  version = "=1.1.0"
 
 [[constraint]]
   name = "github.com/gorilla/websocket"
-  version = "~1.2.0"
+  version = "=1.2.0"
 
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "=0.8.0"
 
-[[constraint]]
-  name = "github.com/rcrowley/go-metrics"
-  branch = "master"
-
 [[constraint]]
   name = "github.com/spf13/cobra"
   version = "=0.0.1"
@@ -67,29 +54,50 @@
 
 [[constraint]]
   name = "github.com/stretchr/testify"
-  version = "~1.2.1"
+  version = "=1.2.1"
 
 [[constraint]]
   name = "github.com/tendermint/go-amino"
-  version = "~0.10.1"
+  version = "=v0.10.1"
 
 [[constraint]]
   name = "google.golang.org/grpc"
-  version = "~1.11.3"
+  version = "=1.13.0"
+
+[[constraint]]
+  name = "github.com/fortytw2/leaktest"
+  version = "=1.2.0"
+
+###################################
+## Some repos dont have releases.
+## Pin to revision
 
-# this got updated and broke, so locked to an old working commit ...
 [[override]]
-  name = "google.golang.org/genproto"
-  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
+  name = "github.com/jmhodges/levigo"
+  revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
+
+[[constraint]]
+  name = "github.com/ebuchman/fail-test"
+  revision = "95f809107225be108efcf10a3509e4ea6ceef3c4"
+
+# last revision used by go-crypto
+[[constraint]]
+  name = "github.com/btcsuite/btcutil"
+  revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4"
+
+# Haven't made a release since 2016.
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632"
+
+[[constraint]]
+  name = "github.com/rcrowley/go-metrics"
+  revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
+
+[[constraint]]
+  name = "golang.org/x/net"
+  revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f"
 
 [prune]
   go-tests = true
   unused-packages = true
-
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  branch = "master"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/net"

Makefile

@@ -5,39 +5,46 @@ GOTOOLS = \
 	github.com/gogo/protobuf/protoc-gen-gogo \
 	github.com/gogo/protobuf/gogoproto \
 	github.com/square/certstrap
-PACKAGES=$(shell go list ./... | grep -v '/vendor/')
+PACKAGES=$(shell go list ./...)
+
 INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf
-BUILD_TAGS?=tendermint
+BUILD_TAGS?='tendermint'
 BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
 
 all: check build test install
 
-check: check_tools ensure_deps
+check: check_tools get_vendor_deps
 
 
 ########################################
 ### Build Tendermint
 
 build:
-	CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/
+	CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint/
 
 build_race:
-	CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint
+	CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint
 
 install:
-	CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint
+	CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint
+
+########################################
+### Protobuf
+
+protoc_all: protoc_libs protoc_abci protoc_grpc
+
+%.pb.go: %.proto
+	## If you get the following error,
+	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
+	## See https://stackoverflow.com/a/25518702
+	protoc $(INCLUDE) $< --gogo_out=Mgoogle/protobuf/timestamp.proto=github.com/golang/protobuf/ptypes/timestamp,plugins=grpc:.
+	@echo "--> adding nolint declarations to protobuf generated files"
+	@awk -i inplace '/^\s*package \w+/ { print "//nolint" }1' $@
 
 ########################################
 ### Build ABCI
 
-protoc_abci:
-	## If you get the following error,
-	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
-	## See https://stackoverflow.com/a/25518702
-	protoc $(INCLUDE) --gogo_out=plugins=grpc:. abci/types/*.proto
-	@echo "--> adding nolint declarations to protobuf generated files"
-	@awk '/package abci/types/ { print "//nolint: gas"; print; next }1' abci/types/types.pb.go > abci/types/types.pb.go.new
-	@mv abci/types/types.pb.go.new abci/types/types.pb.go
+protoc_abci: abci/types/types.pb.go
 
 build_abci:
 	@go build -i ./abci/cmd/...
@@ -51,7 +58,7 @@ install_abci:
 # dist builds binaries for all platforms and packages them for distribution
 # TODO add abci to these scripts
 dist:
-	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
+	@BUILD_TAGS=$(BUILD_TAGS) sh -c "'$(CURDIR)/scripts/dist.sh'"
 
 ########################################
 ### Tools & dependencies
@@ -70,16 +77,8 @@ update_tools:
 	@echo "--> Updating tools"
 	@go get -u $(GOTOOLS)
 
-#Run this from CI
+#Update dependencies
 get_vendor_deps:
-	@rm -rf vendor/
-	@echo "--> Running dep"
-	@dep ensure -vendor-only
-
-
-#Run this locally.
-ensure_deps:
-	@rm -rf vendor/
 	@echo "--> Running dep"
 	@dep ensure
 
@@ -101,21 +100,14 @@ draw_deps:
 
 get_deps_bin_size:
 	@# Copy of build recipe with additional flags to perform binary size analysis
-	$(eval $(shell go build -work -a $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ 2>&1))
+	$(eval $(shell go build -work -a $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o build/tendermint ./cmd/tendermint/ 2>&1))
 	@find $(WORK) -type f -name "*.a" | xargs -I{} du -hxs "{}" | sort -rh | sed -e s:${WORK}/::g > deps_bin_size.log
 	@echo "Results can be found here: $(CURDIR)/deps_bin_size.log"
 
 ########################################
 ### Libs
 
-protoc_libs:
-	## If you get the following error,
-	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
-	## See https://stackoverflow.com/a/25518702
-	protoc $(INCLUDE) --go_out=plugins=grpc:. libs/common/*.proto
-	@echo "--> adding nolint declarations to protobuf generated files"
-	@awk '/package libs/common/ { print "//nolint: gas"; print; next }1' libs/common/types.pb.go > libs/common/types.pb.go.new
-	@mv libs/common/types.pb.go.new libs/common/types.pb.go
+protoc_libs: libs/common/types.pb.go
 
 gen_certs: clean_certs
 	## Generating certificates for TLS testing...
@@ -130,12 +122,14 @@ clean_certs:
 	rm -f db/remotedb/::.crt db/remotedb/::.key
 
 test_libs: gen_certs
-	GOCACHE=off go test -tags gcc $(shell go list ./... | grep -v vendor)
+	GOCACHE=off go test -tags gcc $(PACKAGES)
 	make clean_certs
 
 grpc_dbserver:
 	protoc -I db/remotedb/proto/ db/remotedb/proto/defs.proto --go_out=plugins=grpc:db/remotedb/proto
 
+protoc_grpc: rpc/grpc/types.pb.go
+
 ########################################
 ### Testing
 
@@ -206,7 +200,7 @@ vagrant_test:
 ### go tests
 test:
 	@echo "--> Running go test"
-	@go test $(PACKAGES)
+	@GOCACHE=off go test $(PACKAGES)
 
 test_race:
 	@echo "--> Running go test --race"
@@ -252,6 +246,16 @@ metalinter_all:
 	@echo "--> Running linter (all)"
 	gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./...
 
+DESTINATION = ./index.html.md
+
+rpc-docs:
+	cat rpc/core/slate_header.txt > $(DESTINATION)
+	godoc2md -template rpc/core/doc_template.txt github.com/tendermint/tendermint/rpc/core | grep -v -e "pipe.go" -e "routes.go" -e "dev.go" | sed 's,/src/target,https://github.com/tendermint/tendermint/tree/master/rpc/core,' >> $(DESTINATION)
+
+check_dep:
+	dep status >> /dev/null
+	!(grep -n branch Gopkg.toml)
+
 ###########################################################
 ### Docker image
 
@@ -307,4 +311,4 @@ build-slate:
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: check build build_race build_abci dist install install_abci check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate
+.PHONY: check build build_race build_abci dist install install_abci check_dep check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt rpc-docs build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all

README.md

@@ -22,7 +22,7 @@ develop   | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/deve
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language -
 and securely replicates it on many machines.
 
-For protocol details, see [the specification](/docs/spec).
+For protocol details, see [the specification](/docs/spec). For a consensus proof and detailed protocol analysis checkout our recent paper, "[The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938)".
 
 ## A Note on Production Readiness
 
@@ -50,13 +50,13 @@ Go version | Go1.9 or higher
 
 ## Install
 
-See the [install instructions](/docs/install.md)
+See the [install instructions](/docs/introduction/install.md)
 
 ## Quick Start
 
 - [Single node](/docs/using-tendermint.md)
 - [Local cluster using docker-compose](/networks/local)
-- [Remote cluster using terraform and ansible](/docs/terraform-and-ansible.md)
+- [Remote cluster using terraform and ansible](/docs/networks/terraform-and-ansible.md)
 - [Join the public testnet](https://cosmos.network/testnet)
 
 ## Resources

abci/README.md

@@ -29,7 +29,7 @@ The two guides to focus on are the `Application Development Guide` and `Using AB
 To compile the protobuf file, run:
 
 ```
-make protoc
+cd $GOPATH/src/github.com/tendermint/tendermint/; make protoc_abci
 ```
 
 See `protoc --help` and [the Protocol Buffers site](https://developers.google.com/protocol-buffers)

abci/client/grpc_client.go

@@ -63,7 +63,7 @@ RETRY_LOOP:
 
 	ENSURE_CONNECTED:
 		for {
-			_, err := client.Echo(context.Background(), &types.RequestEcho{"hello"}, grpc.FailFast(true))
+			_, err := client.Echo(context.Background(), &types.RequestEcho{Message: "hello"}, grpc.FailFast(true))
 			if err == nil {
 				break ENSURE_CONNECTED
 			}
@@ -129,7 +129,7 @@ func (cli *grpcClient) EchoAsync(msg string) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_Echo{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Echo{res}})
 }
 
 func (cli *grpcClient) FlushAsync() *ReqRes {
@@ -138,7 +138,7 @@ func (cli *grpcClient) FlushAsync() *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_Flush{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Flush{res}})
 }
 
 func (cli *grpcClient) InfoAsync(params types.RequestInfo) *ReqRes {
@@ -147,7 +147,7 @@ func (cli *grpcClient) InfoAsync(params types.RequestInfo) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_Info{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Info{res}})
 }
 
 func (cli *grpcClient) SetOptionAsync(params types.RequestSetOption) *ReqRes {
@@ -156,7 +156,7 @@ func (cli *grpcClient) SetOptionAsync(params types.RequestSetOption) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_SetOption{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_SetOption{res}})
 }
 
 func (cli *grpcClient) DeliverTxAsync(tx []byte) *ReqRes {
@@ -165,7 +165,7 @@ func (cli *grpcClient) DeliverTxAsync(tx []byte) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_DeliverTx{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_DeliverTx{res}})
 }
 
 func (cli *grpcClient) CheckTxAsync(tx []byte) *ReqRes {
@@ -174,7 +174,7 @@ func (cli *grpcClient) CheckTxAsync(tx []byte) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_CheckTx{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_CheckTx{res}})
 }
 
 func (cli *grpcClient) QueryAsync(params types.RequestQuery) *ReqRes {
@@ -183,7 +183,7 @@ func (cli *grpcClient) QueryAsync(params types.RequestQuery) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_Query{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Query{res}})
 }
 
 func (cli *grpcClient) CommitAsync() *ReqRes {
@@ -192,7 +192,7 @@ func (cli *grpcClient) CommitAsync() *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_Commit{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Commit{res}})
 }
 
 func (cli *grpcClient) InitChainAsync(params types.RequestInitChain) *ReqRes {
@@ -201,7 +201,7 @@ func (cli *grpcClient) InitChainAsync(params types.RequestInitChain) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_InitChain{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_InitChain{res}})
 }
 
 func (cli *grpcClient) BeginBlockAsync(params types.RequestBeginBlock) *ReqRes {
@@ -210,7 +210,7 @@ func (cli *grpcClient) BeginBlockAsync(params types.RequestBeginBlock) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_BeginBlock{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_BeginBlock{res}})
 }
 
 func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes {
@@ -219,7 +219,7 @@ func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes {
 	if err != nil {
 		cli.StopForError(err)
 	}
-	return cli.finishAsyncCall(req, &types.Response{&types.Response_EndBlock{res}})
+	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_EndBlock{res}})
 }
 
 func (cli *grpcClient) finishAsyncCall(req *types.Request, res *types.Response) *ReqRes {

abci/client/local_client.go

@@ -149,7 +149,7 @@ func (app *localClient) FlushSync() error {
 }
 
 func (app *localClient) EchoSync(msg string) (*types.ResponseEcho, error) {
-	return &types.ResponseEcho{msg}, nil
+	return &types.ResponseEcho{Message: msg}, nil
 }
 
 func (app *localClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) {

abci/client/socket_client.go

@@ -357,6 +357,13 @@ func (cli *socketClient) queueRequest(req *types.Request) *ReqRes {
 }
 
 func (cli *socketClient) flushQueue() {
+	// mark all in-flight messages as resolved (they will get cli.Error())
+	for req := cli.reqSent.Front(); req != nil; req = req.Next() {
+		reqres := req.Value.(*ReqRes)
+		reqres.Done()
+	}
+
+	// mark all queued messages as resolved
 LOOP:
 	for {
 		select {

abci/client/socket_client_test.go

@@ -2,10 +2,17 @@ package abcicli_test
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/abci/client"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/server"
+	"github.com/tendermint/tendermint/abci/types"
+	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
 func TestSocketClientStopForErrorDeadlock(t *testing.T) {
@@ -26,3 +33,89 @@ func TestSocketClientStopForErrorDeadlock(t *testing.T) {
 		t.Fatalf("Test took too long, potential deadlock still exists")
 	}
 }
+
+func TestProperSyncCalls(t *testing.T) {
+	app := slowApp{}
+
+	s, c := setupClientServer(t, app)
+	defer s.Stop()
+	defer c.Stop()
+
+	resp := make(chan error, 1)
+	go func() {
+		// This is BeginBlockSync unrolled....
+		reqres := c.BeginBlockAsync(types.RequestBeginBlock{})
+		c.FlushSync()
+		res := reqres.Response.GetBeginBlock()
+		require.NotNil(t, res)
+		resp <- c.Error()
+	}()
+
+	select {
+	case <-time.After(time.Second):
+		require.Fail(t, "No response arrived")
+	case err, ok := <-resp:
+		require.True(t, ok, "Must not close channel")
+		assert.NoError(t, err, "This should return success")
+	}
+}
+
+func TestHangingSyncCalls(t *testing.T) {
+	app := slowApp{}
+
+	s, c := setupClientServer(t, app)
+	defer s.Stop()
+	defer c.Stop()
+
+	resp := make(chan error, 1)
+	go func() {
+		// Start BeginBlock and flush it
+		reqres := c.BeginBlockAsync(types.RequestBeginBlock{})
+		flush := c.FlushAsync()
+		// wait 20 ms for all events to travel socket, but
+		// no response yet from server
+		time.Sleep(20 * time.Millisecond)
+		// kill the server, so the connections break
+		s.Stop()
+
+		// wait for the response from BeginBlock
+		reqres.Wait()
+		flush.Wait()
+		resp <- c.Error()
+	}()
+
+	select {
+	case <-time.After(time.Second):
+		require.Fail(t, "No response arrived")
+	case err, ok := <-resp:
+		require.True(t, ok, "Must not close channel")
+		assert.Error(t, err, "We should get EOF error")
+	}
+}
+
+func setupClientServer(t *testing.T, app types.Application) (
+	cmn.Service, abcicli.Client) {
+	// some port between 20k and 30k
+	port := 20000 + cmn.RandInt32()%10000
+	addr := fmt.Sprintf("localhost:%d", port)
+
+	s, err := server.NewServer(addr, "socket", app)
+	require.NoError(t, err)
+	err = s.Start()
+	require.NoError(t, err)
+
+	c := abcicli.NewSocketClient(addr, true)
+	err = c.Start()
+	require.NoError(t, err)
+
+	return s, c
+}
+
+type slowApp struct {
+	types.BaseApplication
+}
+
+func (slowApp) BeginBlock(req types.RequestBeginBlock) types.ResponseBeginBlock {
+	time.Sleep(200 * time.Millisecond)
+	return types.ResponseBeginBlock{}
+}

abci/cmd/abci-cli/abci-cli.go

@@ -514,7 +514,7 @@ func cmdInfo(cmd *cobra.Command, args []string) error {
 	if len(args) == 1 {
 		version = args[0]
 	}
-	res, err := client.InfoSync(types.RequestInfo{version})
+	res, err := client.InfoSync(types.RequestInfo{Version: version})
 	if err != nil {
 		return err
 	}
@@ -537,7 +537,7 @@ func cmdSetOption(cmd *cobra.Command, args []string) error {
 	}
 
 	key, val := args[0], args[1]
-	_, err := client.SetOptionSync(types.RequestSetOption{key, val})
+	_, err := client.SetOptionSync(types.RequestSetOption{Key: key, Value: val})
 	if err != nil {
 		return err
 	}

abci/example/example_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"google.golang.org/grpc"
 
 	"golang.org/x/net/context"
@@ -43,7 +45,7 @@ func testStream(t *testing.T, app types.Application) {
 	server := abciserver.NewSocketServer("unix://test.sock", app)
 	server.SetLogger(log.TestingLogger().With("module", "abci-server"))
 	if err := server.Start(); err != nil {
-		t.Fatalf("Error starting socket server: %v", err.Error())
+		require.NoError(t, err, "Error starting socket server")
 	}
 	defer server.Stop()
 
@@ -132,7 +134,7 @@ func testGRPCSync(t *testing.T, app *types.GRPCApplication) {
 	// Write requests
 	for counter := 0; counter < numDeliverTxs; counter++ {
 		// Send request
-		response, err := client.DeliverTx(context.Background(), &types.RequestDeliverTx{[]byte("test")})
+		response, err := client.DeliverTx(context.Background(), &types.RequestDeliverTx{Tx: []byte("test")})
 		if err != nil {
 			t.Fatalf("Error in GRPC DeliverTx: %v", err.Error())
 		}

abci/example/kvstore/kvstore_test.go

@@ -90,8 +90,8 @@ func TestPersistentKVStoreInfo(t *testing.T) {
 	header := types.Header{
 		Height: int64(height),
 	}
-	kvstore.BeginBlock(types.RequestBeginBlock{hash, header, nil, nil})
-	kvstore.EndBlock(types.RequestEndBlock{header.Height})
+	kvstore.BeginBlock(types.RequestBeginBlock{Hash: hash, Header: header})
+	kvstore.EndBlock(types.RequestEndBlock{Height: header.Height})
 	kvstore.Commit()
 
 	resInfo = kvstore.Info(types.RequestInfo{})
@@ -176,13 +176,13 @@ func makeApplyBlock(t *testing.T, kvstore types.Application, heightInt int, diff
 		Height: height,
 	}
 
-	kvstore.BeginBlock(types.RequestBeginBlock{hash, header, nil, nil})
+	kvstore.BeginBlock(types.RequestBeginBlock{Hash: hash, Header: header})
 	for _, tx := range txs {
 		if r := kvstore.DeliverTx(tx); r.IsErr() {
 			t.Fatal(r)
 		}
 	}
-	resEndBlock := kvstore.EndBlock(types.RequestEndBlock{header.Height})
+	resEndBlock := kvstore.EndBlock(types.RequestEndBlock{Height: header.Height})
 	kvstore.Commit()
 
 	valsEqual(t, diff, resEndBlock.ValidatorUpdates)

abci/tests/test_app/app.go

@@ -26,7 +26,7 @@ func startClient(abciType string) abcicli.Client {
 }
 
 func setOption(client abcicli.Client, key, value string) {
-	_, err := client.SetOptionSync(types.RequestSetOption{key, value})
+	_, err := client.SetOptionSync(types.RequestSetOption{Key: key, Value: value})
 	if err != nil {
 		panicf("setting %v=%v: \nerr: %v", key, value, err)
 	}

abci/types/application.go

@@ -85,7 +85,7 @@ func NewGRPCApplication(app Application) *GRPCApplication {
 }
 
 func (app *GRPCApplication) Echo(ctx context.Context, req *RequestEcho) (*ResponseEcho, error) {
-	return &ResponseEcho{req.Message}, nil
+	return &ResponseEcho{Message: req.Message}, nil
 }
 
 func (app *GRPCApplication) Flush(ctx context.Context, req *RequestFlush) (*ResponseFlush, error) {

abci/types/messages.go

@@ -71,7 +71,7 @@ func encodeVarint(w io.Writer, i int64) (err error) {
 
 func ToRequestEcho(message string) *Request {
 	return &Request{
-		Value: &Request_Echo{&RequestEcho{message}},
+		Value: &Request_Echo{&RequestEcho{Message: message}},
 	}
 }
 
@@ -95,13 +95,13 @@ func ToRequestSetOption(req RequestSetOption) *Request {
 
 func ToRequestDeliverTx(tx []byte) *Request {
 	return &Request{
-		Value: &Request_DeliverTx{&RequestDeliverTx{tx}},
+		Value: &Request_DeliverTx{&RequestDeliverTx{Tx: tx}},
 	}
 }
 
 func ToRequestCheckTx(tx []byte) *Request {
 	return &Request{
-		Value: &Request_CheckTx{&RequestCheckTx{tx}},
+		Value: &Request_CheckTx{&RequestCheckTx{Tx: tx}},
 	}
 }
 
@@ -139,13 +139,13 @@ func ToRequestEndBlock(req RequestEndBlock) *Request {
 
 func ToResponseException(errStr string) *Response {
 	return &Response{
-		Value: &Response_Exception{&ResponseException{errStr}},
+		Value: &Response_Exception{&ResponseException{Error: errStr}},
 	}
 }
 
 func ToResponseEcho(message string) *Response {
 	return &Response{
-		Value: &Response_Echo{&ResponseEcho{message}},
+		Value: &Response_Echo{&ResponseEcho{Message: message}},
 	}
 }
 

abci/types/messages_test.go

@@ -85,7 +85,6 @@ func TestWriteReadMessage2(t *testing.T) {
 			Tags: []cmn.KVPair{
 				cmn.KVPair{[]byte("abc"), []byte("def")},
 			},
-			// Fee: cmn.KI64Pair{
 		},
 		// TODO: add the rest
 	}

abci/types/types.proto

@@ -4,12 +4,22 @@ package types;
 // For more information on gogo.proto, see:
 // https://github.com/gogo/protobuf/blob/master/extensions.md
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-import "github.com/tendermint/tmlibs/common/types.proto";
+import "google/protobuf/timestamp.proto";
+import "github.com/tendermint/tendermint/libs/common/types.proto";
 
 // This file is copied from http://github.com/tendermint/abci
 // NOTE: When using custom types, mind the warnings.
 // https://github.com/gogo/protobuf/blob/master/custom_types.md#warnings-and-issues
 
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.goproto_registration) = true;
+// Generate tests
+option (gogoproto.populate_all) = true;
+option (gogoproto.equal_all) = true;
+option (gogoproto.testgen_all) = true;
+
 //----------------------------------------
 // Request types
 
@@ -47,7 +57,7 @@ message RequestSetOption {
 }
 
 message RequestInitChain {
-  int64 time = 1;
+  google.protobuf.Timestamp time = 1  [(gogoproto.nullable)=false, (gogoproto.stdtime)=true];
   string chain_id = 2;
   ConsensusParams consensus_params = 3;
   repeated Validator validators = 4  [(gogoproto.nullable)=false];
@@ -61,10 +71,11 @@ message RequestQuery {
   bool prove = 4;
 }
 
+// NOTE: validators here have empty pubkeys.
 message RequestBeginBlock {
   bytes hash = 1;
   Header header = 2 [(gogoproto.nullable)=false];
-  repeated SigningValidator validators = 3  [(gogoproto.nullable)=false];
+  LastCommitInfo last_commit_info = 3 [(gogoproto.nullable)=false];
   repeated Evidence byzantine_validators = 4 [(gogoproto.nullable)=false];
 }
 
@@ -159,7 +170,6 @@ message ResponseCheckTx {
   int64 gas_wanted  = 5;
   int64 gas_used = 6;
   repeated common.KVPair tags = 7 [(gogoproto.nullable)=false, (gogoproto.jsontag)="tags,omitempty"];
-  common.KI64Pair fee = 8 [(gogoproto.nullable)=false];
 }
 
 message ResponseDeliverTx {
@@ -170,7 +180,6 @@ message ResponseDeliverTx {
   int64 gas_wanted = 5;
   int64 gas_used = 6;
   repeated common.KVPair tags = 7 [(gogoproto.nullable)=false, (gogoproto.jsontag)="tags,omitempty"];
-  common.KI64Pair fee = 8 [(gogoproto.nullable)=false];
 }
 
 message ResponseEndBlock {
@@ -195,14 +204,14 @@ message ConsensusParams {
   BlockGossip block_gossip = 3;
 }
 
-// BlockSize contain limits on the block size.
+// BlockSize contains limits on the block size.
 message BlockSize {
   int32 max_bytes = 1;
   int32 max_txs = 2;
   int64 max_gas = 3;
 }
 
-// TxSize contain limits on the tx size.
+// TxSize contains limits on the tx size.
 message TxSize {
   int32 max_bytes = 1;
   int64 max_gas = 2;
@@ -215,6 +224,11 @@ message BlockGossip {
   int32 block_part_size_bytes = 1;
 }
 
+message LastCommitInfo {
+  int32 commit_round = 1;
+  repeated SigningValidator validators = 2 [(gogoproto.nullable)=false];
+}
+
 //----------------------------------------
 // Blockchain Types
 
@@ -223,7 +237,7 @@ message Header {
   // basics
   string chain_id = 1 [(gogoproto.customname)="ChainID"];
   int64 height = 2;
-  int64 time = 3;
+  google.protobuf.Timestamp time = 3 [(gogoproto.nullable)=false, (gogoproto.stdtime)=true];
 
   // txs
   int32 num_txs = 4;
@@ -260,7 +274,7 @@ message Evidence {
   string type = 1;
   Validator validator = 2 [(gogoproto.nullable)=false];
   int64 height = 3;
-  int64 time = 4;
+  google.protobuf.Timestamp time = 4 [(gogoproto.nullable)=false, (gogoproto.stdtime)=true];
   int64 total_voting_power = 5;
 }
 

benchmarks/codec_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/tendermint/go-amino"
 
 	proto "github.com/tendermint/tendermint/benchmarks/proto"
-	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	"github.com/tendermint/tendermint/p2p"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 )
@@ -16,7 +16,7 @@ func BenchmarkEncodeStatusWire(b *testing.B) {
 	b.StopTimer()
 	cdc := amino.NewCodec()
 	ctypes.RegisterAmino(cdc)
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	status := &ctypes.ResultStatus{
 		NodeInfo: p2p.NodeInfo{
 			ID:         nodeKey.ID(),
@@ -52,7 +52,7 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) {
 	b.StopTimer()
 	cdc := amino.NewCodec()
 	ctypes.RegisterAmino(cdc)
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	nodeInfo := p2p.NodeInfo{
 		ID:         nodeKey.ID(),
 		Moniker:    "SOMENAME",
@@ -77,7 +77,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 	b.StopTimer()
 	cdc := amino.NewCodec()
 	ctypes.RegisterAmino(cdc)
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	nodeInfo := p2p.NodeInfo{
 		ID:         nodeKey.ID(),
 		Moniker:    "SOMENAME",
@@ -98,7 +98,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoProto(b *testing.B) {
 	b.StopTimer()
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	nodeID := string(nodeKey.ID())
 	someName := "SOMENAME"
 	someAddr := "SOMEADDR"

blockchain/pool.go

@@ -29,10 +29,10 @@ eg, L = latency = 0.1s
 */
 
 const (
-	requestIntervalMS         = 100
-	maxTotalRequesters        = 1000
+	requestIntervalMS         = 2
+	maxTotalRequesters        = 600
 	maxPendingRequests        = maxTotalRequesters
-	maxPendingRequestsPerPeer = 50
+	maxPendingRequestsPerPeer = 20
 
 	// Minimum recv rate to ensure we're receiving blocks from a peer fast
 	// enough. If a peer is not sending us data at at least that rate, we
@@ -219,14 +219,12 @@ func (pool *BlockPool) RedoRequest(height int64) p2p.ID {
 	defer pool.mtx.Unlock()
 
 	request := pool.requesters[height]
-
-	if request.block == nil {
-		panic("Expected block to be non-nil")
+	peerID := request.getPeerID()
+	if peerID != p2p.ID("") {
+		// RemovePeer will redo all requesters associated with this peer.
+		pool.removePeer(peerID)
 	}
-
-	// RemovePeer will redo all requesters associated with this peer.
-	pool.removePeer(request.peerID)
-	return request.peerID
+	return peerID
 }
 
 // TODO: ensure that blocks come in order for each peer.

blockchain/pool_test.go

@@ -1,7 +1,6 @@
 package blockchain
 
 import (
-	"math/rand"
 	"testing"
 	"time"
 
@@ -25,7 +24,7 @@ func makePeers(numPeers int, minHeight, maxHeight int64) map[p2p.ID]testPeer {
 	peers := make(map[p2p.ID]testPeer, numPeers)
 	for i := 0; i < numPeers; i++ {
 		peerID := p2p.ID(cmn.RandStr(12))
-		height := minHeight + rand.Int63n(maxHeight-minHeight)
+		height := minHeight + cmn.RandInt63n(maxHeight-minHeight)
 		peers[peerID] = testPeer{peerID, height}
 	}
 	return peers
@@ -80,7 +79,7 @@ func TestBasic(t *testing.T) {
 			}
 			// Request desired, pretend like we got the block immediately.
 			go func() {
-				block := &types.Block{Header: &types.Header{Height: request.Height}}
+				block := &types.Block{Header: types.Header{Height: request.Height}}
 				pool.AddBlock(request.PeerID, block, 123)
 				t.Logf("Added block from peer %v (height: %v)", request.PeerID, request.Height)
 			}()

blockchain/reactor.go

@@ -18,7 +18,8 @@ const (
 	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
 	BlockchainChannel = byte(0x40)
 
-	trySyncIntervalMS = 50
+	trySyncIntervalMS = 10
+
 	// stop syncing when last block's time is
 	// within this much of the system time.
 	// stopSyncingDurationMinutes = 10
@@ -76,8 +77,9 @@ func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *Bl
 			store.Height()))
 	}
 
-	const capacity = 1000 // must be bigger than peers count
-	requestsCh := make(chan BlockRequest, capacity)
+	requestsCh := make(chan BlockRequest, maxTotalRequesters)
+
+	const capacity = 1000                      // must be bigger than peers count
 	errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock
 
 	pool := NewBlockPool(
@@ -107,9 +109,6 @@ func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
 
 // OnStart implements cmn.Service.
 func (bcR *BlockchainReactor) OnStart() error {
-	if err := bcR.BaseReactor.OnStart(); err != nil {
-		return err
-	}
 	if bcR.fastSync {
 		err := bcR.pool.Start()
 		if err != nil {
@@ -122,7 +121,6 @@ func (bcR *BlockchainReactor) OnStart() error {
 
 // OnStop implements cmn.Service.
 func (bcR *BlockchainReactor) OnStop() {
-	bcR.BaseReactor.OnStop()
 	bcR.pool.Stop()
 }
 
@@ -209,7 +207,6 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 
 // Handle messages from the poolReactor telling the reactor what to do.
 // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
-// (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.)
 func (bcR *BlockchainReactor) poolRoutine() {
 
 	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
@@ -224,6 +221,8 @@ func (bcR *BlockchainReactor) poolRoutine() {
 	lastHundred := time.Now()
 	lastRate := 0.0
 
+	didProcessCh := make(chan struct{}, 1)
+
 FOR_LOOP:
 	for {
 		select {
@@ -239,14 +238,17 @@ FOR_LOOP:
 				// The pool handles timeouts, just let it go.
 				continue FOR_LOOP
 			}
+
 		case err := <-bcR.errorsCh:
 			peer := bcR.Switch.Peers().Get(err.peerID)
 			if peer != nil {
 				bcR.Switch.StopPeerForError(peer, err)
 			}
+
 		case <-statusUpdateTicker.C:
 			// ask for status updates
 			go bcR.BroadcastStatusRequest() // nolint: errcheck
+
 		case <-switchToConsensusTicker.C:
 			height, numPending, lenRequesters := bcR.pool.GetStatus()
 			outbound, inbound, _ := bcR.Switch.NumPeers()
@@ -261,60 +263,78 @@ FOR_LOOP:
 
 				break FOR_LOOP
 			}
+
 		case <-trySyncTicker.C: // chan time
-			// This loop can be slow as long as it's doing syncing work.
-		SYNC_LOOP:
-			for i := 0; i < 10; i++ {
-				// See if there are any blocks to sync.
-				first, second := bcR.pool.PeekTwoBlocks()
-				//bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
-				if first == nil || second == nil {
-					// We need both to sync the first block.
-					break SYNC_LOOP
+			select {
+			case didProcessCh <- struct{}{}:
+			default:
+			}
+
+		case <-didProcessCh:
+			// NOTE: It is a subtle mistake to process more than a single block
+			// at a time (e.g. 10) here, because we only TrySend 1 request per
+			// loop.  The ratio mismatch can result in starving of blocks, a
+			// sudden burst of requests and responses, and repeat.
+			// Consequently, it is better to split these routines rather than
+			// coupling them as it's written here.  TODO uncouple from request
+			// routine.
+
+			// See if there are any blocks to sync.
+			first, second := bcR.pool.PeekTwoBlocks()
+			//bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
+			if first == nil || second == nil {
+				// We need both to sync the first block.
+				continue FOR_LOOP
+			} else {
+				// Try again quickly next loop.
+				didProcessCh <- struct{}{}
+			}
+
+			firstParts := first.MakePartSet(state.ConsensusParams.BlockPartSizeBytes)
+			firstPartsHeader := firstParts.Header()
+			firstID := types.BlockID{first.Hash(), firstPartsHeader}
+			// Finally, verify the first block using the second's commit
+			// NOTE: we can probably make this more efficient, but note that calling
+			// first.Hash() doesn't verify the tx contents, so MakePartSet() is
+			// currently necessary.
+			err := state.Validators.VerifyCommit(
+				chainID, firstID, first.Height, second.LastCommit)
+			if err != nil {
+				bcR.Logger.Error("Error in validation", "err", err)
+				peerID := bcR.pool.RedoRequest(first.Height)
+				peer := bcR.Switch.Peers().Get(peerID)
+				if peer != nil {
+					// NOTE: we've already removed the peer's request, but we
+					// still need to clean up the rest.
+					bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err))
 				}
-				firstParts := first.MakePartSet(state.ConsensusParams.BlockPartSizeBytes)
-				firstPartsHeader := firstParts.Header()
-				firstID := types.BlockID{first.Hash(), firstPartsHeader}
-				// Finally, verify the first block using the second's commit
-				// NOTE: we can probably make this more efficient, but note that calling
-				// first.Hash() doesn't verify the tx contents, so MakePartSet() is
-				// currently necessary.
-				err := state.Validators.VerifyCommit(
-					chainID, firstID, first.Height, second.LastCommit)
+				continue FOR_LOOP
+			} else {
+				bcR.pool.PopRequest()
+
+				// TODO: batch saves so we dont persist to disk every block
+				bcR.store.SaveBlock(first, firstParts, second.LastCommit)
+
+				// TODO: same thing for app - but we would need a way to
+				// get the hash without persisting the state
+				var err error
+				state, err = bcR.blockExec.ApplyBlock(state, firstID, first)
 				if err != nil {
-					bcR.Logger.Error("Error in validation", "err", err)
-					peerID := bcR.pool.RedoRequest(first.Height)
-					peer := bcR.Switch.Peers().Get(peerID)
-					if peer != nil {
-						bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err))
-					}
-					break SYNC_LOOP
-				} else {
-					bcR.pool.PopRequest()
+					// TODO This is bad, are we zombie?
+					cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v",
+						first.Height, first.Hash(), err))
+				}
+				blocksSynced++
 
-					// TODO: batch saves so we dont persist to disk every block
-					bcR.store.SaveBlock(first, firstParts, second.LastCommit)
-
-					// TODO: same thing for app - but we would need a way to
-					// get the hash without persisting the state
-					var err error
-					state, err = bcR.blockExec.ApplyBlock(state, firstID, first)
-					if err != nil {
-						// TODO This is bad, are we zombie?
-						cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v",
-							first.Height, first.Hash(), err))
-					}
-					blocksSynced++
-
-					if blocksSynced%100 == 0 {
-						lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
-						bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
-							"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
-						lastHundred = time.Now()
-					}
+				if blocksSynced%100 == 0 {
+					lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+					bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
+						"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
+					lastHundred = time.Now()
 				}
 			}
 			continue FOR_LOOP
+
 		case <-bcR.Quit():
 			break FOR_LOOP
 		}

blockchain/reactor_test.go

@@ -156,7 +156,7 @@ func makeTxs(height int64) (txs []types.Tx) {
 }
 
 func makeBlock(height int64, state sm.State) *types.Block {
-	block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit))
+	block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit), nil)
 	return block
 }
 
@@ -206,3 +206,4 @@ func (tp *bcrTestPeer) IsPersistent() bool                   { return true }
 func (tp *bcrTestPeer) Get(s string) interface{}             { return s }
 func (tp *bcrTestPeer) Set(string, interface{})              {}
 func (tp *bcrTestPeer) RemoteIP() net.IP                     { return []byte{127, 0, 0, 1} }
+func (tp *bcrTestPeer) OriginalAddr() *p2p.NetAddress        { return nil }

blockchain/store_test.go

@@ -126,7 +126,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		eraseSeenCommitInDB   bool
 	}{
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 		},
@@ -137,19 +137,19 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:     newBlock(&header2, commitAtH10),
+			block:     newBlock(header2, commitAtH10),
 			parts:     uncontiguousPartSet,
 			wantPanic: "only save contiguous blocks", // and incomplete and uncontiguous parts
 		},
 
 		{
-			block:     newBlock(&header1, commitAtH10),
+			block:     newBlock(header1, commitAtH10),
 			parts:     incompletePartSet,
 			wantPanic: "only save complete block", // incomplete parts
 		},
 
 		{
-			block:             newBlock(&header1, commitAtH10),
+			block:             newBlock(header1, commitAtH10),
 			parts:             validPartSet,
 			seenCommit:        seenCommit1,
 			corruptCommitInDB: true, // Corrupt the DB's commit entry
@@ -157,7 +157,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:            newBlock(&header1, commitAtH10),
+			block:            newBlock(header1, commitAtH10),
 			parts:            validPartSet,
 			seenCommit:       seenCommit1,
 			wantPanic:        "unmarshal to types.BlockMeta failed",
@@ -165,7 +165,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 
@@ -174,7 +174,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 
@@ -183,7 +183,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 
@@ -375,7 +375,7 @@ func doFn(fn func() (interface{}, error)) (res interface{}, err error, panicErr
 	return res, err, panicErr
 }
 
-func newBlock(hdr *types.Header, lastCommit *types.Commit) *types.Block {
+func newBlock(hdr types.Header, lastCommit *types.Commit) *types.Block {
 	return &types.Block{
 		Header:     hdr,
 		LastCommit: lastCommit,

blockchain/wire.go

@@ -2,12 +2,12 @@ package blockchain
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/types"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
 	RegisterBlockchainMessages(cdc)
-	crypto.RegisterAmino(cdc)
+	types.RegisterBlockAmino(cdc)
 }

cmd/priv_val_server/main.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"os"
 
-	crypto "github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
 
@@ -37,7 +37,7 @@ func main() {
 		*chainID,
 		*addr,
 		pv,
-		crypto.GenPrivKeyEd25519(),
+		ed25519.GenPrivKey(),
 	)
 	err := rs.Start()
 	if err != nil {

cmd/tendermint/commands/wire.go

@@ -2,11 +2,11 @@ package commands
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 }

codecov.yml

@@ -21,3 +21,4 @@ ignore:
   - "docs"
   - "DOCKER"
   - "scripts"
+  - "**/*.pb.go"

config/config.go

@@ -284,7 +284,6 @@ type P2PConfig struct {
 	Seeds string `mapstructure:"seeds"`
 
 	// Comma separated list of nodes to keep persistent connections to
-	// Do not add private peers to this list if you don't want them advertised
 	PersistentPeers string `mapstructure:"persistent_peers"`
 
 	// UPNP port forwarding
@@ -349,9 +348,9 @@ func DefaultP2PConfig() *P2PConfig {
 		AddrBookStrict:          true,
 		MaxNumPeers:             50,
 		FlushThrottleTimeout:    100,
-		MaxPacketMsgPayloadSize: 1024,   // 1 kB
-		SendRate:                512000, // 500 kB/s
-		RecvRate:                512000, // 500 kB/s
+		MaxPacketMsgPayloadSize: 1024,    // 1 kB
+		SendRate:                5120000, // 5 mB/s
+		RecvRate:                5120000, // 5 mB/s
 		PexReactor:              true,
 		SeedMode:                false,
 		AllowDuplicateIP:        true, // so non-breaking yet

config/toml.go

@@ -152,7 +152,6 @@ external_address = "{{ .P2P.ExternalAddress }}"
 seeds = "{{ .P2P.Seeds }}"
 
 # Comma separated list of nodes to keep persistent connections to
-# Do not add private peers to this list if you don't want them advertised
 persistent_peers = "{{ .P2P.PersistentPeers }}"
 
 # UPNP port forwarding

consensus/common_test.go

@@ -17,14 +17,14 @@ import (
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
 	cstypes "github.com/tendermint/tendermint/consensus/types"
+	cmn "github.com/tendermint/tendermint/libs/common"
+	dbm "github.com/tendermint/tendermint/libs/db"
+	"github.com/tendermint/tendermint/libs/log"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/privval"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
-	cmn "github.com/tendermint/tendermint/libs/common"
-	dbm "github.com/tendermint/tendermint/libs/db"
-	"github.com/tendermint/tendermint/libs/log"
 
 	"github.com/tendermint/tendermint/abci/example/counter"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
@@ -378,8 +378,11 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 		if i < nValidators {
 			privVal = privVals[i]
 		} else {
-			_, tempFilePath := cmn.Tempfile("priv_validator_")
-			privVal = privval.GenFilePV(tempFilePath)
+			tempFile, err := ioutil.TempFile("", "priv_validator_")
+			if err != nil {
+				panic(err)
+			}
+			privVal = privval.GenFilePV(tempFile.Name())
 		}
 
 		app := appFunc()
@@ -429,7 +432,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G
 func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []types.PrivValidator) {
 	genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower)
 	s0, _ := sm.MakeGenesisState(genDoc)
-	db := dbm.NewMemDB()
+	db := dbm.NewMemDB() // remove this ?
 	sm.SaveState(db, s0)
 	return s0, privValidators
 }

consensus/reactor.go

@@ -58,9 +58,6 @@ func NewConsensusReactor(consensusState *ConsensusState, fastSync bool) *Consens
 // broadcasted to other peers and starting state if we're not in fast sync.
 func (conR *ConsensusReactor) OnStart() error {
 	conR.Logger.Info("ConsensusReactor ", "fastSync", conR.FastSync())
-	if err := conR.BaseReactor.OnStart(); err != nil {
-		return err
-	}
 
 	conR.subscribeToBroadcastEvents()
 
@@ -77,9 +74,11 @@ func (conR *ConsensusReactor) OnStart() error {
 // OnStop implements BaseService by unsubscribing from events and stopping
 // state.
 func (conR *ConsensusReactor) OnStop() {
-	conR.BaseReactor.OnStop()
 	conR.unsubscribeFromBroadcastEvents()
 	conR.conS.Stop()
+	if !conR.FastSync() {
+		conR.conS.Wait()
+	}
 }
 
 // SwitchToConsensus switches from fast_sync mode to consensus mode.

consensus/reactor_test.go

@@ -4,15 +4,22 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path"
 	"runtime"
 	"runtime/pprof"
 	"sync"
 	"testing"
 	"time"
 
+	abcicli "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
+	abci "github.com/tendermint/tendermint/abci/types"
+	bc "github.com/tendermint/tendermint/blockchain"
 	cmn "github.com/tendermint/tendermint/libs/common"
+	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	mempl "github.com/tendermint/tendermint/mempool"
+	sm "github.com/tendermint/tendermint/state"
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
@@ -91,6 +98,120 @@ func TestReactorBasic(t *testing.T) {
 	}, css)
 }
 
+// Ensure we can process blocks with evidence
+func TestReactorWithEvidence(t *testing.T) {
+	nValidators := 4
+	testName := "consensus_reactor_test"
+	tickerFunc := newMockTickerFunc(true)
+	appFunc := newCounter
+
+	// heed the advice from https://www.sandimetz.com/blog/2016/1/20/the-wrong-abstraction
+	// to unroll unwieldy abstractions. Here we duplicate the code from:
+	// css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+
+	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
+	css := make([]*ConsensusState, nValidators)
+	logger := consensusLogger()
+	for i := 0; i < nValidators; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
+		thisConfig := ResetConfig(cmn.Fmt("%s_%d", testName, i))
+		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		app := appFunc()
+		vals := types.TM2PB.Validators(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		pv := privVals[i]
+		// duplicate code from:
+		// css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], app)
+
+		blockDB := dbm.NewMemDB()
+		blockStore := bc.NewBlockStore(blockDB)
+
+		// one for mempool, one for consensus
+		mtx := new(sync.Mutex)
+		proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
+		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+
+		// Make Mempool
+		mempool := mempl.NewMempool(thisConfig.Mempool, proxyAppConnMem, 0)
+		mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
+		if thisConfig.Consensus.WaitForTxs() {
+			mempool.EnableTxsAvailable()
+		}
+
+		// mock the evidence pool
+		// everyone includes evidence of another double signing
+		vIdx := (i + 1) % nValidators
+		evpool := newMockEvidencePool(privVals[vIdx].GetAddress())
+
+		// Make ConsensusState
+		blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+		cs := NewConsensusState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+		cs.SetLogger(log.TestingLogger().With("module", "consensus"))
+		cs.SetPrivValidator(pv)
+
+		eventBus := types.NewEventBus()
+		eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+		eventBus.Start()
+		cs.SetEventBus(eventBus)
+
+		cs.SetTimeoutTicker(tickerFunc())
+		cs.SetLogger(logger.With("validator", i, "module", "consensus"))
+
+		css[i] = cs
+	}
+
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, nValidators)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// wait till everyone makes the first new block with no evidence
+	timeoutWaitGroup(t, nValidators, func(j int) {
+		blockI := <-eventChans[j]
+		block := blockI.(types.EventDataNewBlock).Block
+		assert.True(t, len(block.Evidence.Evidence) == 0)
+	}, css)
+
+	// second block should have evidence
+	timeoutWaitGroup(t, nValidators, func(j int) {
+		blockI := <-eventChans[j]
+		block := blockI.(types.EventDataNewBlock).Block
+		assert.True(t, len(block.Evidence.Evidence) > 0)
+	}, css)
+}
+
+// mock evidence pool returns no evidence for block 1,
+// and returnes one piece for all higher blocks. The one piece
+// is for a given validator at block 1.
+type mockEvidencePool struct {
+	height int
+	ev     []types.Evidence
+}
+
+func newMockEvidencePool(val []byte) *mockEvidencePool {
+	return &mockEvidencePool{
+		ev: []types.Evidence{types.NewMockGoodEvidence(1, 1, val)},
+	}
+}
+
+func (m *mockEvidencePool) PendingEvidence() []types.Evidence {
+	if m.height > 0 {
+		return m.ev
+	}
+	return nil
+}
+func (m *mockEvidencePool) AddEvidence(types.Evidence) error { return nil }
+func (m *mockEvidencePool) Update(block *types.Block, state sm.State) {
+	if m.height > 0 {
+		if len(block.Evidence.Evidence) == 0 {
+			panic("block has no evidence")
+		}
+	}
+	m.height += 1
+}
+
+//------------------------------------
+
 // Ensure a testnet sends proposal heartbeats and makes blocks when there are txs
 func TestReactorProposalHeartbeats(t *testing.T) {
 	N := 4

consensus/replay.go

@@ -227,7 +227,7 @@ func (h *Handshaker) NBlocks() int {
 func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 
 	// Handshake is done via ABCI Info on the query conn.
-	res, err := proxyApp.Query().InfoSync(abci.RequestInfo{version.Version})
+	res, err := proxyApp.Query().InfoSync(abci.RequestInfo{Version: version.Version})
 	if err != nil {
 		return fmt.Errorf("Error calling Info: %v", err)
 	}
@@ -269,11 +269,11 @@ func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight
 		validators := types.TM2PB.Validators(state.Validators)
 		csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams)
 		req := abci.RequestInitChain{
-			Time:            h.genDoc.GenesisTime.Unix(), // TODO
+			Time:            h.genDoc.GenesisTime,
 			ChainId:         h.genDoc.ChainID,
 			ConsensusParams: csParams,
 			Validators:      validators,
-			AppStateBytes:   h.genDoc.AppStateJSON,
+			AppStateBytes:   h.genDoc.AppState,
 		}
 		res, err := proxyApp.Consensus().InitChainSync(req)
 		if err != nil {

consensus/replay_test.go

@@ -376,7 +376,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	defer proxyApp.Stop()
 
 	// get the latest app hash from the app
-	res, err := proxyApp.Query().InfoSync(abci.RequestInfo{""})
+	res, err := proxyApp.Query().InfoSync(abci.RequestInfo{Version: ""})
 	if err != nil {
 		t.Fatal(err)
 	}

consensus/state.go

@@ -81,7 +81,7 @@ type ConsensusState struct {
 	evpool     sm.EvidencePool
 
 	// internal state
-	mtx sync.Mutex
+	mtx sync.RWMutex
 	cstypes.RoundState
 	state sm.State // State until height-1.
 
@@ -192,15 +192,15 @@ func (cs *ConsensusState) String() string {
 
 // GetState returns a copy of the chain state.
 func (cs *ConsensusState) GetState() sm.State {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	return cs.state.Copy()
 }
 
 // GetRoundState returns a shallow copy of the internal consensus state.
 func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 
 	rs := cs.RoundState // copy
 	return &rs
@@ -208,24 +208,24 @@ func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
 
 // GetRoundStateJSON returns a json of RoundState, marshalled using go-amino.
 func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 
 	return cdc.MarshalJSON(cs.RoundState)
 }
 
 // GetRoundStateSimpleJSON returns a json of RoundStateSimple, marshalled using go-amino.
 func (cs *ConsensusState) GetRoundStateSimpleJSON() ([]byte, error) {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 
 	return cdc.MarshalJSON(cs.RoundState.RoundStateSimple())
 }
 
 // GetValidators returns a copy of the current validators.
 func (cs *ConsensusState) GetValidators() (int64, []*types.Validator) {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators
 }
 
@@ -245,8 +245,8 @@ func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) {
 
 // LoadCommit loads the commit for a given height.
 func (cs *ConsensusState) LoadCommit(height int64) *types.Commit {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	if height == cs.blockStore.Height() {
 		return cs.blockStore.LoadSeenCommit(height)
 	}
@@ -553,9 +553,30 @@ func (cs *ConsensusState) newStep() {
 // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities.
 // ConsensusState must be locked before any internal state is updated.
 func (cs *ConsensusState) receiveRoutine(maxSteps int) {
+	onExit := func(cs *ConsensusState) {
+		// NOTE: the internalMsgQueue may have signed messages from our
+		// priv_val that haven't hit the WAL, but its ok because
+		// priv_val tracks LastSig
+
+		// close wal now that we're done writing to it
+		cs.wal.Stop()
+		cs.wal.Wait()
+
+		close(cs.done)
+	}
+
 	defer func() {
 		if r := recover(); r != nil {
 			cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack()))
+			// stop gracefully
+			//
+			// NOTE: We most probably shouldn't be running any further when there is
+			// some unexpected panic. Some unknown error happened, and so we don't
+			// know if that will result in the validator signing an invalid thing. It
+			// might be worthwhile to explore a mechanism for manual resuming via
+			// some console or secure RPC system, but for now, halting the chain upon
+			// unexpected consensus bugs sounds like the better option.
+			onExit(cs)
 		}
 	}()
 
@@ -571,8 +592,8 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 		var mi msgInfo
 
 		select {
-		case height := <-cs.mempool.TxsAvailable():
-			cs.handleTxsAvailable(height)
+		case <-cs.mempool.TxsAvailable():
+			cs.handleTxsAvailable()
 		case mi = <-cs.peerMsgQueue:
 			cs.wal.Write(mi)
 			// handles proposals, block parts, votes
@@ -588,16 +609,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 			// go to the next step
 			cs.handleTimeout(ti, rs)
 		case <-cs.Quit():
-
-			// NOTE: the internalMsgQueue may have signed messages from our
-			// priv_val that haven't hit the WAL, but its ok because
-			// priv_val tracks LastSig
-
-			// close wal now that we're done writing to it
-			cs.wal.Stop()
-			cs.wal.Wait()
-
-			close(cs.done)
+			onExit(cs)
 			return
 		}
 	}
@@ -683,11 +695,11 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) {
 
 }
 
-func (cs *ConsensusState) handleTxsAvailable(height int64) {
+func (cs *ConsensusState) handleTxsAvailable() {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	// we only need to do this for round 0
-	cs.enterPropose(height, 0)
+	cs.enterPropose(cs.Height, 0)
 }
 
 //-----------------------------------------------------------------------------
@@ -907,6 +919,8 @@ func (cs *ConsensusState) isProposalComplete() bool {
 }
 
 // Create the next block to propose and return it.
+// We really only need to return the parts, but the block
+// is returned for convenience so we can log the proposal block.
 // Returns nil block upon error.
 // NOTE: keep it side-effect free for clarity.
 func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
@@ -926,9 +940,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 
 	// Mempool validated transactions
 	txs := cs.mempool.Reap(cs.state.ConsensusParams.BlockSize.MaxTxs)
-	block, parts := cs.state.MakeBlock(cs.Height, txs, commit)
 	evidence := cs.evpool.PendingEvidence()
-	block.AddEvidence(evidence)
+	block, parts := cs.state.MakeBlock(cs.Height, txs, commit, evidence)
 	return block, parts
 }
 

consensus/types/round_state_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
-	"github.com/tendermint/tendermint/types"
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
+	"github.com/tendermint/tendermint/types"
 )
 
 func BenchmarkRoundStateDeepCopy(b *testing.B) {
@@ -23,7 +23,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 			Hash: cmn.RandBytes(20),
 		},
 	}
-	sig := crypto.SignatureEd25519{}
+	sig := make([]byte, ed25519.SignatureEd25519Size)
 	for i := 0; i < nval; i++ {
 		precommits[i] = &types.Vote{
 			ValidatorAddress: types.Address(cmn.RandBytes(20)),
@@ -38,7 +38,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 	}
 	// Random block
 	block := &types.Block{
-		Header: &types.Header{
+		Header: types.Header{
 			ChainID:         cmn.RandStr(12),
 			Time:            time.Now(),
 			LastBlockID:     blockID,
@@ -50,7 +50,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 			LastResultsHash: cmn.RandBytes(20),
 			EvidenceHash:    cmn.RandBytes(20),
 		},
-		Data: &types.Data{
+		Data: types.Data{
 			Txs: txs,
 		},
 		Evidence: types.EvidenceData{},

consensus/types/wire.go

@@ -2,11 +2,11 @@ package types
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/types"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	types.RegisterBlockAmino(cdc)
 }

consensus/wire.go

@@ -2,7 +2,7 @@ package consensus
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/types"
 )
 
 var cdc = amino.NewCodec()
@@ -10,5 +10,5 @@ var cdc = amino.NewCodec()
 func init() {
 	RegisterConsensusMessages(cdc)
 	RegisterWALMessages(cdc)
-	crypto.RegisterAmino(cdc)
+	types.RegisterBlockAmino(cdc)
 }

crypto/README.md

@@ -3,8 +3,15 @@
 crypto is the cryptographic package adapted for Tendermint's uses
 
 ## Importing it
+To get the interfaces,
 `import "github.com/tendermint/tendermint/crypto"`
 
+For any specific algorithm, use its specific module e.g.
+`import "github.com/tendermint/tendermint/crypto/ed25519"`
+
+If you want to decode bytes into one of the types, but don't care about the specific algorithm, use
+`import "github.com/tendermint/tendermint/crypto/amino"`
+
 ## Binary encoding
 
 For Binary encoding, please refer to the [Tendermint encoding spec](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/encoding.md).
@@ -16,10 +23,8 @@ crypto `.Bytes()` uses Amino:binary encoding, but Amino:JSON is also supported.
 ```go
 Example Amino:JSON encodings:
 
-crypto.PrivKeyEd25519     - {"type":"954568A3288910","value":"EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="}
-crypto.SignatureEd25519   - {"type":"6BF5903DA1DB28","value":"77sQNZOrf7ltExpf7AV1WaYPCHbyRLgjBsoWVzcduuLk+jIGmYk+s5R6Emm29p12HeiNAuhUJgdFGmwkpeGJCA=="}
-crypto.PubKeyEd25519      - {"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="}
+ed25519.PrivKeyEd25519     - {"type":"954568A3288910","value":"EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="}
+ed25519.PubKeyEd25519      - {"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="}
 crypto.PrivKeySecp256k1   - {"type":"019E82E1B0F798","value":"zx4Pnh67N+g2V+5vZbQzEyRerX9c4ccNZOVzM9RvJ0Y="}
-crypto.SignatureSecp256k1 - {"type":"6D1EA416E1FEE8","value":"MEUCIQCIg5TqS1l7I+MKTrSPIuUN2+4m5tA29dcauqn3NhEJ2wIgICaZ+lgRc5aOTVahU/XoLopXKn8BZcl0bnuYWLvohR8="}
 crypto.PubKeySecp256k1    - {"type":"F8CCEAEB5AE980","value":"A8lPKJXcNl5VHt1FK8a244K9EJuS4WX1hFBnwisi0IJx"}
 ```

crypto/amino.go

@@ -1,37 +0,0 @@
-package crypto
-
-import (
-	amino "github.com/tendermint/go-amino"
-)
-
-var cdc = amino.NewCodec()
-
-func init() {
-	// NOTE: It's important that there be no conflicts here,
-	// as that would change the canonical representations,
-	// and therefore change the address.
-	// TODO: Add feature to go-amino to ensure that there
-	// are no conflicts.
-	RegisterAmino(cdc)
-}
-
-// RegisterAmino registers all crypto related types in the given (amino) codec.
-func RegisterAmino(cdc *amino.Codec) {
-	cdc.RegisterInterface((*PubKey)(nil), nil)
-	cdc.RegisterConcrete(PubKeyEd25519{},
-		"tendermint/PubKeyEd25519", nil)
-	cdc.RegisterConcrete(PubKeySecp256k1{},
-		"tendermint/PubKeySecp256k1", nil)
-
-	cdc.RegisterInterface((*PrivKey)(nil), nil)
-	cdc.RegisterConcrete(PrivKeyEd25519{},
-		"tendermint/PrivKeyEd25519", nil)
-	cdc.RegisterConcrete(PrivKeySecp256k1{},
-		"tendermint/PrivKeySecp256k1", nil)
-
-	cdc.RegisterInterface((*Signature)(nil), nil)
-	cdc.RegisterConcrete(SignatureEd25519{},
-		"tendermint/SignatureEd25519", nil)
-	cdc.RegisterConcrete(SignatureSecp256k1{},
-		"tendermint/SignatureSecp256k1", nil)
-}

crypto/armor/armor.go

@@ -1,4 +1,4 @@
-package crypto
+package armor
 
 import (
 	"bytes"

crypto/armor/armor_test.go

@@ -1,4 +1,4 @@
-package crypto
+package armor
 
 import (
 	"testing"

crypto/crypto.go

@@ -0,0 +1,30 @@
+package crypto
+
+import (
+	cmn "github.com/tendermint/tendermint/libs/common"
+)
+
+type PrivKey interface {
+	Bytes() []byte
+	Sign(msg []byte) ([]byte, error)
+	PubKey() PubKey
+	Equals(PrivKey) bool
+}
+
+// An address is a []byte, but hex-encoded even in JSON.
+// []byte leaves us the option to change the address length.
+// Use an alias so Unmarshal methods (with ptr receivers) are available too.
+type Address = cmn.HexBytes
+
+type PubKey interface {
+	Address() Address
+	Bytes() []byte
+	VerifyBytes(msg []byte, sig []byte) bool
+	Equals(PubKey) bool
+}
+
+type Symmetric interface {
+	Keygen() []byte
+	Encrypt(plaintext []byte, secret []byte) (ciphertext []byte)
+	Decrypt(ciphertext []byte, secret []byte) (plaintext []byte, err error)
+}

crypto/doc.go

@@ -22,7 +22,7 @@
 //     pubKey := key.PubKey()
 
 // For example:
-//     privKey, err := crypto.GenPrivKeyEd25519()
+//     privKey, err := ed25519.GenPrivKey()
 //     if err != nil {
 // 	...
 //     }

crypto/ed25519/bench_test.go

@@ -0,0 +1,26 @@
+package ed25519
+
+import (
+	"io"
+	"testing"
+
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/internal/benchmarking"
+)
+
+func BenchmarkKeyGeneration(b *testing.B) {
+	benchmarkKeygenWrapper := func(reader io.Reader) crypto.PrivKey {
+		return genPrivKey(reader)
+	}
+	benchmarking.BenchmarkKeyGeneration(b, benchmarkKeygenWrapper)
+}
+
+func BenchmarkSigning(b *testing.B) {
+	priv := GenPrivKey()
+	benchmarking.BenchmarkSigning(b, priv)
+}
+
+func BenchmarkVerification(b *testing.B) {
+	priv := GenPrivKey()
+	benchmarking.BenchmarkVerification(b, priv)
+}

crypto/ed25519/ed25519.go

@@ -0,0 +1,196 @@
+package ed25519
+
+import (
+	"bytes"
+	"crypto/subtle"
+	"fmt"
+	"io"
+
+	"github.com/tendermint/ed25519"
+	"github.com/tendermint/ed25519/extra25519"
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+)
+
+//-------------------------------------
+
+var _ crypto.PrivKey = PrivKeyEd25519{}
+
+const (
+	Ed25519PrivKeyAminoRoute = "tendermint/PrivKeyEd25519"
+	Ed25519PubKeyAminoRoute  = "tendermint/PubKeyEd25519"
+	// Size of an Edwards25519 signature. Namely the size of a compressed
+	// Edwards25519 point, and a field element. Both of which are 32 bytes.
+	SignatureEd25519Size = 64
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
+	cdc.RegisterConcrete(PubKeyEd25519{},
+		Ed25519PubKeyAminoRoute, nil)
+
+	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
+	cdc.RegisterConcrete(PrivKeyEd25519{},
+		Ed25519PrivKeyAminoRoute, nil)
+}
+
+// PrivKeyEd25519 implements crypto.PrivKey.
+type PrivKeyEd25519 [64]byte
+
+// Bytes marshals the privkey using amino encoding.
+func (privKey PrivKeyEd25519) Bytes() []byte {
+	return cdc.MustMarshalBinaryBare(privKey)
+}
+
+// Sign produces a signature on the provided message.
+func (privKey PrivKeyEd25519) Sign(msg []byte) ([]byte, error) {
+	privKeyBytes := [64]byte(privKey)
+	signatureBytes := ed25519.Sign(&privKeyBytes, msg)
+	return signatureBytes[:], nil
+}
+
+// PubKey gets the corresponding public key from the private key.
+func (privKey PrivKeyEd25519) PubKey() crypto.PubKey {
+	privKeyBytes := [64]byte(privKey)
+	initialized := false
+	// If the latter 32 bytes of the privkey are all zero, compute the pubkey
+	// otherwise privkey is initialized and we can use the cached value inside
+	// of the private key.
+	for _, v := range privKeyBytes[32:] {
+		if v != 0 {
+			initialized = true
+			break
+		}
+	}
+	if initialized {
+		var pubkeyBytes [PubKeyEd25519Size]byte
+		copy(pubkeyBytes[:], privKeyBytes[32:])
+		return PubKeyEd25519(pubkeyBytes)
+	}
+
+	pubBytes := *ed25519.MakePublicKey(&privKeyBytes)
+	return PubKeyEd25519(pubBytes)
+}
+
+// Equals - you probably don't need to use this.
+// Runs in constant time based on length of the keys.
+func (privKey PrivKeyEd25519) Equals(other crypto.PrivKey) bool {
+	if otherEd, ok := other.(PrivKeyEd25519); ok {
+		return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1
+	} else {
+		return false
+	}
+}
+
+// ToCurve25519 takes a private key and returns its representation on
+// Curve25519. Curve25519 is birationally equivalent to Edwards25519,
+// which Ed25519 uses internally. This method is intended for use in
+// an X25519 Diffie Hellman key exchange.
+func (privKey PrivKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte {
+	keyCurve25519 := new([32]byte)
+	privKeyBytes := [64]byte(privKey)
+	extra25519.PrivateKeyToCurve25519(keyCurve25519, &privKeyBytes)
+	return keyCurve25519
+}
+
+// GenPrivKey generates a new ed25519 private key.
+// It uses OS randomness in conjunction with the current global random seed
+// in tendermint/libs/common to generate the private key.
+func GenPrivKey() PrivKeyEd25519 {
+	return genPrivKey(crypto.CReader())
+}
+
+// genPrivKey generates a new ed25519 private key using the provided reader.
+func genPrivKey(rand io.Reader) PrivKeyEd25519 {
+	privKey := new([64]byte)
+	_, err := io.ReadFull(rand, privKey[:32])
+	if err != nil {
+		panic(err)
+	}
+	// ed25519.MakePublicKey(privKey) alters the last 32 bytes of privKey.
+	// It places the pubkey in the last 32 bytes of privKey, and returns the
+	// public key.
+	ed25519.MakePublicKey(privKey)
+	return PrivKeyEd25519(*privKey)
+}
+
+// GenPrivKeyFromSecret hashes the secret with SHA2, and uses
+// that 32 byte output to create the private key.
+// NOTE: secret should be the output of a KDF like bcrypt,
+// if it's derived from user input.
+func GenPrivKeyFromSecret(secret []byte) PrivKeyEd25519 {
+	privKey32 := crypto.Sha256(secret) // Not Ripemd160 because we want 32 bytes.
+	privKey := new([64]byte)
+	copy(privKey[:32], privKey32)
+	// ed25519.MakePublicKey(privKey) alters the last 32 bytes of privKey.
+	// It places the pubkey in the last 32 bytes of privKey, and returns the
+	// public key.
+	ed25519.MakePublicKey(privKey)
+	return PrivKeyEd25519(*privKey)
+}
+
+//-------------------------------------
+
+var _ crypto.PubKey = PubKeyEd25519{}
+
+// PubKeyEd25519Size is the number of bytes in an Ed25519 signature.
+const PubKeyEd25519Size = 32
+
+// PubKeyEd25519 implements crypto.PubKey for the Ed25519 signature scheme.
+type PubKeyEd25519 [PubKeyEd25519Size]byte
+
+// Address is the SHA256-20 of the raw pubkey bytes.
+func (pubKey PubKeyEd25519) Address() crypto.Address {
+	return crypto.Address(tmhash.Sum(pubKey[:]))
+}
+
+// Bytes marshals the PubKey using amino encoding.
+func (pubKey PubKeyEd25519) Bytes() []byte {
+	bz, err := cdc.MarshalBinaryBare(pubKey)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (pubKey PubKeyEd25519) VerifyBytes(msg []byte, sig_ []byte) bool {
+	// make sure we use the same algorithm to sign
+	if len(sig_) != SignatureEd25519Size {
+		return false
+	}
+	sig := new([SignatureEd25519Size]byte)
+	copy(sig[:], sig_)
+	pubKeyBytes := [PubKeyEd25519Size]byte(pubKey)
+	return ed25519.Verify(&pubKeyBytes, msg, sig)
+}
+
+// ToCurve25519 takes a public key and returns its representation on
+// Curve25519. Curve25519 is birationally equivalent to Edwards25519,
+// which Ed25519 uses internally. This method is intended for use in
+// an X25519 Diffie Hellman key exchange.
+//
+// If there is an error, then this function returns nil.
+func (pubKey PubKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte {
+	keyCurve25519, pubKeyBytes := new([PubKeyEd25519Size]byte), [PubKeyEd25519Size]byte(pubKey)
+	ok := extra25519.PublicKeyToCurve25519(keyCurve25519, &pubKeyBytes)
+	if !ok {
+		return nil
+	}
+	return keyCurve25519
+}
+
+func (pubKey PubKeyEd25519) String() string {
+	return fmt.Sprintf("PubKeyEd25519{%X}", pubKey[:])
+}
+
+// nolint: golint
+func (pubKey PubKeyEd25519) Equals(other crypto.PubKey) bool {
+	if otherEd, ok := other.(PubKeyEd25519); ok {
+		return bytes.Equal(pubKey[:], otherEd[:])
+	} else {
+		return false
+	}
+}

crypto/ed25519/ed25519_test.go

@@ -0,0 +1,29 @@
+package ed25519_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+)
+
+func TestSignAndValidateEd25519(t *testing.T) {
+
+	privKey := ed25519.GenPrivKey()
+	pubKey := privKey.PubKey()
+
+	msg := crypto.CRandBytes(128)
+	sig, err := privKey.Sign(msg)
+	require.Nil(t, err)
+
+	// Test the signature
+	assert.True(t, pubKey.VerifyBytes(msg, sig))
+
+	// Mutate the signature, just one bit.
+	// TODO: Replace this with a much better fuzzer, tendermint/ed25519/issues/10
+	sig[7] ^= byte(0x01)
+
+	assert.False(t, pubKey.VerifyBytes(msg, sig))
+}

crypto/encoding/amino/amino.go

@@ -0,0 +1,46 @@
+package cryptoAmino
+
+import (
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	// NOTE: It's important that there be no conflicts here,
+	// as that would change the canonical representations,
+	// and therefore change the address.
+	// TODO: Remove above note when
+	// https://github.com/tendermint/go-amino/issues/9
+	// is resolved
+	RegisterAmino(cdc)
+}
+
+// RegisterAmino registers all crypto related types in the given (amino) codec.
+func RegisterAmino(cdc *amino.Codec) {
+	// These are all written here instead of
+	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
+	cdc.RegisterConcrete(ed25519.PubKeyEd25519{},
+		"tendermint/PubKeyEd25519", nil)
+	cdc.RegisterConcrete(secp256k1.PubKeySecp256k1{},
+		"tendermint/PubKeySecp256k1", nil)
+
+	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
+	cdc.RegisterConcrete(ed25519.PrivKeyEd25519{},
+		"tendermint/PrivKeyEd25519", nil)
+	cdc.RegisterConcrete(secp256k1.PrivKeySecp256k1{},
+		"tendermint/PrivKeySecp256k1", nil)
+}
+
+func PrivKeyFromBytes(privKeyBytes []byte) (privKey crypto.PrivKey, err error) {
+	err = cdc.UnmarshalBinaryBare(privKeyBytes, &privKey)
+	return
+}
+
+func PubKeyFromBytes(pubKeyBytes []byte) (pubKey crypto.PubKey, err error) {
+	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
+	return
+}

crypto/encoding/amino/encode_test.go

@@ -1,4 +1,4 @@
-package crypto
+package cryptoAmino
 
 import (
 	"os"
@@ -6,18 +6,23 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
 )
 
 type byter interface {
 	Bytes() []byte
 }
 
-func checkAminoBinary(t *testing.T, src byter, dst interface{}, size int) {
+func checkAminoBinary(t *testing.T, src, dst interface{}, size int) {
 	// Marshal to binary bytes.
 	bz, err := cdc.MarshalBinaryBare(src)
 	require.Nil(t, err, "%+v", err)
-	// Make sure this is compatible with current (Bytes()) encoding.
-	assert.Equal(t, src.Bytes(), bz, "Amino binary vs Bytes() mismatch")
+	if byterSrc, ok := src.(byter); ok {
+		// Make sure this is compatible with current (Bytes()) encoding.
+		assert.Equal(t, byterSrc.Bytes(), bz, "Amino binary vs Bytes() mismatch")
+	}
 	// Make sure we have the expected length.
 	if size != -1 {
 		assert.Equal(t, size, len(bz), "Amino binary size mismatch")
@@ -50,70 +55,72 @@ func ExamplePrintRegisteredTypes() {
 	//| PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE987 | 0x21 |  |
 	//| PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288910 | 0x40 |  |
 	//| PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79B | 0x20 |  |
-	//| SignatureEd25519 | tendermint/SignatureEd25519 | 0x2031EA53 | 0x40 |  |
-	//| SignatureSecp256k1 | tendermint/SignatureSecp256k1 | 0x7FC4A495 | variable |  |
 }
 
 func TestKeyEncodings(t *testing.T) {
 	cases := []struct {
-		privKey           PrivKey
+		privKey           crypto.PrivKey
 		privSize, pubSize int // binary sizes
 	}{
 		{
-			privKey:  GenPrivKeyEd25519(),
+			privKey:  ed25519.GenPrivKey(),
 			privSize: 69,
 			pubSize:  37,
 		},
 		{
-			privKey:  GenPrivKeySecp256k1(),
+			privKey:  secp256k1.GenPrivKey(),
 			privSize: 37,
 			pubSize:  38,
 		},
 	}
 
-	for _, tc := range cases {
+	for tcIndex, tc := range cases {
 
 		// Check (de/en)codings of PrivKeys.
-		var priv2, priv3 PrivKey
+		var priv2, priv3 crypto.PrivKey
 		checkAminoBinary(t, tc.privKey, &priv2, tc.privSize)
-		assert.EqualValues(t, tc.privKey, priv2)
+		assert.EqualValues(t, tc.privKey, priv2, "tc #%d", tcIndex)
 		checkAminoJSON(t, tc.privKey, &priv3, false) // TODO also check Prefix bytes.
-		assert.EqualValues(t, tc.privKey, priv3)
+		assert.EqualValues(t, tc.privKey, priv3, "tc #%d", tcIndex)
 
 		// Check (de/en)codings of Signatures.
-		var sig1, sig2, sig3 Signature
+		var sig1, sig2 []byte
 		sig1, err := tc.privKey.Sign([]byte("something"))
-		assert.NoError(t, err)
+		assert.NoError(t, err, "tc #%d", tcIndex)
 		checkAminoBinary(t, sig1, &sig2, -1) // Signature size changes for Secp anyways.
-		assert.EqualValues(t, sig1, sig2)
-		checkAminoJSON(t, sig1, &sig3, false) // TODO also check Prefix bytes.
-		assert.EqualValues(t, sig1, sig3)
+		assert.EqualValues(t, sig1, sig2, "tc #%d", tcIndex)
 
 		// Check (de/en)codings of PubKeys.
 		pubKey := tc.privKey.PubKey()
-		var pub2, pub3 PubKey
+		var pub2, pub3 crypto.PubKey
 		checkAminoBinary(t, pubKey, &pub2, tc.pubSize)
-		assert.EqualValues(t, pubKey, pub2)
+		assert.EqualValues(t, pubKey, pub2, "tc #%d", tcIndex)
 		checkAminoJSON(t, pubKey, &pub3, false) // TODO also check Prefix bytes.
-		assert.EqualValues(t, pubKey, pub3)
+		assert.EqualValues(t, pubKey, pub3, "tc #%d", tcIndex)
 	}
 }
 
 func TestNilEncodings(t *testing.T) {
 
 	// Check nil Signature.
-	var a, b Signature
+	var a, b []byte
 	checkAminoJSON(t, &a, &b, true)
 	assert.EqualValues(t, a, b)
 
 	// Check nil PubKey.
-	var c, d PubKey
+	var c, d crypto.PubKey
 	checkAminoJSON(t, &c, &d, true)
 	assert.EqualValues(t, c, d)
 
 	// Check nil PrivKey.
-	var e, f PrivKey
+	var e, f crypto.PrivKey
 	checkAminoJSON(t, &e, &f, true)
 	assert.EqualValues(t, e, f)
 
 }
+
+func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) {
+	pk, err := PubKeyFromBytes([]byte("foo"))
+	require.NotNil(t, err, "expecting a non-nil error")
+	require.Nil(t, pk, "expecting an empty public key on error")
+}

crypto/hash.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"crypto/sha256"
+
 	"golang.org/x/crypto/ripemd160"
 )
 

crypto/hkdfchacha20poly1305/hkdfchachapoly.go

@@ -1,105 +0,0 @@
-// Package hkdfchacha20poly1305 creates an AEAD using hkdf, chacha20, and poly1305
-// When sealing and opening, the hkdf is used to obtain the nonce and subkey for
-// chacha20. Other than the change for the how the subkey and nonce for chacha
-// are obtained, this is the same as chacha20poly1305
-package hkdfchacha20poly1305
-
-import (
-	"crypto/cipher"
-	"crypto/sha256"
-	"errors"
-	"io"
-
-	"golang.org/x/crypto/chacha20poly1305"
-	"golang.org/x/crypto/hkdf"
-)
-
-type hkdfchacha20poly1305 struct {
-	key [KeySize]byte
-}
-
-const (
-	// KeySize is the size of the key used by this AEAD, in bytes.
-	KeySize = 32
-	// NonceSize is the size of the nonce used with this AEAD, in bytes.
-	NonceSize = 24
-	// TagSize is the size added from poly1305
-	TagSize = 16
-	// MaxPlaintextSize is the max size that can be passed into a single call of Seal
-	MaxPlaintextSize = (1 << 38) - 64
-	// MaxCiphertextSize is the max size that can be passed into a single call of Open,
-	// this differs from plaintext size due to the tag
-	MaxCiphertextSize = (1 << 38) - 48
-	// HkdfInfo is the parameter used internally for Hkdf's info parameter.
-	HkdfInfo = "TENDERMINT_SECRET_CONNECTION_FRAME_KEY_DERIVE"
-)
-
-//New xChaChapoly1305 AEAD with 24 byte nonces
-func New(key []byte) (cipher.AEAD, error) {
-	if len(key) != KeySize {
-		return nil, errors.New("chacha20poly1305: bad key length")
-	}
-	ret := new(hkdfchacha20poly1305)
-	copy(ret.key[:], key)
-	return ret, nil
-
-}
-func (c *hkdfchacha20poly1305) NonceSize() int {
-	return NonceSize
-}
-
-func (c *hkdfchacha20poly1305) Overhead() int {
-	return TagSize
-}
-
-func (c *hkdfchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
-	if len(nonce) != NonceSize {
-		panic("hkdfchacha20poly1305: bad nonce length passed to Seal")
-	}
-
-	if uint64(len(plaintext)) > MaxPlaintextSize {
-		panic("hkdfchacha20poly1305: plaintext too large")
-	}
-
-	subKey, chachaNonce := getSubkeyAndChachaNonceFromHkdf(&c.key, &nonce)
-
-	aead, err := chacha20poly1305.New(subKey[:])
-	if err != nil {
-		panic("hkdfchacha20poly1305: failed to initialize chacha20poly1305")
-	}
-
-	return aead.Seal(dst, chachaNonce[:], plaintext, additionalData)
-}
-
-func (c *hkdfchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
-	if len(nonce) != NonceSize {
-		return nil, errors.New("hkdfchacha20poly1305: bad nonce length passed to Open")
-	}
-	if uint64(len(ciphertext)) > MaxCiphertextSize {
-		return nil, errors.New("hkdfchacha20poly1305: ciphertext too large")
-	}
-
-	subKey, chachaNonce := getSubkeyAndChachaNonceFromHkdf(&c.key, &nonce)
-
-	aead, err := chacha20poly1305.New(subKey[:])
-	if err != nil {
-		panic("hkdfchacha20poly1305: failed to initialize chacha20poly1305")
-	}
-
-	return aead.Open(dst, chachaNonce[:], ciphertext, additionalData)
-}
-
-func getSubkeyAndChachaNonceFromHkdf(cKey *[32]byte, nonce *[]byte) (
-	subKey [KeySize]byte, chachaNonce [chacha20poly1305.NonceSize]byte) {
-	hash := sha256.New
-	hkdf := hkdf.New(hash, (*cKey)[:], *nonce, []byte(HkdfInfo))
-	_, err := io.ReadFull(hkdf, subKey[:])
-	if err != nil {
-		panic("hkdfchacha20poly1305: failed to read subkey from hkdf")
-	}
-	_, err = io.ReadFull(hkdf, chachaNonce[:])
-	if err != nil {
-		panic("hkdfchacha20poly1305: failed to read chachaNonce from hkdf")
-	}
-	return
-}

crypto/internal/benchmarking/bench.go

@@ -0,0 +1,88 @@
+package benchmarking
+
+import (
+	"io"
+	"testing"
+
+	"github.com/tendermint/tendermint/crypto"
+)
+
+// The code in this file is adapted from agl/ed25519.
+// As such it is under the following license.
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found at the bottom of this file.
+
+type zeroReader struct{}
+
+func (zeroReader) Read(buf []byte) (int, error) {
+	for i := range buf {
+		buf[i] = 0
+	}
+	return len(buf), nil
+}
+
+// BenchmarkKeyGeneration benchmarks the given key generation algorithm using
+// a dummy reader.
+func BenchmarkKeyGeneration(b *testing.B, GenerateKey func(reader io.Reader) crypto.PrivKey) {
+	var zero zeroReader
+	for i := 0; i < b.N; i++ {
+		GenerateKey(zero)
+	}
+}
+
+// BenchmarkSigning benchmarks the given signing algorithm using
+// the provided privkey.
+func BenchmarkSigning(b *testing.B, priv crypto.PrivKey) {
+	message := []byte("Hello, world!")
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		priv.Sign(message)
+	}
+}
+
+// BenchmarkVerification benchmarks the given verification algorithm using
+// the provided privkey on a constant message.
+func BenchmarkVerification(b *testing.B, priv crypto.PrivKey) {
+	pub := priv.PubKey()
+	// use a short message, so this time doesn't get dominated by hashing.
+	message := []byte("Hello, world!")
+	signature, err := priv.Sign(message)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pub.VerifyBytes(message, signature)
+	}
+}
+
+// Below is the aforementioned license.
+
+// Copyright (c) 2012 The Go Authors. All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

crypto/priv_key.go

@@ -1,164 +0,0 @@
-package crypto
-
-import (
-	"crypto/subtle"
-
-	secp256k1 "github.com/btcsuite/btcd/btcec"
-	"github.com/tendermint/ed25519"
-	"github.com/tendermint/ed25519/extra25519"
-)
-
-func PrivKeyFromBytes(privKeyBytes []byte) (privKey PrivKey, err error) {
-	err = cdc.UnmarshalBinaryBare(privKeyBytes, &privKey)
-	return
-}
-
-//----------------------------------------
-
-type PrivKey interface {
-	Bytes() []byte
-	Sign(msg []byte) (Signature, error)
-	PubKey() PubKey
-	Equals(PrivKey) bool
-}
-
-//-------------------------------------
-
-var _ PrivKey = PrivKeyEd25519{}
-
-// Implements PrivKey
-type PrivKeyEd25519 [64]byte
-
-func (privKey PrivKeyEd25519) Bytes() []byte {
-	return cdc.MustMarshalBinaryBare(privKey)
-}
-
-func (privKey PrivKeyEd25519) Sign(msg []byte) (Signature, error) {
-	privKeyBytes := [64]byte(privKey)
-	signatureBytes := ed25519.Sign(&privKeyBytes, msg)
-	return SignatureEd25519(*signatureBytes), nil
-}
-
-func (privKey PrivKeyEd25519) PubKey() PubKey {
-	privKeyBytes := [64]byte(privKey)
-	pubBytes := *ed25519.MakePublicKey(&privKeyBytes)
-	return PubKeyEd25519(pubBytes)
-}
-
-// Equals - you probably don't need to use this.
-// Runs in constant time based on length of the keys.
-func (privKey PrivKeyEd25519) Equals(other PrivKey) bool {
-	if otherEd, ok := other.(PrivKeyEd25519); ok {
-		return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1
-	} else {
-		return false
-	}
-}
-
-func (privKey PrivKeyEd25519) ToCurve25519() *[32]byte {
-	keyCurve25519 := new([32]byte)
-	privKeyBytes := [64]byte(privKey)
-	extra25519.PrivateKeyToCurve25519(keyCurve25519, &privKeyBytes)
-	return keyCurve25519
-}
-
-// Deterministically generates new priv-key bytes from key.
-func (privKey PrivKeyEd25519) Generate(index int) PrivKeyEd25519 {
-	bz, err := cdc.MarshalBinaryBare(struct {
-		PrivKey [64]byte
-		Index   int
-	}{privKey, index})
-	if err != nil {
-		panic(err)
-	}
-	newBytes := Sha256(bz)
-	newKey := new([64]byte)
-	copy(newKey[:32], newBytes)
-	ed25519.MakePublicKey(newKey)
-	return PrivKeyEd25519(*newKey)
-}
-
-func GenPrivKeyEd25519() PrivKeyEd25519 {
-	privKeyBytes := new([64]byte)
-	copy(privKeyBytes[:32], CRandBytes(32))
-	ed25519.MakePublicKey(privKeyBytes)
-	return PrivKeyEd25519(*privKeyBytes)
-}
-
-// NOTE: secret should be the output of a KDF like bcrypt,
-// if it's derived from user input.
-func GenPrivKeyEd25519FromSecret(secret []byte) PrivKeyEd25519 {
-	privKey32 := Sha256(secret) // Not Ripemd160 because we want 32 bytes.
-	privKeyBytes := new([64]byte)
-	copy(privKeyBytes[:32], privKey32)
-	ed25519.MakePublicKey(privKeyBytes)
-	return PrivKeyEd25519(*privKeyBytes)
-}
-
-//-------------------------------------
-
-var _ PrivKey = PrivKeySecp256k1{}
-
-// Implements PrivKey
-type PrivKeySecp256k1 [32]byte
-
-func (privKey PrivKeySecp256k1) Bytes() []byte {
-	return cdc.MustMarshalBinaryBare(privKey)
-}
-
-func (privKey PrivKeySecp256k1) Sign(msg []byte) (Signature, error) {
-	priv__, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
-	sig__, err := priv__.Sign(Sha256(msg))
-	if err != nil {
-		return nil, err
-	}
-	return SignatureSecp256k1(sig__.Serialize()), nil
-}
-
-func (privKey PrivKeySecp256k1) PubKey() PubKey {
-	_, pub__ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
-	var pub PubKeySecp256k1
-	copy(pub[:], pub__.SerializeCompressed())
-	return pub
-}
-
-// Equals - you probably don't need to use this.
-// Runs in constant time based on length of the keys.
-func (privKey PrivKeySecp256k1) Equals(other PrivKey) bool {
-	if otherSecp, ok := other.(PrivKeySecp256k1); ok {
-		return subtle.ConstantTimeCompare(privKey[:], otherSecp[:]) == 1
-	} else {
-		return false
-	}
-}
-
-/*
-// Deterministically generates new priv-key bytes from key.
-func (key PrivKeySecp256k1) Generate(index int) PrivKeySecp256k1 {
-	newBytes := cdc.BinarySha256(struct {
-		PrivKey [64]byte
-		Index   int
-	}{key, index})
-	var newKey [64]byte
-	copy(newKey[:], newBytes)
-	return PrivKeySecp256k1(newKey)
-}
-*/
-
-func GenPrivKeySecp256k1() PrivKeySecp256k1 {
-	privKeyBytes := [32]byte{}
-	copy(privKeyBytes[:], CRandBytes(32))
-	priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKeyBytes[:])
-	copy(privKeyBytes[:], priv.Serialize())
-	return PrivKeySecp256k1(privKeyBytes)
-}
-
-// NOTE: secret should be the output of a KDF like bcrypt,
-// if it's derived from user input.
-func GenPrivKeySecp256k1FromSecret(secret []byte) PrivKeySecp256k1 {
-	privKey32 := Sha256(secret) // Not Ripemd160 because we want 32 bytes.
-	priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey32)
-	privKeyBytes := [32]byte{}
-	copy(privKeyBytes[:], priv.Serialize())
-	return PrivKeySecp256k1(privKeyBytes)
-}

crypto/priv_key_test.go

@@ -1,60 +0,0 @@
-package crypto_test
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/tendermint/tendermint/crypto"
-)
-
-func TestGeneratePrivKey(t *testing.T) {
-	testPriv := crypto.GenPrivKeyEd25519()
-	testGenerate := testPriv.Generate(1)
-	signBytes := []byte("something to sign")
-	pub := testGenerate.PubKey()
-	sig, err := testGenerate.Sign(signBytes)
-	assert.NoError(t, err)
-	assert.True(t, pub.VerifyBytes(signBytes, sig))
-}
-
-/*
-
-type BadKey struct {
-	PrivKeyEd25519
-}
-
-func TestReadPrivKey(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	// garbage in, garbage out
-	garbage := []byte("hjgewugfbiewgofwgewr")
-	XXX This test wants to register BadKey globally to crypto,
-	but we don't want to support that.
-	_, err := PrivKeyFromBytes(garbage)
-	require.Error(err)
-
-	edKey := GenPrivKeyEd25519()
-	badKey := BadKey{edKey}
-
-	cases := []struct {
-		key   PrivKey
-		valid bool
-	}{
-		{edKey, true},
-		{badKey, false},
-	}
-
-	for i, tc := range cases {
-		data := tc.key.Bytes()
-		fmt.Println(">>>", data)
-		key, err := PrivKeyFromBytes(data)
-		fmt.Printf("!!! %#v\n", key, err)
-		if tc.valid {
-			assert.NoError(err, "%d", i)
-			assert.Equal(tc.key, key, "%d", i)
-		} else {
-			assert.Error(err, "%d: %#v", i, key)
-		}
-	}
-}
-*/

crypto/pub_key.go

@@ -1,153 +0,0 @@
-package crypto
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"fmt"
-
-	"golang.org/x/crypto/ripemd160"
-
-	secp256k1 "github.com/btcsuite/btcd/btcec"
-
-	"github.com/tendermint/ed25519"
-	"github.com/tendermint/ed25519/extra25519"
-
-	cmn "github.com/tendermint/tendermint/libs/common"
-
-	"github.com/tendermint/tendermint/crypto/tmhash"
-)
-
-// An address is a []byte, but hex-encoded even in JSON.
-// []byte leaves us the option to change the address length.
-// Use an alias so Unmarshal methods (with ptr receivers) are available too.
-type Address = cmn.HexBytes
-
-func PubKeyFromBytes(pubKeyBytes []byte) (pubKey PubKey, err error) {
-	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
-	return
-}
-
-//----------------------------------------
-
-type PubKey interface {
-	Address() Address
-	Bytes() []byte
-	VerifyBytes(msg []byte, sig Signature) bool
-	Equals(PubKey) bool
-}
-
-//-------------------------------------
-
-var _ PubKey = PubKeyEd25519{}
-
-const PubKeyEd25519Size = 32
-
-// Implements PubKeyInner
-type PubKeyEd25519 [PubKeyEd25519Size]byte
-
-// Address is the SHA256-20 of the raw pubkey bytes.
-func (pubKey PubKeyEd25519) Address() Address {
-	return Address(tmhash.Sum(pubKey[:]))
-}
-
-func (pubKey PubKeyEd25519) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(pubKey)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (pubKey PubKeyEd25519) VerifyBytes(msg []byte, sig_ Signature) bool {
-	// make sure we use the same algorithm to sign
-	sig, ok := sig_.(SignatureEd25519)
-	if !ok {
-		return false
-	}
-	pubKeyBytes := [PubKeyEd25519Size]byte(pubKey)
-	sigBytes := [SignatureEd25519Size]byte(sig)
-	return ed25519.Verify(&pubKeyBytes, msg, &sigBytes)
-}
-
-// For use with golang/crypto/nacl/box
-// If error, returns nil.
-func (pubKey PubKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte {
-	keyCurve25519, pubKeyBytes := new([PubKeyEd25519Size]byte), [PubKeyEd25519Size]byte(pubKey)
-	ok := extra25519.PublicKeyToCurve25519(keyCurve25519, &pubKeyBytes)
-	if !ok {
-		return nil
-	}
-	return keyCurve25519
-}
-
-func (pubKey PubKeyEd25519) String() string {
-	return fmt.Sprintf("PubKeyEd25519{%X}", pubKey[:])
-}
-
-func (pubKey PubKeyEd25519) Equals(other PubKey) bool {
-	if otherEd, ok := other.(PubKeyEd25519); ok {
-		return bytes.Equal(pubKey[:], otherEd[:])
-	} else {
-		return false
-	}
-}
-
-//-------------------------------------
-
-var _ PubKey = PubKeySecp256k1{}
-
-const PubKeySecp256k1Size = 33
-
-// Implements PubKey.
-// Compressed pubkey (just the x-cord),
-// prefixed with 0x02 or 0x03, depending on the y-cord.
-type PubKeySecp256k1 [PubKeySecp256k1Size]byte
-
-// Implements Bitcoin style addresses: RIPEMD160(SHA256(pubkey))
-func (pubKey PubKeySecp256k1) Address() Address {
-	hasherSHA256 := sha256.New()
-	hasherSHA256.Write(pubKey[:]) // does not error
-	sha := hasherSHA256.Sum(nil)
-
-	hasherRIPEMD160 := ripemd160.New()
-	hasherRIPEMD160.Write(sha) // does not error
-	return Address(hasherRIPEMD160.Sum(nil))
-}
-
-func (pubKey PubKeySecp256k1) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(pubKey)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sig_ Signature) bool {
-	// and assert same algorithm to sign and verify
-	sig, ok := sig_.(SignatureSecp256k1)
-	if !ok {
-		return false
-	}
-
-	pub__, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256())
-	if err != nil {
-		return false
-	}
-	sig__, err := secp256k1.ParseDERSignature(sig[:], secp256k1.S256())
-	if err != nil {
-		return false
-	}
-	return sig__.Verify(Sha256(msg), pub__)
-}
-
-func (pubKey PubKeySecp256k1) String() string {
-	return fmt.Sprintf("PubKeySecp256k1{%X}", pubKey[:])
-}
-
-func (pubKey PubKeySecp256k1) Equals(other PubKey) bool {
-	if otherSecp, ok := other.(PubKeySecp256k1); ok {
-		return bytes.Equal(pubKey[:], otherSecp[:])
-	} else {
-		return false
-	}
-}

crypto/pub_key_test.go

@@ -1,50 +0,0 @@
-package crypto
-
-import (
-	"encoding/hex"
-	"testing"
-
-	"github.com/btcsuite/btcutil/base58"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type keyData struct {
-	priv string
-	pub  string
-	addr string
-}
-
-var secpDataTable = []keyData{
-	{
-		priv: "a96e62ed3955e65be32703f12d87b6b5cf26039ecfa948dc5107a495418e5330",
-		pub:  "02950e1cdfcb133d6024109fd489f734eeb4502418e538c28481f22bce276f248c",
-		addr: "1CKZ9Nx4zgds8tU7nJHotKSDr4a9bYJCa3",
-	},
-}
-
-func TestPubKeySecp256k1Address(t *testing.T) {
-	for _, d := range secpDataTable {
-		privB, _ := hex.DecodeString(d.priv)
-		pubB, _ := hex.DecodeString(d.pub)
-		addrBbz, _, _ := base58.CheckDecode(d.addr)
-		addrB := Address(addrBbz)
-
-		var priv PrivKeySecp256k1
-		copy(priv[:], privB)
-
-		pubKey := priv.PubKey()
-		pubT, _ := pubKey.(PubKeySecp256k1)
-		pub := pubT[:]
-		addr := pubKey.Address()
-
-		assert.Equal(t, pub, pubB, "Expected pub keys to match")
-		assert.Equal(t, addr, addrB, "Expected addresses to match")
-	}
-}
-
-func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) {
-	pk, err := PubKeyFromBytes([]byte("foo"))
-	require.NotNil(t, err, "expecting a non-nil error")
-	require.Nil(t, pk, "expecting an empty public key on error")
-}

crypto/secp256k1/bench_test.go

@@ -0,0 +1,26 @@
+package secp256k1
+
+import (
+	"io"
+	"testing"
+
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/internal/benchmarking"
+)
+
+func BenchmarkKeyGeneration(b *testing.B) {
+	benchmarkKeygenWrapper := func(reader io.Reader) crypto.PrivKey {
+		return genPrivKey(reader)
+	}
+	benchmarking.BenchmarkKeyGeneration(b, benchmarkKeygenWrapper)
+}
+
+func BenchmarkSigning(b *testing.B) {
+	priv := GenPrivKey()
+	benchmarking.BenchmarkSigning(b, priv)
+}
+
+func BenchmarkVerification(b *testing.B) {
+	priv := GenPrivKey()
+	benchmarking.BenchmarkVerification(b, priv)
+}

crypto/secp256k1/secp256k1.go

@@ -0,0 +1,160 @@
+package secp256k1
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/subtle"
+	"fmt"
+	"io"
+
+	secp256k1 "github.com/btcsuite/btcd/btcec"
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+	"golang.org/x/crypto/ripemd160"
+)
+
+//-------------------------------------
+const (
+	Secp256k1PrivKeyAminoRoute = "tendermint/PrivKeySecp256k1"
+	Secp256k1PubKeyAminoRoute  = "tendermint/PubKeySecp256k1"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
+	cdc.RegisterConcrete(PubKeySecp256k1{},
+		Secp256k1PubKeyAminoRoute, nil)
+
+	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
+	cdc.RegisterConcrete(PrivKeySecp256k1{},
+		Secp256k1PrivKeyAminoRoute, nil)
+}
+
+//-------------------------------------
+
+var _ crypto.PrivKey = PrivKeySecp256k1{}
+
+// PrivKeySecp256k1 implements PrivKey.
+type PrivKeySecp256k1 [32]byte
+
+// Bytes marshalls the private key using amino encoding.
+func (privKey PrivKeySecp256k1) Bytes() []byte {
+	return cdc.MustMarshalBinaryBare(privKey)
+}
+
+// Sign creates an ECDSA signature on curve Secp256k1, using SHA256 on the msg.
+func (privKey PrivKeySecp256k1) Sign(msg []byte) ([]byte, error) {
+	priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
+	sig, err := priv.Sign(crypto.Sha256(msg))
+	if err != nil {
+		return nil, err
+	}
+	return sig.Serialize(), nil
+}
+
+// PubKey performs the point-scalar multiplication from the privKey on the
+// generator point to get the pubkey.
+func (privKey PrivKeySecp256k1) PubKey() crypto.PubKey {
+	_, pubkeyObject := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
+	var pubkeyBytes PubKeySecp256k1
+	copy(pubkeyBytes[:], pubkeyObject.SerializeCompressed())
+	return pubkeyBytes
+}
+
+// Equals - you probably don't need to use this.
+// Runs in constant time based on length of the keys.
+func (privKey PrivKeySecp256k1) Equals(other crypto.PrivKey) bool {
+	if otherSecp, ok := other.(PrivKeySecp256k1); ok {
+		return subtle.ConstantTimeCompare(privKey[:], otherSecp[:]) == 1
+	}
+	return false
+}
+
+// GenPrivKey generates a new ECDSA private key on curve secp256k1 private key.
+// It uses OS randomness in conjunction with the current global random seed
+// in tendermint/libs/common to generate the private key.
+func GenPrivKey() PrivKeySecp256k1 {
+	return genPrivKey(crypto.CReader())
+}
+
+// genPrivKey generates a new secp256k1 private key using the provided reader.
+func genPrivKey(rand io.Reader) PrivKeySecp256k1 {
+	privKeyBytes := [32]byte{}
+	_, err := io.ReadFull(rand, privKeyBytes[:])
+	if err != nil {
+		panic(err)
+	}
+	// crypto.CRandBytes is guaranteed to be 32 bytes long, so it can be
+	// casted to PrivKeySecp256k1.
+	return PrivKeySecp256k1(privKeyBytes)
+}
+
+// GenPrivKeySecp256k1 hashes the secret with SHA2, and uses
+// that 32 byte output to create the private key.
+// NOTE: secret should be the output of a KDF like bcrypt,
+// if it's derived from user input.
+func GenPrivKeySecp256k1(secret []byte) PrivKeySecp256k1 {
+	privKey32 := sha256.Sum256(secret)
+	// sha256.Sum256() is guaranteed to be 32 bytes long, so it can be
+	// casted to PrivKeySecp256k1.
+	return PrivKeySecp256k1(privKey32)
+}
+
+//-------------------------------------
+
+var _ crypto.PubKey = PubKeySecp256k1{}
+
+// PubKeySecp256k1Size is comprised of 32 bytes for one field element
+// (the x-coordinate), plus one byte for the parity of the y-coordinate.
+const PubKeySecp256k1Size = 33
+
+// PubKeySecp256k1 implements crypto.PubKey.
+// It is the compressed form of the pubkey. The first byte depends is a 0x02 byte
+// if the y-coordinate is the lexicographically largest of the two associated with
+// the x-coordinate. Otherwise the first byte is a 0x03.
+// This prefix is followed with the x-coordinate.
+type PubKeySecp256k1 [PubKeySecp256k1Size]byte
+
+// Address returns a Bitcoin style addresses: RIPEMD160(SHA256(pubkey))
+func (pubKey PubKeySecp256k1) Address() crypto.Address {
+	hasherSHA256 := sha256.New()
+	hasherSHA256.Write(pubKey[:]) // does not error
+	sha := hasherSHA256.Sum(nil)
+
+	hasherRIPEMD160 := ripemd160.New()
+	hasherRIPEMD160.Write(sha) // does not error
+	return crypto.Address(hasherRIPEMD160.Sum(nil))
+}
+
+// Bytes returns the pubkey marshalled with amino encoding.
+func (pubKey PubKeySecp256k1) Bytes() []byte {
+	bz, err := cdc.MarshalBinaryBare(pubKey)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sig []byte) bool {
+	pub, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256())
+	if err != nil {
+		return false
+	}
+	parsedSig, err := secp256k1.ParseDERSignature(sig[:], secp256k1.S256())
+	if err != nil {
+		return false
+	}
+	return parsedSig.Verify(crypto.Sha256(msg), pub)
+}
+
+func (pubKey PubKeySecp256k1) String() string {
+	return fmt.Sprintf("PubKeySecp256k1{%X}", pubKey[:])
+}
+
+func (pubKey PubKeySecp256k1) Equals(other crypto.PubKey) bool {
+	if otherSecp, ok := other.(PubKeySecp256k1); ok {
+		return bytes.Equal(pubKey[:], otherSecp[:])
+	}
+	return false
+}

crypto/secp256k1/secpk256k1_test.go

@@ -0,0 +1,86 @@
+package secp256k1_test
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/btcsuite/btcutil/base58"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
+
+	underlyingSecp256k1 "github.com/btcsuite/btcd/btcec"
+)
+
+type keyData struct {
+	priv string
+	pub  string
+	addr string
+}
+
+var secpDataTable = []keyData{
+	{
+		priv: "a96e62ed3955e65be32703f12d87b6b5cf26039ecfa948dc5107a495418e5330",
+		pub:  "02950e1cdfcb133d6024109fd489f734eeb4502418e538c28481f22bce276f248c",
+		addr: "1CKZ9Nx4zgds8tU7nJHotKSDr4a9bYJCa3",
+	},
+}
+
+func TestPubKeySecp256k1Address(t *testing.T) {
+	for _, d := range secpDataTable {
+		privB, _ := hex.DecodeString(d.priv)
+		pubB, _ := hex.DecodeString(d.pub)
+		addrBbz, _, _ := base58.CheckDecode(d.addr)
+		addrB := crypto.Address(addrBbz)
+
+		var priv secp256k1.PrivKeySecp256k1
+		copy(priv[:], privB)
+
+		pubKey := priv.PubKey()
+		pubT, _ := pubKey.(secp256k1.PubKeySecp256k1)
+		pub := pubT[:]
+		addr := pubKey.Address()
+
+		assert.Equal(t, pub, pubB, "Expected pub keys to match")
+		assert.Equal(t, addr, addrB, "Expected addresses to match")
+	}
+}
+
+func TestSignAndValidateSecp256k1(t *testing.T) {
+	privKey := secp256k1.GenPrivKey()
+	pubKey := privKey.PubKey()
+
+	msg := crypto.CRandBytes(128)
+	sig, err := privKey.Sign(msg)
+	require.Nil(t, err)
+
+	assert.True(t, pubKey.VerifyBytes(msg, sig))
+
+	// Mutate the signature, just one bit.
+	sig[3] ^= byte(0x01)
+
+	assert.False(t, pubKey.VerifyBytes(msg, sig))
+}
+
+// This test is intended to justify the removal of calls to the underlying library
+// in creating the privkey.
+func TestSecp256k1LoadPrivkeyAndSerializeIsIdentity(t *testing.T) {
+	numberOfTests := 256
+	for i := 0; i < numberOfTests; i++ {
+		// Seed the test case with some random bytes
+		privKeyBytes := [32]byte{}
+		copy(privKeyBytes[:], crypto.CRandBytes(32))
+
+		// This function creates a private and public key in the underlying libraries format.
+		// The private key is basically calling new(big.Int).SetBytes(pk), which removes leading zero bytes
+		priv, _ := underlyingSecp256k1.PrivKeyFromBytes(underlyingSecp256k1.S256(), privKeyBytes[:])
+		// this takes the bytes returned by `(big int).Bytes()`, and if the length is less than 32 bytes,
+		// pads the bytes from the left with zero bytes. Therefore these two functions composed
+		// result in the identity function on privKeyBytes, hence the following equality check
+		// always returning true.
+		serializedBytes := priv.Serialize()
+		require.Equal(t, privKeyBytes[:], serializedBytes)
+	}
+}

crypto/signature.go

@@ -1,90 +0,0 @@
-package crypto
-
-import (
-	"fmt"
-
-	"crypto/subtle"
-
-	. "github.com/tendermint/tendermint/libs/common"
-)
-
-func SignatureFromBytes(pubKeyBytes []byte) (pubKey Signature, err error) {
-	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
-	return
-}
-
-//----------------------------------------
-
-type Signature interface {
-	Bytes() []byte
-	IsZero() bool
-	Equals(Signature) bool
-}
-
-//-------------------------------------
-
-var _ Signature = SignatureEd25519{}
-
-const SignatureEd25519Size = 64
-
-// Implements Signature
-type SignatureEd25519 [SignatureEd25519Size]byte
-
-func (sig SignatureEd25519) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(sig)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (sig SignatureEd25519) IsZero() bool { return len(sig) == 0 }
-
-func (sig SignatureEd25519) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }
-
-func (sig SignatureEd25519) Equals(other Signature) bool {
-	if otherEd, ok := other.(SignatureEd25519); ok {
-		return subtle.ConstantTimeCompare(sig[:], otherEd[:]) == 1
-	} else {
-		return false
-	}
-}
-
-func SignatureEd25519FromBytes(data []byte) Signature {
-	var sig SignatureEd25519
-	copy(sig[:], data)
-	return sig
-}
-
-//-------------------------------------
-
-var _ Signature = SignatureSecp256k1{}
-
-// Implements Signature
-type SignatureSecp256k1 []byte
-
-func (sig SignatureSecp256k1) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(sig)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (sig SignatureSecp256k1) IsZero() bool { return len(sig) == 0 }
-
-func (sig SignatureSecp256k1) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }
-
-func (sig SignatureSecp256k1) Equals(other Signature) bool {
-	if otherSecp, ok := other.(SignatureSecp256k1); ok {
-		return subtle.ConstantTimeCompare(sig[:], otherSecp[:]) == 1
-	} else {
-		return false
-	}
-}
-
-func SignatureSecp256k1FromBytes(data []byte) Signature {
-	sig := make(SignatureSecp256k1, len(data))
-	copy(sig[:], data)
-	return sig
-}

crypto/signature_test.go

@@ -1,46 +0,0 @@
-package crypto
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSignAndValidateEd25519(t *testing.T) {
-
-	privKey := GenPrivKeyEd25519()
-	pubKey := privKey.PubKey()
-
-	msg := CRandBytes(128)
-	sig, err := privKey.Sign(msg)
-	require.Nil(t, err)
-
-	// Test the signature
-	assert.True(t, pubKey.VerifyBytes(msg, sig))
-
-	// Mutate the signature, just one bit.
-	sigEd := sig.(SignatureEd25519)
-	sigEd[7] ^= byte(0x01)
-	sig = sigEd
-
-	assert.False(t, pubKey.VerifyBytes(msg, sig))
-}
-
-func TestSignAndValidateSecp256k1(t *testing.T) {
-	privKey := GenPrivKeySecp256k1()
-	pubKey := privKey.PubKey()
-
-	msg := CRandBytes(128)
-	sig, err := privKey.Sign(msg)
-	require.Nil(t, err)
-
-	assert.True(t, pubKey.VerifyBytes(msg, sig))
-
-	// Mutate the signature, just one bit.
-	sigEd := sig.(SignatureSecp256k1)
-	sigEd[3] ^= byte(0x01)
-	sig = sigEd
-
-	assert.False(t, pubKey.VerifyBytes(msg, sig))
-}

crypto/xchacha20poly1305/vector_test.go

@@ -0,0 +1,103 @@
+package xchacha20poly1305
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+)
+
+func toHex(bits []byte) string {
+	return hex.EncodeToString(bits)
+}
+
+func fromHex(bits string) []byte {
+	b, err := hex.DecodeString(bits)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}
+
+func TestHChaCha20(t *testing.T) {
+	for i, v := range hChaCha20Vectors {
+		var key [32]byte
+		var nonce [16]byte
+		copy(key[:], v.key)
+		copy(nonce[:], v.nonce)
+
+		HChaCha20(&key, &nonce, &key)
+		if !bytes.Equal(key[:], v.keystream) {
+			t.Errorf("Test %d: keystream mismatch:\n \t got:  %s\n \t want: %s", i, toHex(key[:]), toHex(v.keystream))
+		}
+	}
+}
+
+var hChaCha20Vectors = []struct {
+	key, nonce, keystream []byte
+}{
+	{
+		fromHex("0000000000000000000000000000000000000000000000000000000000000000"),
+		fromHex("000000000000000000000000000000000000000000000000"),
+		fromHex("1140704c328d1d5d0e30086cdf209dbd6a43b8f41518a11cc387b669b2ee6586"),
+	},
+	{
+		fromHex("8000000000000000000000000000000000000000000000000000000000000000"),
+		fromHex("000000000000000000000000000000000000000000000000"),
+		fromHex("7d266a7fd808cae4c02a0a70dcbfbcc250dae65ce3eae7fc210f54cc8f77df86"),
+	},
+	{
+		fromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+		fromHex("000000000000000000000000000000000000000000000002"),
+		fromHex("e0c77ff931bb9163a5460c02ac281c2b53d792b1c43fea817e9ad275ae546963"),
+	},
+	{
+		fromHex("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"),
+		fromHex("000102030405060708090a0b0c0d0e0f1011121314151617"),
+		fromHex("51e3ff45a895675c4b33b46c64f4a9ace110d34df6a2ceab486372bacbd3eff6"),
+	},
+	{
+		fromHex("24f11cce8a1b3d61e441561a696c1c1b7e173d084fd4812425435a8896a013dc"),
+		fromHex("d9660c5900ae19ddad28d6e06e45fe5e"),
+		fromHex("5966b3eec3bff1189f831f06afe4d4e3be97fa9235ec8c20d08acfbbb4e851e3"),
+	},
+}
+
+func TestVectors(t *testing.T) {
+	for i, v := range vectors {
+		if len(v.plaintext) == 0 {
+			v.plaintext = make([]byte, len(v.ciphertext))
+		}
+
+		var nonce [24]byte
+		copy(nonce[:], v.nonce)
+
+		aead, err := New(v.key)
+		if err != nil {
+			t.Error(err)
+		}
+
+		dst := aead.Seal(nil, nonce[:], v.plaintext, v.ad)
+		if !bytes.Equal(dst, v.ciphertext) {
+			t.Errorf("Test %d: ciphertext mismatch:\n \t got:  %s\n \t want: %s", i, toHex(dst), toHex(v.ciphertext))
+		}
+		open, err := aead.Open(nil, nonce[:], dst, v.ad)
+		if err != nil {
+			t.Error(err)
+		}
+		if !bytes.Equal(open, v.plaintext) {
+			t.Errorf("Test %d: plaintext mismatch:\n \t got:  %s\n \t want: %s", i, string(open), string(v.plaintext))
+		}
+	}
+}
+
+var vectors = []struct {
+	key, nonce, ad, plaintext, ciphertext []byte
+}{
+	{
+		[]byte{0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f},
+		[]byte{0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b},
+		[]byte{0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7},
+		[]byte("Ladies and Gentlemen of the class of '99: If I could offer you only one tip for the future, sunscreen would be it."),
+		[]byte{0x45, 0x3c, 0x06, 0x93, 0xa7, 0x40, 0x7f, 0x04, 0xff, 0x4c, 0x56, 0xae, 0xdb, 0x17, 0xa3, 0xc0, 0xa1, 0xaf, 0xff, 0x01, 0x17, 0x49, 0x30, 0xfc, 0x22, 0x28, 0x7c, 0x33, 0xdb, 0xcf, 0x0a, 0xc8, 0xb8, 0x9a, 0xd9, 0x29, 0x53, 0x0a, 0x1b, 0xb3, 0xab, 0x5e, 0x69, 0xf2, 0x4c, 0x7f, 0x60, 0x70, 0xc8, 0xf8, 0x40, 0xc9, 0xab, 0xb4, 0xf6, 0x9f, 0xbf, 0xc8, 0xa7, 0xff, 0x51, 0x26, 0xfa, 0xee, 0xbb, 0xb5, 0x58, 0x05, 0xee, 0x9c, 0x1c, 0xf2, 0xce, 0x5a, 0x57, 0x26, 0x32, 0x87, 0xae, 0xc5, 0x78, 0x0f, 0x04, 0xec, 0x32, 0x4c, 0x35, 0x14, 0x12, 0x2c, 0xfc, 0x32, 0x31, 0xfc, 0x1a, 0x8b, 0x71, 0x8a, 0x62, 0x86, 0x37, 0x30, 0xa2, 0x70, 0x2b, 0xb7, 0x63, 0x66, 0x11, 0x6b, 0xed, 0x09, 0xe0, 0xfd, 0x5c, 0x6d, 0x84, 0xb6, 0xb0, 0xc1, 0xab, 0xaf, 0x24, 0x9d, 0x5d, 0xd0, 0xf7, 0xf5, 0xa7, 0xea},
+	},
+}

crypto/xchacha20poly1305/xchachapoly.go

@@ -0,0 +1,261 @@
+// Package xchacha20poly1305 creates an AEAD using hchacha, chacha, and poly1305
+// This allows for randomized nonces to be used in conjunction with chacha.
+package xchacha20poly1305
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+// Implements crypto.AEAD
+type xchacha20poly1305 struct {
+	key [KeySize]byte
+}
+
+const (
+	// KeySize is the size of the key used by this AEAD, in bytes.
+	KeySize = 32
+	// NonceSize is the size of the nonce used with this AEAD, in bytes.
+	NonceSize = 24
+	// TagSize is the size added from poly1305
+	TagSize = 16
+	// MaxPlaintextSize is the max size that can be passed into a single call of Seal
+	MaxPlaintextSize = (1 << 38) - 64
+	// MaxCiphertextSize is the max size that can be passed into a single call of Open,
+	// this differs from plaintext size due to the tag
+	MaxCiphertextSize = (1 << 38) - 48
+
+	// sigma are constants used in xchacha.
+	// Unrolled from a slice so that they can be inlined, as slices can't be constants.
+	sigma0 = uint32(0x61707865)
+	sigma1 = uint32(0x3320646e)
+	sigma2 = uint32(0x79622d32)
+	sigma3 = uint32(0x6b206574)
+)
+
+// New returns a new xchachapoly1305 AEAD
+func New(key []byte) (cipher.AEAD, error) {
+	if len(key) != KeySize {
+		return nil, errors.New("xchacha20poly1305: bad key length")
+	}
+	ret := new(xchacha20poly1305)
+	copy(ret.key[:], key)
+	return ret, nil
+}
+
+// nolint
+func (c *xchacha20poly1305) NonceSize() int {
+	return NonceSize
+}
+
+// nolint
+func (c *xchacha20poly1305) Overhead() int {
+	return TagSize
+}
+
+func (c *xchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if len(nonce) != NonceSize {
+		panic("xchacha20poly1305: bad nonce length passed to Seal")
+	}
+
+	if uint64(len(plaintext)) > MaxPlaintextSize {
+		panic("xchacha20poly1305: plaintext too large")
+	}
+
+	var subKey [KeySize]byte
+	var hNonce [16]byte
+	var subNonce [chacha20poly1305.NonceSize]byte
+	copy(hNonce[:], nonce[:16])
+
+	HChaCha20(&subKey, &hNonce, &c.key)
+
+	// This can't error because we always provide a correctly sized key
+	chacha20poly1305, _ := chacha20poly1305.New(subKey[:])
+
+	copy(subNonce[4:], nonce[16:])
+
+	return chacha20poly1305.Seal(dst, subNonce[:], plaintext, additionalData)
+}
+
+func (c *xchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	if len(nonce) != NonceSize {
+		return nil, fmt.Errorf("xchacha20poly1305: bad nonce length passed to Open")
+	}
+	if uint64(len(ciphertext)) > MaxCiphertextSize {
+		return nil, fmt.Errorf("xchacha20poly1305: ciphertext too large")
+	}
+	var subKey [KeySize]byte
+	var hNonce [16]byte
+	var subNonce [chacha20poly1305.NonceSize]byte
+	copy(hNonce[:], nonce[:16])
+
+	HChaCha20(&subKey, &hNonce, &c.key)
+
+	// This can't error because we always provide a correctly sized key
+	chacha20poly1305, _ := chacha20poly1305.New(subKey[:])
+
+	copy(subNonce[4:], nonce[16:])
+
+	return chacha20poly1305.Open(dst, subNonce[:], ciphertext, additionalData)
+}
+
+// HChaCha exported from
+// https://github.com/aead/chacha20/blob/8b13a72661dae6e9e5dea04f344f0dc95ea29547/chacha/chacha_generic.go#L194
+// TODO: Add support for the different assembly instructions used there.
+
+// The MIT License (MIT)
+
+// Copyright (c) 2016 Andreas Auernhammer
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// HChaCha20 generates 32 pseudo-random bytes from a 128 bit nonce and a 256 bit secret key.
+// It can be used as a key-derivation-function (KDF).
+func HChaCha20(out *[32]byte, nonce *[16]byte, key *[32]byte) { hChaCha20Generic(out, nonce, key) }
+
+func hChaCha20Generic(out *[32]byte, nonce *[16]byte, key *[32]byte) {
+	v00 := sigma0
+	v01 := sigma1
+	v02 := sigma2
+	v03 := sigma3
+	v04 := binary.LittleEndian.Uint32(key[0:])
+	v05 := binary.LittleEndian.Uint32(key[4:])
+	v06 := binary.LittleEndian.Uint32(key[8:])
+	v07 := binary.LittleEndian.Uint32(key[12:])
+	v08 := binary.LittleEndian.Uint32(key[16:])
+	v09 := binary.LittleEndian.Uint32(key[20:])
+	v10 := binary.LittleEndian.Uint32(key[24:])
+	v11 := binary.LittleEndian.Uint32(key[28:])
+	v12 := binary.LittleEndian.Uint32(nonce[0:])
+	v13 := binary.LittleEndian.Uint32(nonce[4:])
+	v14 := binary.LittleEndian.Uint32(nonce[8:])
+	v15 := binary.LittleEndian.Uint32(nonce[12:])
+
+	for i := 0; i < 20; i += 2 {
+		v00 += v04
+		v12 ^= v00
+		v12 = (v12 << 16) | (v12 >> 16)
+		v08 += v12
+		v04 ^= v08
+		v04 = (v04 << 12) | (v04 >> 20)
+		v00 += v04
+		v12 ^= v00
+		v12 = (v12 << 8) | (v12 >> 24)
+		v08 += v12
+		v04 ^= v08
+		v04 = (v04 << 7) | (v04 >> 25)
+		v01 += v05
+		v13 ^= v01
+		v13 = (v13 << 16) | (v13 >> 16)
+		v09 += v13
+		v05 ^= v09
+		v05 = (v05 << 12) | (v05 >> 20)
+		v01 += v05
+		v13 ^= v01
+		v13 = (v13 << 8) | (v13 >> 24)
+		v09 += v13
+		v05 ^= v09
+		v05 = (v05 << 7) | (v05 >> 25)
+		v02 += v06
+		v14 ^= v02
+		v14 = (v14 << 16) | (v14 >> 16)
+		v10 += v14
+		v06 ^= v10
+		v06 = (v06 << 12) | (v06 >> 20)
+		v02 += v06
+		v14 ^= v02
+		v14 = (v14 << 8) | (v14 >> 24)
+		v10 += v14
+		v06 ^= v10
+		v06 = (v06 << 7) | (v06 >> 25)
+		v03 += v07
+		v15 ^= v03
+		v15 = (v15 << 16) | (v15 >> 16)
+		v11 += v15
+		v07 ^= v11
+		v07 = (v07 << 12) | (v07 >> 20)
+		v03 += v07
+		v15 ^= v03
+		v15 = (v15 << 8) | (v15 >> 24)
+		v11 += v15
+		v07 ^= v11
+		v07 = (v07 << 7) | (v07 >> 25)
+		v00 += v05
+		v15 ^= v00
+		v15 = (v15 << 16) | (v15 >> 16)
+		v10 += v15
+		v05 ^= v10
+		v05 = (v05 << 12) | (v05 >> 20)
+		v00 += v05
+		v15 ^= v00
+		v15 = (v15 << 8) | (v15 >> 24)
+		v10 += v15
+		v05 ^= v10
+		v05 = (v05 << 7) | (v05 >> 25)
+		v01 += v06
+		v12 ^= v01
+		v12 = (v12 << 16) | (v12 >> 16)
+		v11 += v12
+		v06 ^= v11
+		v06 = (v06 << 12) | (v06 >> 20)
+		v01 += v06
+		v12 ^= v01
+		v12 = (v12 << 8) | (v12 >> 24)
+		v11 += v12
+		v06 ^= v11
+		v06 = (v06 << 7) | (v06 >> 25)
+		v02 += v07
+		v13 ^= v02
+		v13 = (v13 << 16) | (v13 >> 16)
+		v08 += v13
+		v07 ^= v08
+		v07 = (v07 << 12) | (v07 >> 20)
+		v02 += v07
+		v13 ^= v02
+		v13 = (v13 << 8) | (v13 >> 24)
+		v08 += v13
+		v07 ^= v08
+		v07 = (v07 << 7) | (v07 >> 25)
+		v03 += v04
+		v14 ^= v03
+		v14 = (v14 << 16) | (v14 >> 16)
+		v09 += v14
+		v04 ^= v09
+		v04 = (v04 << 12) | (v04 >> 20)
+		v03 += v04
+		v14 ^= v03
+		v14 = (v14 << 8) | (v14 >> 24)
+		v09 += v14
+		v04 ^= v09
+		v04 = (v04 << 7) | (v04 >> 25)
+	}
+
+	binary.LittleEndian.PutUint32(out[0:], v00)
+	binary.LittleEndian.PutUint32(out[4:], v01)
+	binary.LittleEndian.PutUint32(out[8:], v02)
+	binary.LittleEndian.PutUint32(out[12:], v03)
+	binary.LittleEndian.PutUint32(out[16:], v12)
+	binary.LittleEndian.PutUint32(out[20:], v13)
+	binary.LittleEndian.PutUint32(out[24:], v14)
+	binary.LittleEndian.PutUint32(out[28:], v15)
+}

crypto/xchacha20poly1305/xchachapoly_test.go

@@ -1,54 +1,12 @@
-package hkdfchacha20poly1305
+package xchacha20poly1305
 
 import (
 	"bytes"
 	cr "crypto/rand"
-	"encoding/hex"
 	mr "math/rand"
 	"testing"
-
-	"github.com/stretchr/testify/assert"
 )
 
-// Test that a test vector we generated is valid. (Ensures backwards
-// compatibility)
-func TestVector(t *testing.T) {
-	key, _ := hex.DecodeString("56f8de45d3c294c7675bcaf457bdd4b71c380b9b2408ce9412b348d0f08b69ee")
-	aead, err := New(key[:])
-	if err != nil {
-		t.Fatal(err)
-	}
-	cts := []string{"e20a8bf42c535ac30125cfc52031577f0b",
-		"657695b37ba30f67b25860d90a6f1d00d8",
-		"e9aa6f3b7f625d957fd50f05bcdf20d014",
-		"8a00b3b5a6014e0d2033bebc5935086245",
-		"aadd74867b923879e6866ea9e03c009039",
-		"fc59773c2c864ee3b4cc971876b3c7bed4",
-		"caec14e3a9a52ce1a2682c6737defa4752",
-		"0b89511ffe490d2049d6950494ee51f919",
-		"7de854ea71f43ca35167a07566c769083d",
-		"cd477327f4ea4765c71e311c5fec1edbfb"}
-
-	for i := 0; i < 10; i++ {
-		ct, _ := hex.DecodeString(cts[i])
-
-		byteArr := []byte{byte(i)}
-		nonce := make([]byte, 24)
-		nonce[0] = byteArr[0]
-
-		// Test that we get the expected plaintext on open
-		plaintext, err := aead.Open(nil, nonce, ct, byteArr)
-		if err != nil {
-			t.Errorf("%dth Open failed", i)
-			continue
-		}
-		assert.Equal(t, byteArr, plaintext)
-		// Test that sealing yields the expected ciphertext
-		ciphertext := aead.Seal(nil, nonce, plaintext, byteArr)
-		assert.Equal(t, ct, ciphertext)
-	}
-}
-
 // The following test is taken from
 // https://github.com/golang/crypto/blob/master/chacha20poly1305/chacha20poly1305_test.go#L69
 // It requires the below copyright notice, where "this source code" refers to the following function.

crypto/xsalsa20symmetric/symmetric.go

@@ -1,12 +1,15 @@
-package crypto
+package xsalsa20symmetric
 
 import (
 	"errors"
 
-	. "github.com/tendermint/tendermint/libs/common"
+	"github.com/tendermint/tendermint/crypto"
+	cmn "github.com/tendermint/tendermint/libs/common"
 	"golang.org/x/crypto/nacl/secretbox"
 )
 
+// TODO, make this into a struct that implements crypto.Symmetric.
+
 const nonceLen = 24
 const secretLen = 32
 
@@ -15,9 +18,9 @@ const secretLen = 32
 // NOTE: call crypto.MixEntropy() first.
 func EncryptSymmetric(plaintext []byte, secret []byte) (ciphertext []byte) {
 	if len(secret) != secretLen {
-		PanicSanity(Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
+		cmn.PanicSanity(cmn.Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
 	}
-	nonce := CRandBytes(nonceLen)
+	nonce := crypto.CRandBytes(nonceLen)
 	nonceArr := [nonceLen]byte{}
 	copy(nonceArr[:], nonce)
 	secretArr := [secretLen]byte{}
@@ -32,7 +35,7 @@ func EncryptSymmetric(plaintext []byte, secret []byte) (ciphertext []byte) {
 // The ciphertext is (secretbox.Overhead + 24) bytes longer than the plaintext.
 func DecryptSymmetric(ciphertext []byte, secret []byte) (plaintext []byte, err error) {
 	if len(secret) != secretLen {
-		PanicSanity(Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
+		cmn.PanicSanity(cmn.Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
 	}
 	if len(ciphertext) <= secretbox.Overhead+nonceLen {
 		return nil, errors.New("Ciphertext is too short")

crypto/xsalsa20symmetric/symmetric_test.go

@@ -1,4 +1,4 @@
-package crypto
+package xsalsa20symmetric
 
 import (
 	"testing"
@@ -6,12 +6,13 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/tendermint/tendermint/crypto"
 	"golang.org/x/crypto/bcrypt"
 )
 
 func TestSimple(t *testing.T) {
 
-	MixEntropy([]byte("someentropy"))
+	crypto.MixEntropy([]byte("someentropy"))
 
 	plaintext := []byte("sometext")
 	secret := []byte("somesecretoflengththirtytwo===32")
@@ -24,7 +25,7 @@ func TestSimple(t *testing.T) {
 
 func TestSimpleWithKDF(t *testing.T) {
 
-	MixEntropy([]byte("someentropy"))
+	crypto.MixEntropy([]byte("someentropy"))
 
 	plaintext := []byte("sometext")
 	secretPass := []byte("somesecret")
@@ -32,7 +33,7 @@ func TestSimpleWithKDF(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	secret = Sha256(secret)
+	secret = crypto.Sha256(secret)
 
 	ciphertext := EncryptSymmetric(plaintext, secret)
 	plaintext2, err := DecryptSymmetric(ciphertext, secret)

docs/DOCS_README.md

@@ -9,7 +9,9 @@ and built using [VuePress](https://vuepress.vuejs.org/) from the tendermint webs
 - https://github.com/tendermint/tendermint.com
 
 which has a [configuration file](https://github.com/tendermint/tendermint.com/blob/develop/docs/.vuepress/config.js) for displaying
-the Table of Contents that lists all the documentation.
+the Table of Contents that lists all the documentation. 
+
+Under the hood, Jenkins listens for changes in ./docs then pushes a `docs-staging` branch to the tendermint.com repo with the latest documentation. That branch must be manually PR'd to `develop` then `master` for staging then production. This process should happen in synchrony with a release.
 
 The `README.md` in this directory is the landing page for
 website documentation and the following folders are intentionally

docs/app-dev/abci-spec.md

@@ -108,8 +108,11 @@ See below for more details on the message types and how they are used.
 ### InitChain
 
 - **Request**:
-  - `Validators ([]Validator)`: Initial genesis validators
-  - `AppStateBytes ([]byte)`: Serialized initial application state
+  - `Time (google.protobuf.Timestamp)`: Genesis time.
+  - `ChainID (string)`: ID of the blockchain.
+  - `ConsensusParams (ConsensusParams)`: Initial consensus-critical parameters.
+  - `Validators ([]Validator)`: Initial genesis validators.
+  - `AppStateBytes ([]byte)`: Serialized initial application state. Amino-encoded JSON bytes.
 - **Response**:
   - `ConsensusParams (ConsensusParams)`: Initial
     consensus-critical parameters.
@@ -157,9 +160,8 @@ See below for more details on the message types and how they are used.
 - **Request**:
   - `Hash ([]byte)`: The block's hash. This can be derived from the
     block header.
-  - `Header (struct{})`: The block header
-  - `Validators ([]SigningValidator)`: List of validators in the current validator
-    set and whether or not they signed a vote in the LastCommit
+  - `Header (struct{})`: The block header.
+  - `LastCommitInfo (LastCommitInfo)`: Info about the last commit.
   - `ByzantineValidators ([]Evidence)`: List of evidence of
     validators that acted maliciously
 - **Response**:
@@ -168,8 +170,9 @@ See below for more details on the message types and how they are used.
   - Signals the beginning of a new block. Called prior to
     any DeliverTxs.
   - The header is expected to at least contain the Height.
-  - The `Validators` and `ByzantineValidators` can be used to
-    determine rewards and punishments for the validators.
+  - The `LastCommitInfo` and `ByzantineValidators` can be used to determine
+    rewards and punishments for the validators. NOTE validators here do not
+    include pubkeys.
 
 ### CheckTx
 
@@ -186,7 +189,6 @@ See below for more details on the message types and how they are used.
   - `GasUsed (int64)`: Amount of gas consumed by transaction.
   - `Tags ([]cmn.KVPair)`: Key-Value tags for filtering and indexing
     transactions (eg. by account).
-  - `Fee (cmn.KI64Pair)`: Fee paid for the transaction.
 - **Usage**: Validate a mempool transaction, prior to broadcasting
   or proposing. CheckTx should perform stateful but light-weight
   checks of the validity of the transaction (like checking signatures
@@ -223,7 +225,6 @@ See below for more details on the message types and how they are used.
   - `GasUsed (int64)`: Amount of gas consumed by transaction.
   - `Tags ([]cmn.KVPair)`: Key-Value tags for filtering and indexing
     transactions (eg. by account).
-  - `Fee (cmn.KI64Pair)`: Fee paid for the transaction.
 - **Usage**:
   - Deliver a transaction to be executed in full by the application.
     If the transaction is valid, returns CodeType.OK.
@@ -265,7 +266,8 @@ See below for more details on the message types and how they are used.
 - **Fields**:
   - `ChainID (string)`: ID of the blockchain
   - `Height (int64)`: Height of the block in the chain
-  - `Time (int64)`: Unix time of the block
+  - `Time (google.protobuf.Timestamp)`: Time of the block. It is the proposer's
+    local time when block was created.
   - `NumTxs (int32)`: Number of transactions in the block
   - `TotalTxs (int64)`: Total number of transactions in the blockchain until
     now
@@ -320,6 +322,14 @@ See below for more details on the message types and how they are used.
     "duplicate/vote".
   - `Validator (Validator`: The offending validator
   - `Height (int64)`: Height when the offense was committed
-  - `Time (int64)`: Unix time of the block at height `Height`
+  - `Time (google.protobuf.Timestamp)`: Time of the block at height `Height`.
+    It is the proposer's local time when block was created.
   - `TotalVotingPower (int64)`: Total voting power of the validator set at
     height `Height`
+
+### LastCommitInfo
+
+- **Fields**:
+  - `CommitRound (int32)`: Commit round.
+  - `Validators ([]SigningValidator)`: List of validators in the current
+    validator set and whether or not they signed a vote.

docs/app-dev/app-development.md

@@ -365,14 +365,14 @@ ResponseBeginBlock requestBeginBlock(RequestBeginBlock req) {
 
 ### EndBlock
 
-The EndBlock request can be used to run some code at the end of every
-block. Additionally, the response may contain a list of validators,
-which can be used to update the validator set. To add a new validator or
-update an existing one, simply include them in the list returned in the
-EndBlock response. To remove one, include it in the list with a `power`
-equal to `0`. Tendermint core will take care of updating the validator
-set. Note the change in voting power must be strictly less than 1/3 per
-block if you want a light client to be able to prove the transition
+The EndBlock request can be used to run some code at the end of every block.
+Additionally, the response may contain a list of validators, which can be used
+to update the validator set. To add a new validator or update an existing one,
+simply include them in the list returned in the EndBlock response. To remove
+one, include it in the list with a `power` equal to `0`. Validator's `address`
+field can be left empty. Tendermint core will take care of updating the
+validator set. Note the change in voting power must be strictly less than 1/3
+per block if you want a light client to be able to prove the transition
 externally. See the [light client
 docs](https://godoc.org/github.com/tendermint/tendermint/lite#hdr-How_We_Track_Validators)
 for details on how it tracks validators.

docs/app-dev/getting-started.md

@@ -93,9 +93,7 @@ like:
   "jsonrpc": "2.0",
   "id": "",
   "result": {
-    "check_tx": {
-      "fee": {}
-    },
+    "check_tx": {},
     "deliver_tx": {
       "tags": [
         {
@@ -106,8 +104,7 @@ like:
           "key": "YXBwLmtleQ==",
           "value": "YWJjZA=="
         }
-      ],
-      "fee": {}
+      ]
     },
     "hash": "9DF66553F98DE3C26E3C3317A3E4CED54F714E39",
     "height": 14
@@ -219,13 +216,10 @@ the number `1`. If instead, we try to send a `5`, we get an error:
   "jsonrpc": "2.0",
   "id": "",
   "result": {
-    "check_tx": {
-      "fee": {}
-    },
+    "check_tx": {},
     "deliver_tx": {
       "code": 2,
-      "log": "Invalid nonce. Expected 1, got 5",
-      "fee": {}
+      "log": "Invalid nonce. Expected 1, got 5"
     },
     "hash": "33B93DFF98749B0D6996A70F64071347060DC19C",
     "height": 34
@@ -241,12 +235,8 @@ But if we send a `1`, it works again:
   "jsonrpc": "2.0",
   "id": "",
   "result": {
-    "check_tx": {
-      "fee": {}
-    },
-    "deliver_tx": {
-      "fee": {}
-    },
+    "check_tx": {},
+    "deliver_tx": {},
     "hash": "F17854A977F6FA7EEA1BD758E296710B86F72F3D",
     "height": 60
   }

docs/architecture/adr-012-peer-transport.md

@@ -0,0 +1,113 @@
+# ADR 012: PeerTransport
+
+## Context
+
+One of the more apparent problems with the current architecture in the p2p
+package is that there is no clear separation of concerns between different
+components. Most notably the `Switch` is currently doing physical connection
+handling. An artifact is the dependency of the Switch on
+`[config.P2PConfig`](https://github.com/tendermint/tendermint/blob/05a76fb517f50da27b4bfcdc7b4cf185fc61eff6/config/config.go#L272-L339).
+
+Addresses:
+* [#2046](https://github.com/tendermint/tendermint/issues/2046)
+* [#2047](https://github.com/tendermint/tendermint/issues/2047)
+
+First iteraton in [#2067](https://github.com/tendermint/tendermint/issues/2067)
+
+## Decision
+
+Transport concerns will be handled by a new component (`PeerTransport`) which
+will provide Peers at its boundary to the caller. In turn `Switch` will use
+this new component accept new `Peer`s and dial them based on `NetAddress`.
+
+### PeerTransport
+
+Responsible for emitting and connecting to Peers. The implementation of `Peer`
+is left to the transport, which implies that the chosen transport dictates the
+characteristics of the implementation handed back to the `Switch`. Each
+transport implementation is responsible to filter establishing peers specific
+to its domain, for the default multiplexed implementation the following will
+apply:
+
+* connections from our own node
+* handshake fails
+* upgrade to secret connection fails
+* prevent duplicate ip
+* prevent duplicate id
+* nodeinfo incompatibility
+
+
+``` go
+// PeerTransport proxies incoming and outgoing peer connections.
+type PeerTransport interface {
+	// Accept returns a newly connected Peer.
+	Accept() (Peer, error)
+
+	// Dial connects to a Peer.
+	Dial(NetAddress) (Peer, error)
+}
+
+// EXAMPLE OF DEFAULT IMPLEMENTATION
+
+// multiplexTransport accepts tcp connections and upgrades to multiplexted
+// peers.
+type multiplexTransport struct {
+	listener net.Listener
+
+	acceptc chan accept
+	closec  <-chan struct{}
+	listenc <-chan struct{}
+
+	dialTimeout      time.Duration
+	handshakeTimeout time.Duration
+	nodeAddr         NetAddress
+	nodeInfo         NodeInfo
+	nodeKey          NodeKey
+
+	// TODO(xla): Remove when MConnection is refactored into mPeer.
+	mConfig conn.MConnConfig
+}
+
+var _ PeerTransport = (*multiplexTransport)(nil)
+
+// NewMTransport returns network connected multiplexed peers.
+func NewMTransport(
+	nodeAddr NetAddress,
+	nodeInfo NodeInfo,
+	nodeKey NodeKey,
+) *multiplexTransport 
+```
+
+### Switch
+
+From now the Switch will depend on a fully setup `PeerTransport` to
+retrieve/reach out to its peers. As the more low-level concerns are pushed to
+the transport, we can omit passing the `config.P2PConfig` to the Switch.
+
+``` go
+func NewSwitch(transport PeerTransport, opts ...SwitchOption) *Switch
+```
+
+## Status
+
+In Review.
+
+## Consequences
+
+### Positive
+
+* free Switch from transport concerns - simpler implementation
+* pluggable transport implementation - simpler test setup
+* remove Switch dependency on P2PConfig - easier to test
+
+### Negative
+
+* more setup for tests which depend on Switches
+
+### Neutral
+
+* multiplexed will be the default implementation
+
+[0] These guards could be potentially extended to be pluggable much like
+middlewares to express different concerns required by differentally configured
+environments.

docs/architecture/adr-013-symmetric-crypto.md

@@ -0,0 +1,93 @@
+# ADR 013: Need for symmetric cryptography
+
+## Context
+
+We require symmetric ciphers to handle how we encrypt keys in the sdk,
+and to potentially encrypt `priv_validator.json` in tendermint.
+
+Currently we use AEAD's to support symmetric encryption,
+which is great since we want data integrity in addition to privacy and authenticity.
+We don't currently have a scenario where we want to encrypt without data integrity,
+so it is fine to optimize our code to just use AEAD's.
+Currently there is not a way to switch out AEAD's easily, this ADR outlines a way
+to easily swap these out.
+
+### How do we encrypt with AEAD's
+
+AEAD's typically require a nonce in addition to the key. 
+For the purposes we require symmetric cryptography for,
+we need encryption to be stateless.
+Because of this we use random nonces. 
+(Thus the AEAD must support random nonces)
+
+We currently construct a random nonce, and encrypt the data with it. 
+The returned value is `nonce || encrypted data`.
+The limitation of this is that does not provide a way to identify
+which algorithm was used in encryption.
+Consequently decryption with multiple algoritms is sub-optimal. 
+(You have to try them all)
+
+## Decision
+
+We should create the following two methods in a new `crypto/encoding/symmetric` package: 
+```golang
+func Encrypt(aead cipher.AEAD, plaintext []byte) (ciphertext []byte, err error)
+func Decrypt(key []byte, ciphertext []byte) (plaintext []byte, err error)
+func Register(aead cipher.AEAD, algo_name string, NewAead func(key []byte) (cipher.Aead, error)) error
+```
+
+This allows you to specify the algorithm in encryption, but not have to specify
+it in decryption. 
+This is intended for ease of use in downstream applications, in addition to people
+looking at the file directly.
+One downside is that for the encrypt function you must have already initialized an AEAD,
+but I don't really see this as an issue. 
+
+If there is no error in encryption, Encrypt will return `algo_name || nonce || aead_ciphertext`. 
+`algo_name` should be length prefixed, using standard varuint encoding.
+This will be binary data, but thats not a problem considering the nonce and ciphertext are also binary.
+
+This solution requires a mapping from aead type to name. 
+We can achieve this via reflection. 
+```golang
+func getType(myvar interface{}) string {
+    if t := reflect.TypeOf(myvar); t.Kind() == reflect.Ptr {
+        return "*" + t.Elem().Name()
+    } else {
+        return t.Name()
+    }
+}
+```
+Then we maintain a map from the name returned from `getType(aead)` to `algo_name`. 
+
+In decryption, we read the `algo_name`, and then instantiate a new AEAD with the key.
+Then we call the AEAD's decrypt method on the provided nonce/ciphertext.
+
+`Register` allows a downstream user to add their own desired AEAD to the symmetric package.
+It will error if the AEAD name is already registered.
+This prevents a malicious import from modifying / nullifying an AEAD at runtime.
+
+## Implementation strategy
+
+The golang implementation of what is proposed is rather straight forward.
+The concern is that we will break existing private keys if we just switch to this.
+If this is concerning, we can make a simple script which doesn't require decoding privkeys,
+for converting from the old format to the new one.
+
+## Status
+
+Proposed.
+
+## Consequences
+
+### Positive
+* Allows us to support new AEAD's, in a way that makes decryption easier
+* Allows downstream users to add their own AEAD
+
+### Negative
+* We will have to break all private keys stored on disk.
+They can be recovered using seed words, and upgrade scripts are simple.
+
+### Neutral
+* Caller has to instantiate the AEAD with the private key.
+However it forces them to be aware of what signing algorithm they are using, which is a positive.

docs/architecture/adr-014-secp-malleability.md

@@ -0,0 +1,61 @@
+# ADR 014: Secp256k1 Signature Malleability
+
+## Context
+
+Secp256k1 has two layers of malleability.
+The signer has a random nonce, and thus can produce many different valid signatures.
+This ADR is not concerned with that.
+The second layer of malleability basically allows one who is given a signature
+to produce exactly one more valid signature for the same message from the same public key.
+(They don't even have to know the message!)
+The math behind this will be explained in the subsequent section.
+
+Note that in many downstream applications, signatures will appear in a transaction, and therefore in the tx hash.
+This means that if someone broadcasts a transaction with secp256k1 signature, the signature can be altered into the other form by anyone in the p2p network.
+Thus the tx hash will change, and this altered tx hash may be committed instead.
+This breaks the assumption that you can broadcast a valid transaction and just wait for its hash to be included on chain.
+One example is if you are broadcasting a tx in cosmos,
+and you wait for it to appear on chain before incrementing your sequence number.
+You may never increment your sequence number if a different tx hash got committed.
+Removing this second layer of signature malleability concerns could ease downstream development.
+
+### ECDSA context
+
+Secp256k1 is ECDSA over a particular curve.
+The signature is of the form `(r, s)`, where `s` is a field element. 
+(The particular field is the `Z_n`, where the elliptic curve has order `n`)
+However `(r, -s)` is also another valid solution.
+Note that anyone can negate a group element, and therefore can get this second signature.
+
+## Decision
+
+We can just distinguish a canonical form for the ECDSA signatures. 
+Then we require that all ECDSA signatures be in the form which we defined as canonical.
+We reject signatures in non-canonical form.
+
+A canonical form is rather easy to define and check. 
+It would just be the smaller of the two values for `s`, defined lexicographically.
+This is a simple check, instead of checking if `s < n`, instead check `s <= (n - 1)/2`.
+An example of another cryptosystem using this 
+is the parity definition here https://github.com/zkcrypto/pairing/pull/30#issuecomment-372910663.
+
+This is the same solution Ethereum has chosen for solving secp malleability.
+
+## Proposed Implementation
+
+Fork https://github.com/btcsuite/btcd, and just update the [parse sig method](https://github.com/btcsuite/btcd/blob/master/btcec/signature.go#195) and serialize functions to enforce our canonical form.
+
+## Status
+
+Proposed.
+
+## Consequences
+
+### Positive
+* Lets us maintain the ability to expect a tx hash to appear in the blockchain.
+
+### Negative
+* More work in all future implementations (Though this is a very simple check)
+* Requires us to maintain another fork
+
+### Neutral

docs/architecture/adr-015-crypto-encoding.md

@@ -0,0 +1,80 @@
+# ADR 015: Crypto encoding 
+
+## Context
+
+We must standardize our method for encoding public keys and signatures on chain. 
+Currently we amino encode the public keys and signatures.
+The reason we are using amino here is primarily due to ease of support in
+parsing for other languages.
+We don't need its upgradability properties in cryptosystems, as a change in
+the crypto that requires adapting the encoding, likely warrants being deemed
+a new cryptosystem.
+(I.e. using new public parameters)
+
+## Decision
+
+### Public keys
+
+For public keys, we will continue to use amino encoding on the canonical
+representation of the pubkey.
+(Canonical as defined by the cryptosystem itself)
+This has two significant drawbacks.
+Amino encoding is less space-efficient, due to requiring support for upgradability.
+Amino encoding support requires forking protobuf and adding this new interface support
+option in the langauge of choice.
+
+The reason for continuing to use amino however is that people can create code
+more easily in languages that already have an up to date amino library.
+It is possible that this will change in the future, if it is deemed that
+requiring amino for interacting with tendermint cryptography is unneccessary.
+
+The arguments for space efficiency here are refuted on the basis that there are
+far more egregious wastages of space in the SDK.
+The space requirement of the public keys doesn't cause many problems beyond
+increasing the space attached to each validator / account.
+
+The alternative to using amino here would be for us to create an enum type.
+Switching to just an enum type is worthy of investigation post-launch.
+For referrence, part of amino encoding interfaces is basically a 4 byte enum
+type definition.
+Enum types would just change that 4 bytes to be a varuint, and it would remove
+the protobuf overhead, but it would be hard to integrate into the existing API.
+
+### Signatures
+
+Signatures should be switched to be `[]byte`.
+Spatial efficiency in the signatures is quite important,
+as it directly affects the gas cost of every transaction,
+and the throughput of the chain.
+Signatures don't need to encode what type they are for (unlike public keys)
+since public keys must already be known.
+Therefore we can validate the signature without needing to encode its type.
+
+When placed in state, signatures will still be amino encoded, but it will be the
+primitive type `[]byte` getting encoded.
+
+#### Ed25519
+Use the canonical representation for signatures.
+
+#### Secp256k1
+There isn't a clear canonical representation here.
+Signatures have two elements `r,s`.
+These bytes are encoded as `r || s`, where `r` and `s` are both exactly
+32 bytes long, encoded big-endian.
+This is basically Ethereum's encoding, but without the leading recovery bit.
+
+## Status
+
+Proposed. The signature section seems to be agreed upon for the most part.
+Needs decision on Enum types.
+
+## Consequences
+
+### Positive
+* More space efficient signatures
+
+### Negative
+* We have an amino dependency for cryptography.
+
+### Neutral
+* No change to public keys

docs/networks/fast-sync.md

@@ -1,8 +1,4 @@
-Fast Sync
-=========
-
-Background
-----------
+# Fast Sync
 
 In a proof of work blockchain, syncing with the chain is the same
 process as staying up-to-date with the consensus: download blocks, and
@@ -14,21 +10,19 @@ scratch can take a very long time. It's much faster to just download
 blocks and check the merkle tree of validators than to run the real-time
 consensus gossip protocol.
 
-Fast Sync
----------
+## Using Fast Sync
 
-To support faster syncing, tendermint offers a ``fast-sync`` mode, which
-is enabled by default, and can be toggled in the ``config.toml`` or via
-``--fast_sync=false``.
+To support faster syncing, tendermint offers a `fast-sync` mode, which
+is enabled by default, and can be toggled in the `config.toml` or via
+`--fast_sync=false`.
 
 In this mode, the tendermint daemon will sync hundreds of times faster
 than if it used the real-time consensus process. Once caught up, the
 daemon will switch out of fast sync and into the normal consensus mode.
-After running for some time, the node is considered ``caught up`` if it
+After running for some time, the node is considered `caught up` if it
 has at least one peer and it's height is at least as high as the max
-reported peer height. See `the IsCaughtUp
-method <https://github.com/tendermint/tendermint/blob/b467515719e686e4678e6da4e102f32a491b85a0/blockchain/pool.go#L128>`__.
+reported peer height. See [the IsCaughtUp
+method](https://github.com/tendermint/tendermint/blob/b467515719e686e4678e6da4e102f32a491b85a0/blockchain/pool.go#L128).
 
 If we're lagging sufficiently, we should go back to fast syncing, but
-this is an open issue:
-https://github.com/tendermint/tendermint/issues/129
+this is an [open issue](https://github.com/tendermint/tendermint/issues/129).

docs/spec/blockchain/blockchain.md

@@ -52,7 +52,8 @@ type Header struct {
     // application
     ResultsHash         []byte  // SimpleMerkle of []abci.Result from prevBlock
     AppHash             []byte  // Arbitrary state digest
-    ValidatorsHash      []byte  // SimpleMerkle of the ValidatorSet
+    ValidatorsHash      []byte  // SimpleMerkle of the current ValidatorSet
+    NextValidatorsHash  []byte  // SimpleMerkle of the next ValidatorSet
     ConsensusParamsHash []byte  // SimpleMerkle of the ConsensusParams
 
     // consensus
@@ -160,9 +161,12 @@ We refer to certain globally available objects:
 `block` is the block under consideration,
 `prevBlock` is the `block` at the previous height,
 and `state` keeps track of the validator set, the consensus parameters
-and other results from the application.
+and other results from the application. At the point when `block` is the block under consideration,
+the current version of the `state` corresponds to the state 
+after executing transactions from the `prevBlock`. 
 Elements of an object are accessed as expected,
-ie. `block.Header`. See [here](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/state.md) for the definition of `state`.
+ie. `block.Header`. 
+See [here](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/state.md) for the definition of `state`.
 
 ### Header
 
@@ -278,7 +282,14 @@ block.ValidatorsHash == SimpleMerkleRoot(state.Validators)
 
 Simple Merkle root of the current validator set that is committing the block.
 This can be used to validate the `LastCommit` included in the next block.
-May be updated by the application.
+
+### NextValidatorsHash
+
+```go
+block.NextValidatorsHash == SimpleMerkleRoot(state.NextValidators)
+```
+Simple Merkle root of the next validator set that will be the validator set that commits the next block.
+Modifications to the validator set are defined by the application.
 
 ### ConsensusParamsHash
 
@@ -407,25 +418,20 @@ set (TODO). Execute is defined as:
 
 ```go
 Execute(s State, app ABCIApp, block Block) State {
-    TODO: just spell out ApplyBlock here
-    and remove ABCIResponses struct.
-    abciResponses := app.ApplyBlock(block)
+    // Fuction ApplyBlock executes block of transactions against the app and returns the new root hash of the app state,
+    // modifications to the validator set and the changes of the consensus parameters.
+    AppHash, ValidatorChanges, ConsensusParamChanges := app.ApplyBlock(block)
 
     return State{
         LastResults: abciResponses.DeliverTxResults,
-        AppHash: abciResponses.AppHash,
-        Validators: UpdateValidators(state.Validators, abciResponses.ValidatorChanges),
+        AppHash: AppHash,
         LastValidators: state.Validators,
-        ConsensusParams: UpdateConsensusParams(state.ConsensusParams, abci.Responses.ConsensusParamChanges),
+        Validators: state.NextValidators,
+        NextValidators: UpdateValidators(state.NextValidators, ValidatorChanges), 
+        ConsensusParams: UpdateConsensusParams(state.ConsensusParams, ConsensusParamChanges),
     }
 }
 
-type ABCIResponses struct {
-    DeliverTxResults        []Result
-    ValidatorChanges        []Validator
-    ConsensusParamChanges   ConsensusParams
-    AppHash                 []byte
-}
 ```
 
 

docs/spec/blockchain/encoding.md

@@ -149,7 +149,33 @@ func MakeParts(obj interface{}, partSize int) []Part
 
 ## Merkle Trees
 
-Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure.
+For an overview of Merkle trees, see
+[wikipedia](https://en.wikipedia.org/wiki/Merkle_tree)
+
+
+A Simple Tree is a simple compact binary tree for a static list of items. Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. In a Simple Tree, the transactions and validation signatures of a block are hashed using this simple merkle tree logic.
+
+If the number of items is not a power of two, the tree will not be full
+and some leaf nodes will be at different levels. Simple Tree tries to
+keep both sides of the tree the same size, but the left side may be one
+greater, for example: 
+
+```
+   Simple Tree with 6 items           Simple Tree with 7 items 
+                                                        
+              *                                  *             
+             / \                                / \            
+           /     \                            /     \          
+         /         \                        /         \        
+       /             \                    /             \      
+      *               *                  *               *     
+     / \             / \                / \             / \    
+    /   \           /   \              /   \           /   \   
+   /     \         /     \            /     \         /     \  
+  *       h2      *       h5         *       *       *       h6
+ / \             / \                / \     / \     / \        
+h0  h1          h3  h4             h0  h1  h2  h3  h4  h5
+```
 
 Tendermint always uses the `TMHASH` hash function, which is the first 20-bytes
 of the SHA256:
@@ -235,6 +261,18 @@ func computeHashFromAunts(index, total int, leafHash []byte, innerHashes [][]byt
 }
 ```
 
+### Simple Tree with Dictionaries
+
+The Simple Tree is used to merkelize a list of items, so to merkelize a
+(short) dictionary of key-value pairs, encode the dictionary as an
+ordered list of ``KVPair`` structs. The block hash is such a hash
+derived from all the fields of the block ``Header``. The state hash is
+similarly derived.
+
+### IAVL+ Tree
+
+Because Tendermint only uses a Simple Merkle Tree, application developers are expect to use their own Merkle tree in their applications. For example, the IAVL+ Tree - an immutable self-balancing binary tree for persisting application state is used by the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk/blob/develop/docs/core/multistore.md)
+
 ## JSON
 
 ### Amino

docs/spec/blockchain/state.md

@@ -3,7 +3,7 @@
 ## State
 
 The state contains information whose cryptographic digest is included in block headers, and thus is
-necessary for validating new blocks. For instance, the set of validators and the results of
+necessary for validating new blocks. For instance, the validators set and the results of
 transactions are never included in blocks, but their Merkle roots are - the state keeps track of them.
 
 Note that the `State` object itself is an implementation detail, since it is never
@@ -18,8 +18,9 @@ type State struct {
     LastResults []Result
     AppHash []byte
 
-    Validators []Validator
     LastValidators []Validator
+    Validators []Validator
+    NextValidators []Validator
 
     ConsensusParams ConsensusParams
 }

docs/spec/consensus/consensus.md

@@ -1,9 +1,329 @@
-We are working to finalize an updated Tendermint specification with formal
-proofs of safety and liveness.
+# Byzantine Consensus Algorithm
 
-In the meantime, see the [description in the
-docs](http://tendermint.readthedocs.io/en/master/specification/byzantine-consensus-algorithm.html).
+## Terms
 
-There are also relevant but somewhat outdated descriptions in Jae Kwon's [original
-whitepaper](https://tendermint.com/static/docs/tendermint.pdf) and Ethan Buchman's [master's
-thesis](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769).
+-   The network is composed of optionally connected *nodes*. Nodes
+    directly connected to a particular node are called *peers*.
+-   The consensus process in deciding the next block (at some *height*
+    `H`) is composed of one or many *rounds*.
+-   `NewHeight`, `Propose`, `Prevote`, `Precommit`, and `Commit`
+    represent state machine states of a round. (aka `RoundStep` or
+    just "step").
+-   A node is said to be *at* a given height, round, and step, or at
+    `(H,R,S)`, or at `(H,R)` in short to omit the step.
+-   To *prevote* or *precommit* something means to broadcast a [prevote
+    vote](https://godoc.org/github.com/tendermint/tendermint/types#Vote)
+    or [first precommit
+    vote](https://godoc.org/github.com/tendermint/tendermint/types#FirstPrecommit)
+    for something.
+-   A vote *at* `(H,R)` is a vote signed with the bytes for `H` and `R`
+    included in its [sign-bytes](block-structure.html#vote-sign-bytes).
+-   *+2/3* is short for "more than 2/3"
+-   *1/3+* is short for "1/3 or more"
+-   A set of +2/3 of prevotes for a particular block or `<nil>` at
+    `(H,R)` is called a *proof-of-lock-change* or *PoLC* for short.
+
+## State Machine Overview
+
+At each height of the blockchain a round-based protocol is run to
+determine the next block. Each round is composed of three *steps*
+(`Propose`, `Prevote`, and `Precommit`), along with two special steps
+`Commit` and `NewHeight`.
+
+In the optimal scenario, the order of steps is:
+
+```
+NewHeight -> (Propose -> Prevote -> Precommit)+ -> Commit -> NewHeight ->...
+```
+
+The sequence `(Propose -> Prevote -> Precommit)` is called a *round*.
+There may be more than one round required to commit a block at a given
+height. Examples for why more rounds may be required include:
+
+-   The designated proposer was not online.
+-   The block proposed by the designated proposer was not valid.
+-   The block proposed by the designated proposer did not propagate
+    in time.
+-   The block proposed was valid, but +2/3 of prevotes for the proposed
+    block were not received in time for enough validator nodes by the
+    time they reached the `Precommit` step. Even though +2/3 of prevotes
+    are necessary to progress to the next step, at least one validator
+    may have voted `<nil>` or maliciously voted for something else.
+-   The block proposed was valid, and +2/3 of prevotes were received for
+    enough nodes, but +2/3 of precommits for the proposed block were not
+    received for enough validator nodes.
+
+Some of these problems are resolved by moving onto the next round &
+proposer. Others are resolved by increasing certain round timeout
+parameters over each successive round.
+
+## State Machine Diagram
+
+```
+                         +-------------------------------------+
+                         v                                     |(Wait til `CommmitTime+timeoutCommit`)
+                   +-----------+                         +-----+-----+
+      +----------> |  Propose  +--------------+          | NewHeight |
+      |            +-----------+              |          +-----------+
+      |                                       |                ^
+      |(Else, after timeoutPrecommit)         v                |
++-----+-----+                           +-----------+          |
+| Precommit |  <------------------------+  Prevote  |          |
++-----+-----+                           +-----------+          |
+      |(When +2/3 Precommits for block found)                  |
+      v                                                        |
++--------------------------------------------------------------------+
+       |  Commit                                                            |
+       |                                                                    |
+       |  * Set CommitTime = now;                                           |
+       |  * Wait for block, then stage/save/commit block;                   |
+       +--------------------------------------------------------------------+
+```
+
+Background Gossip
+=================
+
+A node may not have a corresponding validator private key, but it
+nevertheless plays an active role in the consensus process by relaying
+relevant meta-data, proposals, blocks, and votes to its peers. A node
+that has the private keys of an active validator and is engaged in
+signing votes is called a *validator-node*. All nodes (not just
+validator-nodes) have an associated state (the current height, round,
+and step) and work to make progress.
+
+Between two nodes there exists a `Connection`, and multiplexed on top of
+this connection are fairly throttled `Channel`s of information. An
+epidemic gossip protocol is implemented among some of these channels to
+bring peers up to speed on the most recent state of consensus. For
+example,
+
+-   Nodes gossip `PartSet` parts of the current round's proposer's
+    proposed block. A LibSwift inspired algorithm is used to quickly
+    broadcast blocks across the gossip network.
+-   Nodes gossip prevote/precommit votes. A node `NODE_A` that is ahead
+    of `NODE_B` can send `NODE_B` prevotes or precommits for `NODE_B`'s
+    current (or future) round to enable it to progress forward.
+-   Nodes gossip prevotes for the proposed PoLC (proof-of-lock-change)
+    round if one is proposed.
+-   Nodes gossip to nodes lagging in blockchain height with block
+    [commits](https://godoc.org/github.com/tendermint/tendermint/types#Commit)
+    for older blocks.
+-   Nodes opportunistically gossip `HasVote` messages to hint peers what
+    votes it already has.
+-   Nodes broadcast their current state to all neighboring peers. (but
+    is not gossiped further)
+
+There's more, but let's not get ahead of ourselves here.
+
+## Proposals
+
+A proposal is signed and published by the designated proposer at each
+round. The proposer is chosen by a deterministic and non-choking round
+robin selection algorithm that selects proposers in proportion to their
+voting power (see
+[implementation](https://github.com/tendermint/tendermint/blob/develop/types/validator_set.go)).
+
+A proposal at `(H,R)` is composed of a block and an optional latest
+`PoLC-Round < R` which is included iff the proposer knows of one. This
+hints the network to allow nodes to unlock (when safe) to ensure the
+liveness property.
+
+## State Machine Spec
+
+### Propose Step (height:H,round:R)
+
+Upon entering `Propose`: - The designated proposer proposes a block at
+`(H,R)`.
+
+The `Propose` step ends: - After `timeoutProposeR` after entering
+`Propose`. --> goto `Prevote(H,R)` - After receiving proposal block
+and all prevotes at `PoLC-Round`. --> goto `Prevote(H,R)` - After
+[common exit conditions](#common-exit-conditions)
+
+### Prevote Step (height:H,round:R)
+
+Upon entering `Prevote`, each validator broadcasts its prevote vote.
+
+-   First, if the validator is locked on a block since `LastLockRound`
+    but now has a PoLC for something else at round `PoLC-Round` where
+    `LastLockRound < PoLC-Round < R`, then it unlocks.
+-   If the validator is still locked on a block, it prevotes that.
+-   Else, if the proposed block from `Propose(H,R)` is good, it
+    prevotes that.
+-   Else, if the proposal is invalid or wasn't received on time, it
+    prevotes `<nil>`.
+
+The `Prevote` step ends: - After +2/3 prevotes for a particular block or
+`<nil>`. -->; goto `Precommit(H,R)` - After `timeoutPrevote` after
+receiving any +2/3 prevotes. --> goto `Precommit(H,R)` - After
+[common exit conditions](#common-exit-conditions)
+
+### Precommit Step (height:H,round:R)
+
+Upon entering `Precommit`, each validator broadcasts its precommit vote.
+- If the validator has a PoLC at `(H,R)` for a particular block `B`, it
+(re)locks (or changes lock to) and precommits `B` and sets
+`LastLockRound = R`. - Else, if the validator has a PoLC at `(H,R)` for
+`<nil>`, it unlocks and precommits `<nil>`. - Else, it keeps the lock
+unchanged and precommits `<nil>`.
+
+A precommit for `<nil>` means "I didn’t see a PoLC for this round, but I
+did get +2/3 prevotes and waited a bit".
+
+The Precommit step ends: - After +2/3 precommits for `<nil>`. -->
+goto `Propose(H,R+1)` - After `timeoutPrecommit` after receiving any
++2/3 precommits. --> goto `Propose(H,R+1)` - After [common exit
+conditions](#common-exit-conditions)
+
+### Common exit conditions
+
+-   After +2/3 precommits for a particular block. --> goto
+    `Commit(H)`
+-   After any +2/3 prevotes received at `(H,R+x)`. --> goto
+    `Prevote(H,R+x)`
+-   After any +2/3 precommits received at `(H,R+x)`. --> goto
+    `Precommit(H,R+x)`
+
+### Commit Step (height:H)
+
+-   Set `CommitTime = now()`
+-   Wait until block is received. --> goto `NewHeight(H+1)`
+
+### NewHeight Step (height:H)
+
+-   Move `Precommits` to `LastCommit` and increment height.
+-   Set `StartTime = CommitTime+timeoutCommit`
+-   Wait until `StartTime` to receive straggler commits. --> goto
+    `Propose(H,0)`
+
+## Proofs
+
+### Proof of Safety
+
+Assume that at most -1/3 of the voting power of validators is byzantine.
+If a validator commits block `B` at round `R`, it's because it saw +2/3
+of precommits at round `R`. This implies that 1/3+ of honest nodes are
+still locked at round `R' > R`. These locked validators will remain
+locked until they see a PoLC at `R' > R`, but this won't happen because
+1/3+ are locked and honest, so at most -2/3 are available to vote for
+anything other than `B`.
+
+### Proof of Liveness
+
+If 1/3+ honest validators are locked on two different blocks from
+different rounds, a proposers' `PoLC-Round` will eventually cause nodes
+locked from the earlier round to unlock. Eventually, the designated
+proposer will be one that is aware of a PoLC at the later round. Also,
+`timeoutProposalR` increments with round `R`, while the size of a
+proposal are capped, so eventually the network is able to "fully gossip"
+the whole proposal (e.g. the block & PoLC).
+
+### Proof of Fork Accountability
+
+Define the JSet (justification-vote-set) at height `H` of a validator
+`V1` to be all the votes signed by the validator at `H` along with
+justification PoLC prevotes for each lock change. For example, if `V1`
+signed the following precommits: `Precommit(B1 @ round 0)`,
+`Precommit(<nil> @ round 1)`, `Precommit(B2 @ round 4)` (note that no
+precommits were signed for rounds 2 and 3, and that's ok),
+`Precommit(B1 @ round 0)` must be justified by a PoLC at round 0, and
+`Precommit(B2 @ round 4)` must be justified by a PoLC at round 4; but
+the precommit for `<nil>` at round 1 is not a lock-change by definition
+so the JSet for `V1` need not include any prevotes at round 1, 2, or 3
+(unless `V1` happened to have prevoted for those rounds).
+
+Further, define the JSet at height `H` of a set of validators `VSet` to
+be the union of the JSets for each validator in `VSet`. For a given
+commit by honest validators at round `R` for block `B` we can construct
+a JSet to justify the commit for `B` at `R`. We say that a JSet
+*justifies* a commit at `(H,R)` if all the committers (validators in the
+commit-set) are each justified in the JSet with no duplicitous vote
+signatures (by the committers).
+
+-   **Lemma**: When a fork is detected by the existence of two
+    conflicting [commits](./validators.html#commiting-a-block), the
+    union of the JSets for both commits (if they can be compiled) must
+    include double-signing by at least 1/3+ of the validator set.
+    **Proof**: The commit cannot be at the same round, because that
+    would immediately imply double-signing by 1/3+. Take the union of
+    the JSets of both commits. If there is no double-signing by at least
+    1/3+ of the validator set in the union, then no honest validator
+    could have precommitted any different block after the first commit.
+    Yet, +2/3 did. Reductio ad absurdum.
+
+As a corollary, when there is a fork, an external process can determine
+the blame by requiring each validator to justify all of its round votes.
+Either we will find 1/3+ who cannot justify at least one of their votes,
+and/or, we will find 1/3+ who had double-signed.
+
+### Alternative algorithm
+
+Alternatively, we can take the JSet of a commit to be the "full commit".
+That is, if light clients and validators do not consider a block to be
+committed unless the JSet of the commit is also known, then we get the
+desirable property that if there ever is a fork (e.g. there are two
+conflicting "full commits"), then 1/3+ of the validators are immediately
+punishable for double-signing.
+
+There are many ways to ensure that the gossip network efficiently share
+the JSet of a commit. One solution is to add a new message type that
+tells peers that this node has (or does not have) a +2/3 majority for B
+(or) at (H,R), and a bitarray of which votes contributed towards that
+majority. Peers can react by responding with appropriate votes.
+
+We will implement such an algorithm for the next iteration of the
+Tendermint consensus protocol.
+
+Other potential improvements include adding more data in votes such as
+the last known PoLC round that caused a lock change, and the last voted
+round/step (or, we may require that validators not skip any votes). This
+may make JSet verification/gossip logic easier to implement.
+
+### Censorship Attacks
+
+Due to the definition of a block
+[commit](../../tendermint-core/validator.md#commiting-a-block), any 1/3+ coalition of
+validators can halt the blockchain by not broadcasting their votes. Such
+a coalition can also censor particular transactions by rejecting blocks
+that include these transactions, though this would result in a
+significant proportion of block proposals to be rejected, which would
+slow down the rate of block commits of the blockchain, reducing its
+utility and value. The malicious coalition might also broadcast votes in
+a trickle so as to grind blockchain block commits to a near halt, or
+engage in any combination of these attacks.
+
+If a global active adversary were also involved, it can partition the
+network in such a way that it may appear that the wrong subset of
+validators were responsible for the slowdown. This is not just a
+limitation of Tendermint, but rather a limitation of all consensus
+protocols whose network is potentially controlled by an active
+adversary.
+
+### Overcoming Forks and Censorship Attacks
+
+For these types of attacks, a subset of the validators through external
+means should coordinate to sign a reorg-proposal that chooses a fork
+(and any evidence thereof) and the initial subset of validators with
+their signatures. Validators who sign such a reorg-proposal forego its
+collateral on all other forks. Clients should verify the signatures on
+the reorg-proposal, verify any evidence, and make a judgement or prompt
+the end-user for a decision. For example, a phone wallet app may prompt
+the user with a security warning, while a refrigerator may accept any
+reorg-proposal signed by +1/2 of the original validators.
+
+No non-synchronous Byzantine fault-tolerant algorithm can come to
+consensus when 1/3+ of validators are dishonest, yet a fork assumes that
+1/3+ of validators have already been dishonest by double-signing or
+lock-changing without justification. So, signing the reorg-proposal is a
+coordination problem that cannot be solved by any non-synchronous
+protocol (i.e. automatically, and without making assumptions about the
+reliability of the underlying network). It must be provided by means
+external to the weakly-synchronous Tendermint consensus algorithm. For
+now, we leave the problem of reorg-proposal coordination to human
+coordination via internet media. Validators must take care to ensure
+that there are no significant network partitions, to avoid situations
+where two conflicting reorg-proposals are signed.
+
+Assuming that the external coordination medium and protocol is robust,
+it follows that forks are less of a concern than [censorship
+attacks](#censorship-attacks).

docs/spec/p2p/peer.md

@@ -27,27 +27,24 @@ Both handshakes have configurable timeouts (they should complete quickly).
 ### Authenticated Encryption Handshake
 
 Tendermint implements the Station-to-Station protocol
-using ED25519 keys for Diffie-Helman key-exchange and NACL SecretBox for encryption.
+using X25519 keys for Diffie-Helman key-exchange and chacha20poly1305 for encryption.
 It goes as follows:
-- generate an emphemeral ED25519 keypair
+- generate an ephemeral X25519 keypair
 - send the ephemeral public key to the peer
 - wait to receive the peer's ephemeral public key
 - compute the Diffie-Hellman shared secret using the peers ephemeral public key and our ephemeral private key
-- generate two nonces to use for encryption (sending and receiving) as follows:
-    - sort the ephemeral public keys in ascending order and concatenate them
-    - RIPEMD160 the result
-    - append 4 empty bytes (extending the hash to 24-bytes)
-    - the result is nonce1
-    - flip the last bit of nonce1 to get nonce2
-    - if we had the smaller ephemeral pubkey, use nonce1 for receiving, nonce2 for sending;
-        else the opposite
-- all communications from now on are encrypted using the shared secret and the nonces, where each nonce
-increments by 2 every time it is used
+- generate two keys to use for encryption (sending and receiving) and a challenge for authentication as follows:
+    - create a hkdf-sha256 instance with the key being the diffie hellman shared secret, and info parameter as
+      `TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN`
+    - get 96 bytes of output from hkdf-sha256
+    - if we had the smaller ephemeral pubkey, use the first 32 bytes for the key for receiving, the second 32 bytes for sending; else the opposite
+    - use the last 32 bytes of output for the challenge
+- use a seperate nonce for receiving and sending. Both nonces start at 0, and should support the full 96 bit nonce range
+- all communications from now on are encrypted in 1024 byte frames,
+using the respective secret and nonce. Each nonce is incremented by one after each use. 
 - we now have an encrypted channel, but still need to authenticate
-- generate a common challenge to sign:
-    - SHA256 of the sorted (lowest first) and concatenated ephemeral pub keys
-- sign the common challenge with our persistent private key
-- send the go-wire encoded persistent pubkey and signature to the peer
+- sign the common challenge obtained from the hkdf with our persistent private key
+- send the amino encoded persistent pubkey and signature to the peer
 - wait to receive the persistent public key and signature from the peer
 - verify the signature on the challenge using the peer's persistent public key
 

docs/spec/reactors/pex/pex.md

@@ -12,7 +12,8 @@ them.
 Some peers can be marked as `private`, which means
 we will not put them in the address book or gossip them to others.
 
-All peers except private peers are tracked using the address book.
+All peers except private peers and peers coming from them are tracked using the
+address book.
 
 ## Discovery
 

docs/spec/scripts/crypto.go

@@ -5,12 +5,12 @@ import (
 	"os"
 
 	amino "github.com/tendermint/go-amino"
-	crypto "github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 func main() {
 	cdc := amino.NewCodec()
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 	cdc.PrintTypes(os.Stdout)
 	fmt.Println("")
 }

docs/specification/block-structure.rst

@@ -1,218 +0,0 @@
-Block Structure
-===============
-
-The tendermint consensus engine records all agreements by a
-supermajority of nodes into a blockchain, which is replicated among all
-nodes. This blockchain is accessible via various rpc endpoints, mainly
-``/block?height=`` to get the full block, as well as
-``/blockchain?minHeight=_&maxHeight=_`` to get a list of headers. But
-what exactly is stored in these blocks?
-
-Block
-~~~~~
-
-A
-`Block <https://godoc.org/github.com/tendermint/tendermint/types#Block>`__
-contains:
-
--  a `Header <#header>`__ contains merkle hashes for various chain
-   states
--  the
-   `Data <https://godoc.org/github.com/tendermint/tendermint/types#Data>`__
-   is all transactions which are to be processed
--  the `LastCommit <#commit>`__ > 2/3 signatures for the last block
-
-The signatures returned along with block ``H`` are those validating
-block ``H-1``. This can be a little confusing, but we must also consider
-that the ``Header`` also contains the ``LastCommitHash``. It would be
-impossible for a Header to include the commits that sign it, as it would
-cause an infinite loop here. But when we get block ``H``, we find
-``Header.LastCommitHash``, which must match the hash of ``LastCommit``.
-
-Header
-~~~~~~
-
-The
-`Header <https://godoc.org/github.com/tendermint/tendermint/types#Header>`__
-contains lots of information (follow link for up-to-date info). Notably,
-it maintains the ``Height``, the ``LastBlockID`` (to make it a chain),
-and hashes of the data, the app state, and the validator set. This is
-important as the only item that is signed by the validators is the
-``Header``, and all other data must be validated against one of the
-merkle hashes in the ``Header``.
-
-The ``DataHash`` can provide a nice check on the
-`Data <https://godoc.org/github.com/tendermint/tendermint/types#Data>`__
-returned in this same block. If you are subscribed to new blocks, via
-tendermint RPC, in order to display or process the new transactions you
-should at least validate that the ``DataHash`` is valid. If it is
-important to verify autheniticity, you must wait for the ``LastCommit``
-from the next block to make sure the block header (including
-``DataHash``) was properly signed.
-
-The ``ValidatorHash`` contains a hash of the current
-`Validators <https://godoc.org/github.com/tendermint/tendermint/types#Validator>`__.
-Tracking all changes in the validator set is complex, but a client can
-quickly compare this hash with the `hash of the currently known
-validators <https://godoc.org/github.com/tendermint/tendermint/types#ValidatorSet.Hash>`__
-to see if there have been changes.
-
-The ``AppHash`` serves as the basis for validating any merkle proofs
-that come from the ABCI application. It represents the
-state of the actual application, rather that the state of the blockchain
-itself. This means it's necessary in order to perform any business
-logic, such as verifying an account balance.
-
-**Note** After the transactions are committed to a block, they still
-need to be processed in a separate step, which happens between the
-blocks. If you find a given transaction in the block at height ``H``,
-the effects of running that transaction will be first visible in the
-``AppHash`` from the block header at height ``H+1``.
-
-Like the ``LastCommit`` issue, this is a requirement of the immutability
-of the block chain, as the application only applies transactions *after*
-they are commited to the chain.
-
-Commit
-~~~~~~
-
-The
-`Commit <https://godoc.org/github.com/tendermint/tendermint/types#Commit>`__
-contains a set of
-`Votes <https://godoc.org/github.com/tendermint/tendermint/types#Vote>`__
-that were made by the validator set to reach consensus on this block.
-This is the key to the security in any PoS system, and actually no data
-that cannot be traced back to a block header with a valid set of Votes
-can be trusted. Thus, getting the Commit data and verifying the votes is
-extremely important.
-
-As mentioned above, in order to find the ``precommit votes`` for block
-header ``H``, we need to query block ``H+1``. Then we need to check the
-votes, make sure they really are for that block, and properly formatted.
-Much of this code is implemented in Go in the
-`light-client <https://github.com/tendermint/light-client>`__ package.
-If you look at the code, you will notice that we need to provide the
-``chainID`` of the blockchain in order to properly calculate the votes.
-This is to protect anyone from swapping votes between chains to fake (or
-frame) a validator. Also note that this ``chainID`` is in the
-``genesis.json`` from *Tendermint*, not the ``genesis.json`` from the
-basecoin app (`that is a different
-chainID... <https://github.com/cosmos/cosmos-sdk/issues/32>`__).
-
-Once we have those votes, and we calculated the proper `sign
-bytes <https://godoc.org/github.com/tendermint/tendermint/types#Vote.WriteSignBytes>`__
-using the chainID and a `nice helper
-function <https://godoc.org/github.com/tendermint/tendermint/types#SignBytes>`__,
-we can verify them. The light client is responsible for maintaining a
-set of validators that we trust. Each vote only stores the validators
-``Address``, as well as the ``Signature``. Assuming we have a local copy
-of the trusted validator set, we can look up the ``Public Key`` of the
-validator given its ``Address``, then verify that the ``Signature``
-matches the ``SignBytes`` and ``Public Key``. Then we sum up the total
-voting power of all validators, whose votes fulfilled all these
-stringent requirements. If the total number of voting power for a single
-block is greater than 2/3 of all voting power, then we can finally trust
-the block header, the AppHash, and the proof we got from the ABCI
-application.
-
-Vote Sign Bytes
-^^^^^^^^^^^^^^^
-
-The ``sign-bytes`` of a vote is produced by taking a
-`stable-json <https://github.com/substack/json-stable-stringify>`__-like
-deterministic JSON `wire <./wire-protocol.html>`__ encoding of
-the vote (excluding the ``Signature`` field), and wrapping it with
-``{"chain_id":"my_chain","vote":...}``.
-
-For example, a precommit vote might have the following ``sign-bytes``:
-
-.. code:: json
-
-    {"chain_id":"my_chain","vote":{"block_hash":"611801F57B4CE378DF1A3FFF1216656E89209A99","block_parts_header":{"hash":"B46697379DBE0774CC2C3B656083F07CA7E0F9CE","total":123},"height":1234,"round":1,"type":2}}
-
-Block Hash
-~~~~~~~~~~
-
-The `block
-hash <https://godoc.org/github.com/tendermint/tendermint/types#Block.Hash>`__
-is the `Simple Tree hash <./merkle.html#simple-tree-with-dictionaries>`__
-of the fields of the block ``Header`` encoded as a list of
-``KVPair``\ s.
-
-Transaction
-~~~~~~~~~~~
-
-A transaction is any sequence of bytes. It is up to your
-ABCI application to accept or reject transactions.
-
-BlockID
-~~~~~~~
-
-Many of these data structures refer to the
-`BlockID <https://godoc.org/github.com/tendermint/tendermint/types#BlockID>`__,
-which is the ``BlockHash`` (hash of the block header, also referred to
-by the next block) along with the ``PartSetHeader``. The
-``PartSetHeader`` is explained below and is used internally to
-orchestrate the p2p propogation. For clients, it is basically opaque
-bytes, but they must match for all votes.
-
-PartSetHeader
-~~~~~~~~~~~~~
-
-The
-`PartSetHeader <https://godoc.org/github.com/tendermint/tendermint/types#PartSetHeader>`__
-contains the total number of pieces in a
-`PartSet <https://godoc.org/github.com/tendermint/tendermint/types#PartSet>`__,
-and the Merkle root hash of those pieces.
-
-PartSet
-~~~~~~~
-
-PartSet is used to split a byteslice of data into parts (pieces) for
-transmission. By splitting data into smaller parts and computing a
-Merkle root hash on the list, you can verify that a part is legitimately
-part of the complete data, and the part can be forwarded to other peers
-before all the parts are known. In short, it's a fast way to securely
-propagate a large chunk of data (like a block) over a gossip network.
-
-PartSet was inspired by the LibSwift project.
-
-Usage:
-
-.. code:: go
-
-    data := RandBytes(2 << 20) // Something large
-
-    partSet := NewPartSetFromData(data)
-    partSet.Total()     // Total number of 4KB parts
-    partSet.Count()     // Equal to the Total, since we already have all the parts
-    partSet.Hash()      // The Merkle root hash
-    partSet.BitArray()  // A BitArray of partSet.Total() 1's
-
-    header := partSet.Header() // Send this to the peer
-    header.Total        // Total number of parts
-    header.Hash         // The merkle root hash
-
-    // Now we'll reconstruct the data from the parts
-    partSet2 := NewPartSetFromHeader(header)
-    partSet2.Total()    // Same total as partSet.Total()
-    partSet2.Count()    // Zero, since this PartSet doesn't have any parts yet.
-    partSet2.Hash()     // Same hash as in partSet.Hash()
-    partSet2.BitArray() // A BitArray of partSet.Total() 0's
-
-    // In a gossip network the parts would arrive in arbitrary order, perhaps
-    // in response to explicit requests for parts, or optimistically in response
-    // to the receiving peer's partSet.BitArray().
-    for !partSet2.IsComplete() {
-        part := receivePartFromGossipNetwork()
-        added, err := partSet2.AddPart(part)
-        if err != nil {
-        // A wrong part,
-            // the merkle trail does not hash to partSet2.Hash()
-        } else if !added {
-            // A duplicate part already received
-        }
-    }
-
-    data2, _ := ioutil.ReadAll(partSet2.GetReader())
-    bytes.Equal(data, data2) // true

docs/specification/byzantine-consensus-algorithm.rst

@@ -1,349 +0,0 @@
-Byzantine Consensus Algorithm
-=============================
-
-Terms
------
-
--  The network is composed of optionally connected *nodes*. Nodes
-   directly connected to a particular node are called *peers*.
--  The consensus process in deciding the next block (at some *height*
-   ``H``) is composed of one or many *rounds*.
--  ``NewHeight``, ``Propose``, ``Prevote``, ``Precommit``, and
-   ``Commit`` represent state machine states of a round. (aka
-   ``RoundStep`` or just "step").
--  A node is said to be *at* a given height, round, and step, or at
-   ``(H,R,S)``, or at ``(H,R)`` in short to omit the step.
--  To *prevote* or *precommit* something means to broadcast a `prevote
-   vote <https://godoc.org/github.com/tendermint/tendermint/types#Vote>`__
-   or `first precommit
-   vote <https://godoc.org/github.com/tendermint/tendermint/types#FirstPrecommit>`__
-   for something.
--  A vote *at* ``(H,R)`` is a vote signed with the bytes for ``H`` and
-   ``R`` included in its
-   `sign-bytes <block-structure.html#vote-sign-bytes>`__.
--  *+2/3* is short for "more than 2/3"
--  *1/3+* is short for "1/3 or more"
--  A set of +2/3 of prevotes for a particular block or ``<nil>`` at
-   ``(H,R)`` is called a *proof-of-lock-change* or *PoLC* for short.
-
-State Machine Overview
-----------------------
-
-At each height of the blockchain a round-based protocol is run to
-determine the next block. Each round is composed of three *steps*
-(``Propose``, ``Prevote``, and ``Precommit``), along with two special
-steps ``Commit`` and ``NewHeight``.
-
-In the optimal scenario, the order of steps is:
-
-::
-
-    NewHeight -> (Propose -> Prevote -> Precommit)+ -> Commit -> NewHeight ->...
-
-The sequence ``(Propose -> Prevote -> Precommit)`` is called a *round*.
-There may be more than one round required to commit a block at a given
-height. Examples for why more rounds may be required include:
-
--  The designated proposer was not online.
--  The block proposed by the designated proposer was not valid.
--  The block proposed by the designated proposer did not propagate in
-   time.
--  The block proposed was valid, but +2/3 of prevotes for the proposed
-   block were not received in time for enough validator nodes by the
-   time they reached the ``Precommit`` step. Even though +2/3 of
-   prevotes are necessary to progress to the next step, at least one
-   validator may have voted ``<nil>`` or maliciously voted for something
-   else.
--  The block proposed was valid, and +2/3 of prevotes were received for
-   enough nodes, but +2/3 of precommits for the proposed block were not
-   received for enough validator nodes.
-
-Some of these problems are resolved by moving onto the next round &
-proposer. Others are resolved by increasing certain round timeout
-parameters over each successive round.
-
-State Machine Diagram
----------------------
-
-::
-
-                                +-------------------------------------+
-                                v                                     |(Wait til `CommmitTime+timeoutCommit`)
-                          +-----------+                         +-----+-----+
-             +----------> |  Propose  +--------------+          | NewHeight |
-             |            +-----------+              |          +-----------+
-             |                                       |                ^
-             |(Else, after timeoutPrecommit)         v                |
-       +-----+-----+                           +-----------+          |
-       | Precommit |  <------------------------+  Prevote  |          |
-       +-----+-----+                           +-----------+          |
-             |(When +2/3 Precommits for block found)                  |
-             v                                                        |
-       +--------------------------------------------------------------------+
-       |  Commit                                                            |
-       |                                                                    |
-       |  * Set CommitTime = now;                                           |
-       |  * Wait for block, then stage/save/commit block;                   |
-       +--------------------------------------------------------------------+
-
-Background Gossip
------------------
-
-A node may not have a corresponding validator private key, but it
-nevertheless plays an active role in the consensus process by relaying
-relevant meta-data, proposals, blocks, and votes to its peers. A node
-that has the private keys of an active validator and is engaged in
-signing votes is called a *validator-node*. All nodes (not just
-validator-nodes) have an associated state (the current height, round,
-and step) and work to make progress.
-
-Between two nodes there exists a ``Connection``, and multiplexed on top
-of this connection are fairly throttled ``Channel``\ s of information.
-An epidemic gossip protocol is implemented among some of these channels
-to bring peers up to speed on the most recent state of consensus. For
-example,
-
--  Nodes gossip ``PartSet`` parts of the current round's proposer's
-   proposed block. A LibSwift inspired algorithm is used to quickly
-   broadcast blocks across the gossip network.
--  Nodes gossip prevote/precommit votes. A node NODE\_A that is ahead of
-   NODE\_B can send NODE\_B prevotes or precommits for NODE\_B's current
-   (or future) round to enable it to progress forward.
--  Nodes gossip prevotes for the proposed PoLC (proof-of-lock-change)
-   round if one is proposed.
--  Nodes gossip to nodes lagging in blockchain height with block
-   `commits <https://godoc.org/github.com/tendermint/tendermint/types#Commit>`__
-   for older blocks.
--  Nodes opportunistically gossip ``HasVote`` messages to hint peers
-   what votes it already has.
--  Nodes broadcast their current state to all neighboring peers. (but is
-   not gossiped further)
-
-There's more, but let's not get ahead of ourselves here.
-
-Proposals
----------
-
-A proposal is signed and published by the designated proposer at each
-round. The proposer is chosen by a deterministic and non-choking round
-robin selection algorithm that selects proposers in proportion to their
-voting power. (see
-`implementation <https://github.com/tendermint/tendermint/blob/develop/types/validator_set.go>`__)
-
-A proposal at ``(H,R)`` is composed of a block and an optional latest
-``PoLC-Round < R`` which is included iff the proposer knows of one. This
-hints the network to allow nodes to unlock (when safe) to ensure the
-liveness property.
-
-State Machine Spec
-------------------
-
-Propose Step (height:H,round:R)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upon entering ``Propose``: - The designated proposer proposes a block at
-``(H,R)``.
-
-The ``Propose`` step ends: - After ``timeoutProposeR`` after entering
-``Propose``. --> goto ``Prevote(H,R)`` - After receiving proposal block
-and all prevotes at ``PoLC-Round``. --> goto ``Prevote(H,R)`` - After
-`common exit conditions <#common-exit-conditions>`__
-
-Prevote Step (height:H,round:R)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upon entering ``Prevote``, each validator broadcasts its prevote vote.
-
--  First, if the validator is locked on a block since ``LastLockRound``
-   but now has a PoLC for something else at round ``PoLC-Round`` where
-   ``LastLockRound < PoLC-Round < R``, then it unlocks.
--  If the validator is still locked on a block, it prevotes that.
--  Else, if the proposed block from ``Propose(H,R)`` is good, it
-   prevotes that.
--  Else, if the proposal is invalid or wasn't received on time, it
-   prevotes ``<nil>``.
-
-The ``Prevote`` step ends: - After +2/3 prevotes for a particular block
-or ``<nil>``. --> goto ``Precommit(H,R)`` - After ``timeoutPrevote``
-after receiving any +2/3 prevotes. --> goto ``Precommit(H,R)`` - After
-`common exit conditions <#common-exit-conditions>`__
-
-Precommit Step (height:H,round:R)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upon entering ``Precommit``, each validator broadcasts its precommit
-vote. - If the validator has a PoLC at ``(H,R)`` for a particular block
-``B``, it (re)locks (or changes lock to) and precommits ``B`` and sets
-``LastLockRound = R``. - Else, if the validator has a PoLC at ``(H,R)``
-for ``<nil>``, it unlocks and precommits ``<nil>``. - Else, it keeps the
-lock unchanged and precommits ``<nil>``.
-
-A precommit for ``<nil>`` means "I didn’t see a PoLC for this round, but
-I did get +2/3 prevotes and waited a bit".
-
-The Precommit step ends: - After +2/3 precommits for ``<nil>``. --> goto
-``Propose(H,R+1)`` - After ``timeoutPrecommit`` after receiving any +2/3
-precommits. --> goto ``Propose(H,R+1)`` - After `common exit
-conditions <#common-exit-conditions>`__
-
-common exit conditions
-^^^^^^^^^^^^^^^^^^^^^^
-
--  After +2/3 precommits for a particular block. --> goto ``Commit(H)``
--  After any +2/3 prevotes received at ``(H,R+x)``. --> goto
-   ``Prevote(H,R+x)``
--  After any +2/3 precommits received at ``(H,R+x)``. --> goto
-   ``Precommit(H,R+x)``
-
-Commit Step (height:H)
-~~~~~~~~~~~~~~~~~~~~~~
-
--  Set ``CommitTime = now()``
--  Wait until block is received. --> goto ``NewHeight(H+1)``
-
-NewHeight Step (height:H)
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
--  Move ``Precommits`` to ``LastCommit`` and increment height.
--  Set ``StartTime = CommitTime+timeoutCommit``
--  Wait until ``StartTime`` to receive straggler commits. --> goto
-   ``Propose(H,0)``
-
-Proofs
-------
-
-Proof of Safety
-~~~~~~~~~~~~~~~
-
-Assume that at most -1/3 of the voting power of validators is byzantine.
-If a validator commits block ``B`` at round ``R``, it's because it saw
-+2/3 of precommits at round ``R``. This implies that 1/3+ of honest
-nodes are still locked at round ``R' > R``. These locked validators will
-remain locked until they see a PoLC at ``R' > R``, but this won't happen
-because 1/3+ are locked and honest, so at most -2/3 are available to
-vote for anything other than ``B``.
-
-Proof of Liveness
-~~~~~~~~~~~~~~~~~
-
-If 1/3+ honest validators are locked on two different blocks from
-different rounds, a proposers' ``PoLC-Round`` will eventually cause
-nodes locked from the earlier round to unlock. Eventually, the
-designated proposer will be one that is aware of a PoLC at the later
-round. Also, ``timeoutProposalR`` increments with round ``R``, while the
-size of a proposal are capped, so eventually the network is able to
-"fully gossip" the whole proposal (e.g. the block & PoLC).
-
-Proof of Fork Accountability
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Define the JSet (justification-vote-set) at height ``H`` of a validator
-``V1`` to be all the votes signed by the validator at ``H`` along with
-justification PoLC prevotes for each lock change. For example, if ``V1``
-signed the following precommits: ``Precommit(B1 @ round 0)``,
-``Precommit(<nil> @ round 1)``, ``Precommit(B2 @ round 4)`` (note that
-no precommits were signed for rounds 2 and 3, and that's ok),
-``Precommit(B1 @ round 0)`` must be justified by a PoLC at round 0, and
-``Precommit(B2 @ round 4)`` must be justified by a PoLC at round 4; but
-the precommit for ``<nil>`` at round 1 is not a lock-change by
-definition so the JSet for ``V1`` need not include any prevotes at round
-1, 2, or 3 (unless ``V1`` happened to have prevoted for those rounds).
-
-Further, define the JSet at height ``H`` of a set of validators ``VSet``
-to be the union of the JSets for each validator in ``VSet``. For a given
-commit by honest validators at round ``R`` for block ``B`` we can
-construct a JSet to justify the commit for ``B`` at ``R``. We say that a
-JSet *justifies* a commit at ``(H,R)`` if all the committers (validators
-in the commit-set) are each justified in the JSet with no duplicitous
-vote signatures (by the committers).
-
--  **Lemma**: When a fork is detected by the existence of two
-   conflicting `commits <./validators.html#commiting-a-block>`__,
-   the union of the JSets for both commits (if they can be compiled)
-   must include double-signing by at least 1/3+ of the validator set.
-   **Proof**: The commit cannot be at the same round, because that would
-   immediately imply double-signing by 1/3+. Take the union of the JSets
-   of both commits. If there is no double-signing by at least 1/3+ of
-   the validator set in the union, then no honest validator could have
-   precommitted any different block after the first commit. Yet, +2/3
-   did. Reductio ad absurdum.
-
-As a corollary, when there is a fork, an external process can determine
-the blame by requiring each validator to justify all of its round votes.
-Either we will find 1/3+ who cannot justify at least one of their votes,
-and/or, we will find 1/3+ who had double-signed.
-
-Alternative algorithm
-~~~~~~~~~~~~~~~~~~~~~
-
-Alternatively, we can take the JSet of a commit to be the "full commit".
-That is, if light clients and validators do not consider a block to be
-committed unless the JSet of the commit is also known, then we get the
-desirable property that if there ever is a fork (e.g. there are two
-conflicting "full commits"), then 1/3+ of the validators are immediately
-punishable for double-signing.
-
-There are many ways to ensure that the gossip network efficiently share
-the JSet of a commit. One solution is to add a new message type that
-tells peers that this node has (or does not have) a +2/3 majority for B
-(or ) at (H,R), and a bitarray of which votes contributed towards that
-majority. Peers can react by responding with appropriate votes.
-
-We will implement such an algorithm for the next iteration of the
-Tendermint consensus protocol.
-
-Other potential improvements include adding more data in votes such as
-the last known PoLC round that caused a lock change, and the last voted
-round/step (or, we may require that validators not skip any votes). This
-may make JSet verification/gossip logic easier to implement.
-
-Censorship Attacks
-~~~~~~~~~~~~~~~~~~
-
-Due to the definition of a block
-`commit <validators.html#commiting-a-block>`__, any 1/3+
-coalition of validators can halt the blockchain by not broadcasting
-their votes. Such a coalition can also censor particular transactions by
-rejecting blocks that include these transactions, though this would
-result in a significant proportion of block proposals to be rejected,
-which would slow down the rate of block commits of the blockchain,
-reducing its utility and value. The malicious coalition might also
-broadcast votes in a trickle so as to grind blockchain block commits to
-a near halt, or engage in any combination of these attacks.
-
-If a global active adversary were also involved, it can partition the
-network in such a way that it may appear that the wrong subset of
-validators were responsible for the slowdown. This is not just a
-limitation of Tendermint, but rather a limitation of all consensus
-protocols whose network is potentially controlled by an active
-adversary.
-
-Overcoming Forks and Censorship Attacks
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-For these types of attacks, a subset of the validators through external
-means should coordinate to sign a reorg-proposal that chooses a fork
-(and any evidence thereof) and the initial subset of validators with
-their signatures. Validators who sign such a reorg-proposal forego its
-collateral on all other forks. Clients should verify the signatures on
-the reorg-proposal, verify any evidence, and make a judgement or prompt
-the end-user for a decision. For example, a phone wallet app may prompt
-the user with a security warning, while a refrigerator may accept any
-reorg-proposal signed by +1/2 of the original validators.
-
-No non-synchronous Byzantine fault-tolerant algorithm can come to
-consensus when 1/3+ of validators are dishonest, yet a fork assumes that
-1/3+ of validators have already been dishonest by double-signing or
-lock-changing without justification. So, signing the reorg-proposal is a
-coordination problem that cannot be solved by any non-synchronous
-protocol (i.e. automatically, and without making assumptions about the
-reliability of the underlying network). It must be provided by means
-external to the weakly-synchronous Tendermint consensus algorithm. For
-now, we leave the problem of reorg-proposal coordination to human
-coordination via internet media. Validators must take care to ensure
-that there are no significant network partitions, to avoid situations
-where two conflicting reorg-proposals are signed.
-
-Assuming that the external coordination medium and protocol is robust,
-it follows that forks are less of a concern than `censorship
-attacks <#censorship-attacks>`__.

docs/specification/corruption.rst

@@ -1,70 +0,0 @@
-Corruption
-==========
-
-Important step
---------------
-
-Make sure you have a backup of the Tendermint data directory.
-
-Possible causes
----------------
-
-Remember that most corruption is caused by hardware issues:
-
-- RAID controllers with faulty / worn out battery backup, and an unexpected power loss
-- Hard disk drives with write-back cache enabled, and an unexpected power loss
-- Cheap SSDs with insufficient power-loss protection, and an unexpected power-loss
-- Defective RAM
-- Defective or overheating CPU(s)
-
-Other causes can be:
-
-- Database systems configured with fsync=off and an OS crash or power loss
-- Filesystems configured to use write barriers plus a storage layer that ignores write barriers. LVM is a particular culprit.
-- Tendermint bugs
-- Operating system bugs
-- Admin error
-  - directly modifying Tendermint data-directory contents
-
-(Source: https://wiki.postgresql.org/wiki/Corruption)
-
-WAL Corruption
---------------
-
-If consensus WAL is corrupted at the lastest height and you are trying to start
-Tendermint, replay will fail with panic.
-
-Recovering from data corruption can be hard and time-consuming. Here are two approaches you can take:
-
-1) Delete the WAL file and restart Tendermint. It will attempt to sync with other peers.
-2) Try to repair the WAL file manually:
-
-  1. Create a backup of the corrupted WAL file:
-
-  .. code:: bash
-
-      cp "$TMHOME/data/cs.wal/wal" > /tmp/corrupted_wal_backup
-
-  2. Use ./scripts/wal2json to create a human-readable version
-
-  .. code:: bash
-
-      ./scripts/wal2json/wal2json "$TMHOME/data/cs.wal/wal" > /tmp/corrupted_wal
-
-  3. Search for a "CORRUPTED MESSAGE" line.
-  4. By looking at the previous message and the message after the corrupted one
-       and looking at the logs, try to rebuild the message. If the consequent
-       messages are marked as corrupted too (this may happen if length header
-       got corrupted or some writes did not make it to the WAL ~ truncation),
-       then remove all the lines starting from the corrupted one and restart
-       Tendermint.
-
-  .. code:: bash
-
-      $EDITOR /tmp/corrupted_wal
-
-  5. After editing, convert this file back into binary form by running:
-
-  .. code:: bash
-
-      ./scripts/json2wal/json2wal /tmp/corrupted_wal > "$TMHOME/data/cs.wal/wal"

docs/specification/genesis.rst

@@ -1,71 +0,0 @@
-Genesis
-=======
-
-The genesis.json file in ``$TMHOME/config`` defines the initial TendermintCore
-state upon genesis of the blockchain (`see
-definition <https://github.com/tendermint/tendermint/blob/master/types/genesis.go>`__).
-
-Fields
-~~~~~~
-
--  ``genesis_time``: Official time of blockchain start.
--  ``chain_id``: ID of the blockchain. This must be unique for every
-   blockchain. If your testnet blockchains do not have unique chain IDs,
-   you will have a bad time.
--  ``validators``:
--  ``pub_key``: The first element specifies the pub\_key type. 1 ==
-   Ed25519. The second element are the pubkey bytes.
--  ``power``: The validator's voting power.
--  ``name``: Name of the validator (optional).
--  ``app_hash``: The expected application hash (as returned by the
-   ``ResponseInfo`` ABCI message) upon genesis. If the app's hash does not
-   match, Tendermint will panic.
--  ``app_state``: The application state (e.g. initial distribution of tokens).
-
-Sample genesis.json
-~~~~~~~~~~~~~~~~~~~
-
-.. code:: json
-
-    {
-      "genesis_time": "2016-02-05T06:02:31.526Z",
-      "chain_id": "chain-tTH4mi",
-      "validators": [
-        {
-          "pub_key": [
-            1,
-            "9BC5112CB9614D91CE423FA8744885126CD9D08D9FC9D1F42E552D662BAA411E"
-          ],
-          "power": 1,
-          "name": "mach1"
-        },
-        {
-          "pub_key": [
-            1,
-            "F46A5543D51F31660D9F59653B4F96061A740FF7433E0DC1ECBC30BE8494DE06"
-          ],
-          "power": 1,
-          "name": "mach2"
-        },
-        {
-          "pub_key": [
-            1,
-            "0E7B423C1635FD07C0FC3603B736D5D27953C1C6CA865BB9392CD79DE1A682BB"
-          ],
-          "power": 1,
-          "name": "mach3"
-        },
-        {
-          "pub_key": [
-            1,
-            "4F49237B9A32EB50682EDD83C48CE9CDB1D02A7CFDADCFF6EC8C1FAADB358879"
-          ],
-          "power": 1,
-          "name": "mach4"
-        }
-      ],
-      "app_hash": "15005165891224E721CB664D15CB972240F5703F",
-      "app_state": {
-        {"account": "Bob", "coins": 5000}
-      }
-    }

docs/specification/light-client-protocol.rst

@@ -1,33 +0,0 @@
-Light Client Protocol
-=====================
-
-Light clients are an important part of the complete blockchain system
-for most applications. Tendermint provides unique speed and security
-properties for light client applications.
-
-See our `lite package
-<https://godoc.org/github.com/tendermint/tendermint/lite>`__.
-
-Overview
---------
-
-The objective of the light client protocol is to get a
-`commit <./validators.html#committing-a-block>`__ for a recent
-`block hash <./block-structure.html#block-hash>`__ where the commit
-includes a majority of signatures from the last known validator set.
-From there, all the application state is verifiable with `merkle
-proofs <./merkle.html#iavl-tree>`__.
-
-Properties
-----------
-
--  You get the full collateralized security benefits of Tendermint; No
-   need to wait for confirmations.
--  You get the full speed benefits of Tendermint; transactions commit
-   instantly.
--  You can get the most recent version of the application state
-   non-interactively (without committing anything to the blockchain).
-   For example, this means that you can get the most recent value of a
-   name from the name-registry without worrying about fork censorship
-   attacks, without posting a commit and waiting for confirmations. It's
-   fast, secure, and free!

docs/specification/merkle.rst

@@ -1,88 +0,0 @@
-Merkle
-======
-
-For an overview of Merkle trees, see
-`wikipedia <https://en.wikipedia.org/wiki/Merkle_tree>`__.
-
-There are two types of Merkle trees used in Tendermint.
-
--  **IAVL+ Tree**: An immutable self-balancing binary
-   tree for persistent application state
--  **Simple Tree**: A simple compact binary tree for
-   a static list of items
-
-IAVL+ Tree
-----------
-
-The purpose of this data structure is to provide persistent storage for
-key-value pairs (e.g. account state, name-registrar data, and
-per-contract data) such that a deterministic merkle root hash can be
-computed. The tree is balanced using a variant of the `AVL
-algorithm <http://en.wikipedia.org/wiki/AVL_tree>`__ so all operations
-are O(log(n)).
-
-Nodes of this tree are immutable and indexed by its hash. Thus any node
-serves as an immutable snapshot which lets us stage uncommitted
-transactions from the mempool cheaply, and we can instantly roll back to
-the last committed state to process transactions of a newly committed
-block (which may not be the same set of transactions as those from the
-mempool).
-
-In an AVL tree, the heights of the two child subtrees of any node differ
-by at most one. Whenever this condition is violated upon an update, the
-tree is rebalanced by creating O(log(n)) new nodes that point to
-unmodified nodes of the old tree. In the original AVL algorithm, inner
-nodes can also hold key-value pairs. The AVL+ algorithm (note the plus)
-modifies the AVL algorithm to keep all values on leaf nodes, while only
-using branch-nodes to store keys. This simplifies the algorithm while
-minimizing the size of merkle proofs
-
-In Ethereum, the analog is the `Patricia
-trie <http://en.wikipedia.org/wiki/Radix_tree>`__. There are tradeoffs.
-Keys do not need to be hashed prior to insertion in IAVL+ trees, so this
-provides faster iteration in the key space which may benefit some
-applications. The logic is simpler to implement, requiring only two
-types of nodes -- inner nodes and leaf nodes. The IAVL+ tree is a binary
-tree, so merkle proofs are much shorter than the base 16 Patricia trie.
-On the other hand, while IAVL+ trees provide a deterministic merkle root
-hash, it depends on the order of updates. In practice this shouldn't be
-a problem, since you can efficiently encode the tree structure when
-serializing the tree contents.
-
-Simple Tree
------------
-
-For merkelizing smaller static lists, use the Simple Tree. The
-transactions and validation signatures of a block are hashed using this
-simple merkle tree logic.
-
-If the number of items is not a power of two, the tree will not be full
-and some leaf nodes will be at different levels. Simple Tree tries to
-keep both sides of the tree the same size, but the left side may be one
-greater.
-
-::
-
-        Simple Tree with 6 items           Simple Tree with 7 items 
-                                                             
-                   *                                  *             
-                  / \                                / \            
-                /     \                            /     \          
-              /         \                        /         \        
-            /             \                    /             \      
-           *               *                  *               *     
-          / \             / \                / \             / \    
-         /   \           /   \              /   \           /   \   
-        /     \         /     \            /     \         /     \  
-       *       h2      *       h5         *       *       *       h6
-      / \             / \                / \     / \     / \        
-     h0  h1          h3  h4             h0  h1  h2  h3  h4  h5      
-
-Simple Tree with Dictionaries
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The Simple Tree is used to merkelize a list of items, so to merkelize a
-(short) dictionary of key-value pairs, encode the dictionary as an
-ordered list of ``KVPair`` structs. The block hash is such a hash
-derived from all the fields of the block ``Header``. The state hash is
-similarly derived.

docs/specification/new-spec/README.md

@@ -1 +0,0 @@
-Spec moved to [docs/spec](https://github.com/tendermint/tendermint/tree/master/docs/spec).

docs/specification/wire-protocol.rst

@@ -1,172 +0,0 @@
-Wire Protocol
-=============
-
-The `Tendermint wire protocol <https://github.com/tendermint/go-wire>`__
-encodes data in `c-style binary <#binary>`__ and `JSON <#json>`__ form.
-
-Supported types
----------------
-
--  Primitive types
--  ``uint8`` (aka ``byte``), ``uint16``, ``uint32``, ``uint64``
--  ``int8``, ``int16``, ``int32``, ``int64``
--  ``uint``, ``int``: variable length (un)signed integers
--  ``string``, ``[]byte``
--  ``time``
--  Derived types
--  structs
--  var-length arrays of a particular type
--  fixed-length arrays of a particular type
--  interfaces: registered union types preceded by a ``type byte``
--  pointers
-
-Binary
-------
-
-**Fixed-length primitive types** are encoded with 1,2,3, or 4 big-endian
-bytes. - ``uint8`` (aka ``byte``), ``uint16``, ``uint32``, ``uint64``:
-takes 1,2,3, and 4 bytes respectively - ``int8``, ``int16``, ``int32``,
-``int64``: takes 1,2,3, and 4 bytes respectively - ``time``: ``int64``
-representation of nanoseconds since epoch
-
-**Variable-length integers** are encoded with a single leading byte
-representing the length of the following big-endian bytes. For signed
-negative integers, the most significant bit of the leading byte is a 1.
-
--  ``uint``: 1-byte length prefixed variable-size (0 ~ 255 bytes)
-   unsigned integers
--  ``int``: 1-byte length prefixed variable-size (0 ~ 127 bytes) signed
-   integers
-
-NOTE: While the number 0 (zero) is encoded with a single byte ``x00``,
-the number 1 (one) takes two bytes to represent: ``x0101``. This isn't
-the most efficient representation, but the rules are easier to remember.
-
-+---------------+----------------+----------------+
-| number        | binary         | binary ``int`` |
-|               | ``uint``       |                |
-+===============+================+================+
-| 0             | ``x00``        | ``x00``        |
-+---------------+----------------+----------------+
-| 1             | ``x0101``      | ``x0101``      |
-+---------------+----------------+----------------+
-| 2             | ``x0102``      | ``x0102``      |
-+---------------+----------------+----------------+
-| 256           | ``x020100``    | ``x020100``    |
-+---------------+----------------+----------------+
-| 2^(127\ *8)-1 | ``x800100...`` | overflow       |
-| \|            |                |                |
-| ``x7FFFFF...` |                |                |
-| `             |                |                |
-| \|            |                |                |
-| ``x7FFFFF...` |                |                |
-| `             |                |                |
-| \| \|         |                |                |
-| 2^(127*\ 8)   |                |                |
-+---------------+----------------+----------------+
-| 2^(255\*8)-1  |
-| \|            |
-| ``xFFFFFF...` |
-| `             |
-| \| overflow   |
-| \| \| -1 \|   |
-| n/a \|        |
-| ``x8101`` \|  |
-| \| -2 \| n/a  |
-| \| ``x8102``  |
-| \| \| -256 \| |
-| n/a \|        |
-| ``x820100``   |
-| \|            |
-+---------------+----------------+----------------+
-
-**Structures** are encoded by encoding the field values in order of
-declaration.
-
-.. code:: go
-
-    type Foo struct {
-        MyString string
-        MyUint32 uint32
-    }
-    var foo = Foo{"626172", math.MaxUint32}
-
-    /* The binary representation of foo:
-     0103626172FFFFFFFF
-     0103:               `int` encoded length of string, here 3
-         626172:         3 bytes of string "bar"
-               FFFFFFFF: 4 bytes of uint32 MaxUint32
-    */
-
-**Variable-length arrays** are encoded with a leading ``int`` denoting
-the length of the array followed by the binary representation of the
-items. **Fixed-length arrays** are similar but aren't preceded by the
-leading ``int``.
-
-.. code:: go
-
-    foos := []Foo{foo, foo}
-
-    /* The binary representation of foos:
-     01020103626172FFFFFFFF0103626172FFFFFFFF
-     0102:                                     `int` encoded length of array, here 2
-         0103626172FFFFFFFF:                   the first `foo`
-                           0103626172FFFFFFFF: the second `foo`
-    */
-
-    foos := [2]Foo{foo, foo} // fixed-length array
-
-    /* The binary representation of foos:
-     0103626172FFFFFFFF0103626172FFFFFFFF
-     0103626172FFFFFFFF:                   the first `foo`
-                       0103626172FFFFFFFF: the second `foo`
-    */
-
-**Interfaces** can represent one of any number of concrete types. The
-concrete types of an interface must first be declared with their
-corresponding ``type byte``. An interface is then encoded with the
-leading ``type byte``, then the binary encoding of the underlying
-concrete type.
-
-NOTE: The byte ``x00`` is reserved for the ``nil`` interface value and
-``nil`` pointer values.
-
-.. code:: go
-
-    type Animal interface{}
-    type Dog uint32
-    type Cat string
-
-    RegisterInterface(
-        struct{ Animal }{},          // Convenience for referencing the 'Animal' interface
-        ConcreteType{Dog(0),  0x01}, // Register the byte 0x01 to denote a Dog
-        ConcreteType{Cat(""), 0x02}, // Register the byte 0x02 to denote a Cat
-    )
-
-    var animal Animal = Dog(02)
-
-    /* The binary representation of animal:
-     010102
-     01:     the type byte for a `Dog`
-       0102: the bytes of Dog(02)
-    */
-
-**Pointers** are encoded with a single leading byte ``x00`` for ``nil``
-pointers, otherwise encoded with a leading byte ``x01`` followed by the
-binary encoding of the value pointed to.
-
-NOTE: It's easy to convert pointer types into interface types, since the
-``type byte`` ``x00`` is always ``nil``.
-
-JSON
-----
-
-The JSON codec is compatible with the ```binary`` <#binary>`__ codec,
-and is fairly intuitive if you're already familiar with golang's JSON
-encoding. Some quirks are noted below:
-
--  variable-length and fixed-length bytes are encoded as uppercase
-   hexadecimal strings
--  interface values are encoded as an array of two items:
-   ``[type_byte, concrete_value]``
--  times are encoded as rfc2822 strings

docs/tendermint-core/block-structure.md

@@ -0,0 +1,206 @@
+# Block Structure
+
+The tendermint consensus engine records all agreements by a
+supermajority of nodes into a blockchain, which is replicated among all
+nodes. This blockchain is accessible via various rpc endpoints, mainly
+`/block?height=` to get the full block, as well as
+`/blockchain?minHeight=_&maxHeight=_` to get a list of headers. But what
+exactly is stored in these blocks?
+
+## Block
+
+A
+[Block](https://godoc.org/github.com/tendermint/tendermint/types#Block)
+contains:
+
+-   a [Header](#header) contains merkle hashes for various chain states
+-   the
+    [Data](https://godoc.org/github.com/tendermint/tendermint/types#Data)
+    is all transactions which are to be processed
+-   the [LastCommit](#commit) > 2/3 signatures for the last block
+
+The signatures returned along with block `H` are those validating block
+`H-1`. This can be a little confusing, but we must also consider that
+the `Header` also contains the `LastCommitHash`. It would be impossible
+for a Header to include the commits that sign it, as it would cause an
+infinite loop here. But when we get block `H`, we find
+`Header.LastCommitHash`, which must match the hash of `LastCommit`.
+
+## Header
+
+The
+[Header](https://godoc.org/github.com/tendermint/tendermint/types#Header)
+contains lots of information (follow link for up-to-date info). Notably,
+it maintains the `Height`, the `LastBlockID` (to make it a chain), and
+hashes of the data, the app state, and the validator set. This is
+important as the only item that is signed by the validators is the
+`Header`, and all other data must be validated against one of the merkle
+hashes in the `Header`.
+
+The `DataHash` can provide a nice check on the
+[Data](https://godoc.org/github.com/tendermint/tendermint/types#Data)
+returned in this same block. If you are subscribed to new blocks, via
+tendermint RPC, in order to display or process the new transactions you
+should at least validate that the `DataHash` is valid. If it is
+important to verify autheniticity, you must wait for the `LastCommit`
+from the next block to make sure the block header (including `DataHash`)
+was properly signed.
+
+The `ValidatorHash` contains a hash of the current
+[Validators](https://godoc.org/github.com/tendermint/tendermint/types#Validator).
+Tracking all changes in the validator set is complex, but a client can
+quickly compare this hash with the [hash of the currently known
+validators](https://godoc.org/github.com/tendermint/tendermint/types#ValidatorSet.Hash)
+to see if there have been changes.
+
+The `AppHash` serves as the basis for validating any merkle proofs that
+come from the ABCI application. It represents the state of the actual
+application, rather that the state of the blockchain itself. This means
+it's necessary in order to perform any business logic, such as verifying
+an account balance.
+
+**Note** After the transactions are committed to a block, they still
+need to be processed in a separate step, which happens between the
+blocks. If you find a given transaction in the block at height `H`, the
+effects of running that transaction will be first visible in the
+`AppHash` from the block header at height `H+1`.
+
+Like the `LastCommit` issue, this is a requirement of the immutability
+of the block chain, as the application only applies transactions *after*
+they are commited to the chain.
+
+## Commit
+
+The
+[Commit](https://godoc.org/github.com/tendermint/tendermint/types#Commit)
+contains a set of
+[Votes](https://godoc.org/github.com/tendermint/tendermint/types#Vote)
+that were made by the validator set to reach consensus on this block.
+This is the key to the security in any PoS system, and actually no data
+that cannot be traced back to a block header with a valid set of Votes
+can be trusted. Thus, getting the Commit data and verifying the votes is
+extremely important.
+
+As mentioned above, in order to find the `precommit votes` for block
+header `H`, we need to query block `H+1`. Then we need to check the
+votes, make sure they really are for that block, and properly formatted.
+Much of this code is implemented in Go in the
+[light-client](https://github.com/tendermint/light-client) package. If
+you look at the code, you will notice that we need to provide the
+`chainID` of the blockchain in order to properly calculate the votes.
+This is to protect anyone from swapping votes between chains to fake (or
+frame) a validator. Also note that this `chainID` is in the
+`genesis.json` from *Tendermint*, not the `genesis.json` from the
+basecoin app ([that is a different
+chainID...](https://github.com/cosmos/cosmos-sdk/issues/32)).
+
+Once we have those votes, and we calculated the proper [sign
+bytes](https://godoc.org/github.com/tendermint/tendermint/types#Vote.WriteSignBytes)
+using the chainID and a [nice helper
+function](https://godoc.org/github.com/tendermint/tendermint/types#SignBytes),
+we can verify them. The light client is responsible for maintaining a
+set of validators that we trust. Each vote only stores the validators
+`Address`, as well as the `Signature`. Assuming we have a local copy of
+the trusted validator set, we can look up the `Public Key` of the
+validator given its `Address`, then verify that the `Signature` matches
+the `SignBytes` and `Public Key`. Then we sum up the total voting power
+of all validators, whose votes fulfilled all these stringent
+requirements. If the total number of voting power for a single block is
+greater than 2/3 of all voting power, then we can finally trust the
+block header, the AppHash, and the proof we got from the ABCI
+application.
+
+### Vote Sign Bytes
+
+The `sign-bytes` of a vote is produced by taking a
+[stable-json](https://github.com/substack/json-stable-stringify)-like
+deterministic JSON [wire](./wire-protocol.html) encoding of the vote
+(excluding the `Signature` field), and wrapping it with
+`{"chain_id":"my_chain","vote":...}`.
+
+For example, a precommit vote might have the following `sign-bytes`:
+
+```
+{"chain_id":"my_chain","vote":{"block_hash":"611801F57B4CE378DF1A3FFF1216656E89209A99","block_parts_header":{"hash":"B46697379DBE0774CC2C3B656083F07CA7E0F9CE","total":123},"height":1234,"round":1,"type":2}}
+```
+
+## Block Hash
+
+The [block
+hash](https://godoc.org/github.com/tendermint/tendermint/types#Block.Hash)
+is the [Simple Tree hash](./merkle.html#simple-tree-with-dictionaries)
+of the fields of the block `Header` encoded as a list of `KVPair`s.
+
+## Transaction
+
+A transaction is any sequence of bytes. It is up to your ABCI
+application to accept or reject transactions.
+
+## BlockID
+
+Many of these data structures refer to the
+[BlockID](https://godoc.org/github.com/tendermint/tendermint/types#BlockID),
+which is the `BlockHash` (hash of the block header, also referred to by
+the next block) along with the `PartSetHeader`. The `PartSetHeader` is
+explained below and is used internally to orchestrate the p2p
+propogation. For clients, it is basically opaque bytes, but they must
+match for all votes.
+
+## PartSetHeader
+
+The
+[PartSetHeader](https://godoc.org/github.com/tendermint/tendermint/types#PartSetHeader)
+contains the total number of pieces in a
+[PartSet](https://godoc.org/github.com/tendermint/tendermint/types#PartSet),
+and the Merkle root hash of those pieces.
+
+## PartSet
+
+PartSet is used to split a byteslice of data into parts (pieces) for
+transmission. By splitting data into smaller parts and computing a
+Merkle root hash on the list, you can verify that a part is legitimately
+part of the complete data, and the part can be forwarded to other peers
+before all the parts are known. In short, it's a fast way to securely
+propagate a large chunk of data (like a block) over a gossip network.
+
+PartSet was inspired by the LibSwift project.
+
+Usage:
+
+```
+data := RandBytes(2 << 20) // Something large
+
+partSet := NewPartSetFromData(data)
+partSet.Total()     // Total number of 4KB parts
+partSet.Count()     // Equal to the Total, since we already have all the parts
+partSet.Hash()      // The Merkle root hash
+partSet.BitArray()  // A BitArray of partSet.Total() 1's
+
+header := partSet.Header() // Send this to the peer
+header.Total        // Total number of parts
+header.Hash         // The merkle root hash
+
+// Now we'll reconstruct the data from the parts
+partSet2 := NewPartSetFromHeader(header)
+partSet2.Total()    // Same total as partSet.Total()
+partSet2.Count()    // Zero, since this PartSet doesn't have any parts yet.
+partSet2.Hash()     // Same hash as in partSet.Hash()
+partSet2.BitArray() // A BitArray of partSet.Total() 0's
+
+// In a gossip network the parts would arrive in arbitrary order, perhaps
+// in response to explicit requests for parts, or optimistically in response
+// to the receiving peer's partSet.BitArray().
+for !partSet2.IsComplete() {
+    part := receivePartFromGossipNetwork()
+    added, err := partSet2.AddPart(part)
+    if err != nil {
+    // A wrong part,
+        // the merkle trail does not hash to partSet2.Hash()
+    } else if !added {
+        // A duplicate part already received
+    }
+}
+
+data2, _ := ioutil.ReadAll(partSet2.GetReader())
+bytes.Equal(data, data2) // true
+```