Merge pull request #2076 from tendermint/hotfix/0.22.7

Hotfix/0.22.7

.circleci/config.yml

@@ -16,7 +16,7 @@ jobs:
       - checkout
       - restore_cache:
           keys:
-            - v2-pkg-cache
+            - v3-pkg-cache
       - run:
           name: tools
           command: |
@@ -38,11 +38,11 @@ jobs:
             - bin
             - profiles
       - save_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
           paths:
             - /go/pkg
       - save_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
           paths:
             - /go/src/github.com/tendermint/tendermint
 
@@ -52,9 +52,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: slate docs
           command: |
@@ -68,9 +68,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: metalinter
           command: |
@@ -84,9 +84,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run abci apps tests
           command: |
@@ -101,9 +101,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run abci-cli tests
           command: |
@@ -116,9 +116,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils
       - run:
           name: Run tests
@@ -131,14 +131,14 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run: mkdir -p /tmp/logs
       - run:
           name: Run tests
           command: |
-            for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
+            for pkg in $(go list github.com/tendermint/tendermint/... | circleci tests split --split-by=timings); do
               id=$(basename "$pkg")
 
               GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
@@ -156,9 +156,9 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Run tests
           command: bash test/persist/test_failure_indices.sh
@@ -181,7 +181,7 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - restore_cache:
-          key: v2-tree-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-tree-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: gather
           command: |

.gitignore

@@ -16,7 +16,6 @@ coverage.txt
 docs/_build
 *.log
 abci-cli
-abci/types/types.pb.go
 docs/node_modules/
 
 scripts/wal2json/wal2json
@@ -27,3 +26,10 @@ scripts/cutWALUntil/cutWALUntil
 
 libs/pubsub/query/fuzz_test/output
 shunit2
+
+*/vendor
+*/.glide
+.terraform
+terraform.tfstate
+terraform.tfstate.backup
+terraform.tfstate.d

CHANGELOG.md

@@ -1,7 +1,70 @@
 # Changelog
 
+## 0.22.7
+
+*July 26th, 2018*
+
+BUG FIXES
+
+- [consensus, blockchain] Register the Evidence interface so it can be
+  marshalled/unmarshalled by the blockchain and consensus reactors
+
+## 0.22.6
+
+*July 24th, 2018*
+
+BUG FIXES
+
+- [rpc] Fix `/blockchain` endpoint
+    - (#2049) Fix OOM attack by returning error on negative input
+    - Fix result length to have max 20 (instead of 21) block metas
+- [rpc] Validate height is non-negative in `/abci_query`
+- [consensus] (#2050) Include evidence in proposal block parts (previously evidence was
+  not being included in blocks!)
+- [p2p] (#2046) Close rejected inbound connections so file descriptor doesn't
+  leak
+- [Gopkg] (#2053) Fix versions in the toml
+
+## 0.22.5
+
+*July 23th, 2018*
+
+BREAKING CHANGES:
+- [crypto] Refactor `tendermint/crypto` into many subpackages
+- [libs/common] remove exponentially distributed random numbers
+
+IMPROVEMENTS:
+- [abci, libs/common] Generated gogoproto static marshaller methods
+- [config] Increase default send/recv rates to 5 mB/s
+- [p2p] allow persistent peers to be private
+
+BUG FIXES
+- [mempool] fixed a race condition when `create_empty_blocks=false` where a
+  transaction is published at an old height.
+- [p2p] dial external IP setup by `persistent_peers`, not internal NAT IP
+- [rpc] make `/status` RPC endpoint resistant to consensus halt
+
+## 0.22.4
+
+*July 14th, 2018*
+
+BREAKING CHANGES:
+- [genesis] removed deprecated `app_options` field.
+- [types] Genesis.AppStateJSON -> Genesis.AppState
+
+FEATURES:
+- [tools] Merged in from github.com/tendermint/tools
+
+BUG FIXES:
+- [tools/tm-bench] Various fixes
+- [consensus] Wait for WAL to stop on shutdown
+- [abci] Fix #1891, pending requests cannot hang when abci server dies.
+  Previously a crash in BeginBlock could leave tendermint in broken state.
+
 ## 0.22.3
 
+*July 10th, 2018*
+
 IMPROVEMENTS
 - Update dependencies
     * pin all values in Gopkg.toml to version or commit
@@ -39,6 +102,7 @@ BUG FIXES
   already in the validator set.
 * [consensus] Shut down WAL properly.
 
+
 ## 0.22.0
 
 *July 2nd, 2018*

Gopkg.lock

@@ -11,7 +11,7 @@
   branch = "master"
   name = "github.com/btcsuite/btcd"
   packages = ["btcec"]
-  revision = "fdfc19097e7ac6b57035062056f5b7b4638b8898"
+  revision = "f673a4b563b57b9a95832545c878669a7fa801d9"
 
 [[projects]]
   name = "github.com/btcsuite/btcutil"
@@ -80,8 +80,8 @@
     "sortkeys",
     "types"
   ]
-  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
-  version = "v1.0.0"
+  revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
+  version = "v1.1.1"
 
 [[projects]]
   name = "github.com/golang/protobuf"
@@ -92,8 +92,8 @@
     "ptypes/duration",
     "ptypes/timestamp"
   ]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
+  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
@@ -157,7 +157,7 @@
   branch = "master"
   name = "github.com/mitchellh/mapstructure"
   packages = ["."]
-  revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b"
+  revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac"
 
 [[projects]]
   name = "github.com/pelletier/go-toml"
@@ -189,7 +189,7 @@
   branch = "master"
   name = "github.com/prometheus/client_model"
   packages = ["go"]
-  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
+  revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
 
 [[projects]]
   branch = "master"
@@ -319,7 +319,7 @@
     "ripemd160",
     "salsa20/salsa"
   ]
-  revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
+  revision = "a2144134853fc9a27a7b1e3eb4f19f1a76df13c9"
 
 [[projects]]
   name = "golang.org/x/net"
@@ -342,7 +342,7 @@
     "cpu",
     "unix"
   ]
-  revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2"
+  revision = "ac767d655b305d4e9612f5f6e33120b9176c4ad4"
 
 [[projects]]
   name = "golang.org/x/text"
@@ -382,9 +382,11 @@
     "credentials",
     "encoding",
     "encoding/proto",
-    "grpclb/grpc_lb_v1/messages",
     "grpclog",
     "internal",
+    "internal/backoff",
+    "internal/channelz",
+    "internal/grpcrand",
     "keepalive",
     "metadata",
     "naming",
@@ -397,8 +399,8 @@
     "tap",
     "transport"
   ]
-  revision = "d11072e7ca9811b1100b80ca0269ac831f06d024"
-  version = "v1.11.3"
+  revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
+  version = "v1.13.0"
 
 [[projects]]
   name = "gopkg.in/yaml.v2"
@@ -409,6 +411,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "b0718135d5ade0a75c6b8fe703f70eb9d8064ba871ec31abd9ace3c4ab944100"
+  inputs-digest = "8e519c3716c259c6ecdb052889dd3602539fd98a3650dfbf50a4023e087a5d53"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -35,11 +35,11 @@
 
 [[constraint]]
   name = "github.com/gogo/protobuf"
-  version = "=1.0.0"
+  version = "=1.1.1"
 
 [[constraint]]
   name = "github.com/golang/protobuf"
-  version = "=1.0.0"
+  version = "=1.1.0"
 
 [[constraint]]
   name = "github.com/gorilla/websocket"
@@ -63,11 +63,11 @@
 
 [[constraint]]
   name = "github.com/tendermint/go-amino"
-  version = "=0.10.1"
+  version = "=v0.10.1"
 
 [[constraint]]
   name = "google.golang.org/grpc"
-  version = "=1.11.3"
+  version = "=1.13.0"
 
 [[constraint]]
   name = "github.com/fortytw2/leaktest"

Makefile

@@ -5,7 +5,7 @@ GOTOOLS = \
 	github.com/gogo/protobuf/protoc-gen-gogo \
 	github.com/gogo/protobuf/gogoproto \
 	github.com/square/certstrap
-PACKAGES=$(shell go list ./... | grep -v '/vendor/')
+PACKAGES=$(shell go list ./...)
 INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf
 BUILD_TAGS?=tendermint
 BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
@@ -28,16 +28,22 @@ install:
 	CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint
 
 ########################################
-### Build ABCI
+### Protobuf
 
-protoc_abci:
+protoc_all: protoc_libs protoc_abci protoc_grpc
+
+%.pb.go: %.proto
 	## If you get the following error,
 	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
 	## See https://stackoverflow.com/a/25518702
-	protoc $(INCLUDE) --gogo_out=plugins=grpc:. abci/types/*.proto
+	protoc $(INCLUDE) $< --gogo_out=plugins=grpc:.
 	@echo "--> adding nolint declarations to protobuf generated files"
-	@awk '/package abci/types/ { print "//nolint: gas"; print; next }1' abci/types/types.pb.go > abci/types/types.pb.go.new
-	@mv abci/types/types.pb.go.new abci/types/types.pb.go
+	@awk -i inplace '/^\s*package \w+/ { print "//nolint" }1' $@
+
+########################################
+### Build ABCI
+
+protoc_abci: abci/types/types.pb.go
 
 build_abci:
 	@go build -i ./abci/cmd/...
@@ -108,14 +114,7 @@ get_deps_bin_size:
 ########################################
 ### Libs
 
-protoc_libs:
-	## If you get the following error,
-	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
-	## See https://stackoverflow.com/a/25518702
-	protoc $(INCLUDE) --go_out=plugins=grpc:. libs/common/*.proto
-	@echo "--> adding nolint declarations to protobuf generated files"
-	@awk '/package libs/common/ { print "//nolint: gas"; print; next }1' libs/common/types.pb.go > libs/common/types.pb.go.new
-	@mv libs/common/types.pb.go.new libs/common/types.pb.go
+protoc_libs: libs/common/types.pb.go
 
 gen_certs: clean_certs
 	## Generating certificates for TLS testing...
@@ -130,12 +129,14 @@ clean_certs:
 	rm -f db/remotedb/::.crt db/remotedb/::.key
 
 test_libs: gen_certs
-	GOCACHE=off go test -tags gcc $(shell go list ./... | grep -v vendor)
+	GOCACHE=off go test -tags gcc $(PACKAGES)
 	make clean_certs
 
 grpc_dbserver:
 	protoc -I db/remotedb/proto/ db/remotedb/proto/defs.proto --go_out=plugins=grpc:db/remotedb/proto
 
+protoc_grpc: rpc/grpc/types.pb.go
+
 ########################################
 ### Testing
 
@@ -307,4 +308,4 @@ build-slate:
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: check build build_race build_abci dist install install_abci check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate
+.PHONY: check build build_race build_abci dist install install_abci check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all

README.md

@@ -50,13 +50,13 @@ Go version | Go1.9 or higher
 
 ## Install
 
-See the [install instructions](/docs/install.md)
+See the [install instructions](/docs/introduction/install.md)
 
 ## Quick Start
 
 - [Single node](/docs/using-tendermint.md)
 - [Local cluster using docker-compose](/networks/local)
-- [Remote cluster using terraform and ansible](/docs/terraform-and-ansible.md)
+- [Remote cluster using terraform and ansible](/docs/networks/terraform-and-ansible.md)
 - [Join the public testnet](https://cosmos.network/testnet)
 
 ## Resources

abci/client/socket_client.go

@@ -357,6 +357,13 @@ func (cli *socketClient) queueRequest(req *types.Request) *ReqRes {
 }
 
 func (cli *socketClient) flushQueue() {
+	// mark all in-flight messages as resolved (they will get cli.Error())
+	for req := cli.reqSent.Front(); req != nil; req = req.Next() {
+		reqres := req.Value.(*ReqRes)
+		reqres.Done()
+	}
+
+	// mark all queued messages as resolved
 LOOP:
 	for {
 		select {

abci/client/socket_client_test.go

@@ -2,10 +2,17 @@ package abcicli_test
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/abci/client"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/server"
+	"github.com/tendermint/tendermint/abci/types"
+	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
 func TestSocketClientStopForErrorDeadlock(t *testing.T) {
@@ -26,3 +33,89 @@ func TestSocketClientStopForErrorDeadlock(t *testing.T) {
 		t.Fatalf("Test took too long, potential deadlock still exists")
 	}
 }
+
+func TestProperSyncCalls(t *testing.T) {
+	app := slowApp{}
+
+	s, c := setupClientServer(t, app)
+	defer s.Stop()
+	defer c.Stop()
+
+	resp := make(chan error, 1)
+	go func() {
+		// This is BeginBlockSync unrolled....
+		reqres := c.BeginBlockAsync(types.RequestBeginBlock{})
+		c.FlushSync()
+		res := reqres.Response.GetBeginBlock()
+		require.NotNil(t, res)
+		resp <- c.Error()
+	}()
+
+	select {
+	case <-time.After(time.Second):
+		require.Fail(t, "No response arrived")
+	case err, ok := <-resp:
+		require.True(t, ok, "Must not close channel")
+		assert.NoError(t, err, "This should return success")
+	}
+}
+
+func TestHangingSyncCalls(t *testing.T) {
+	app := slowApp{}
+
+	s, c := setupClientServer(t, app)
+	defer s.Stop()
+	defer c.Stop()
+
+	resp := make(chan error, 1)
+	go func() {
+		// Start BeginBlock and flush it
+		reqres := c.BeginBlockAsync(types.RequestBeginBlock{})
+		flush := c.FlushAsync()
+		// wait 20 ms for all events to travel socket, but
+		// no response yet from server
+		time.Sleep(20 * time.Millisecond)
+		// kill the server, so the connections break
+		s.Stop()
+
+		// wait for the response from BeginBlock
+		reqres.Wait()
+		flush.Wait()
+		resp <- c.Error()
+	}()
+
+	select {
+	case <-time.After(time.Second):
+		require.Fail(t, "No response arrived")
+	case err, ok := <-resp:
+		require.True(t, ok, "Must not close channel")
+		assert.Error(t, err, "We should get EOF error")
+	}
+}
+
+func setupClientServer(t *testing.T, app types.Application) (
+	cmn.Service, abcicli.Client) {
+	// some port between 20k and 30k
+	port := 20000 + cmn.RandInt32()%10000
+	addr := fmt.Sprintf("localhost:%d", port)
+
+	s, err := server.NewServer(addr, "socket", app)
+	require.NoError(t, err)
+	err = s.Start()
+	require.NoError(t, err)
+
+	c := abcicli.NewSocketClient(addr, true)
+	err = c.Start()
+	require.NoError(t, err)
+
+	return s, c
+}
+
+type slowApp struct {
+	types.BaseApplication
+}
+
+func (slowApp) BeginBlock(req types.RequestBeginBlock) types.ResponseBeginBlock {
+	time.Sleep(200 * time.Millisecond)
+	return types.ResponseBeginBlock{}
+}

abci/types/types.proto

@@ -4,12 +4,21 @@ package types;
 // For more information on gogo.proto, see:
 // https://github.com/gogo/protobuf/blob/master/extensions.md
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-import "github.com/tendermint/tmlibs/common/types.proto";
+import "github.com/tendermint/tendermint/libs/common/types.proto";
 
 // This file is copied from http://github.com/tendermint/abci
 // NOTE: When using custom types, mind the warnings.
 // https://github.com/gogo/protobuf/blob/master/custom_types.md#warnings-and-issues
 
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.goproto_registration) = true;
+// Generate tests
+option (gogoproto.populate_all) = true;
+option (gogoproto.equal_all) = true;
+option (gogoproto.testgen_all) = true;
+
 //----------------------------------------
 // Request types
 

benchmarks/codec_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/tendermint/go-amino"
 
 	proto "github.com/tendermint/tendermint/benchmarks/proto"
-	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	"github.com/tendermint/tendermint/p2p"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 )
@@ -16,7 +16,7 @@ func BenchmarkEncodeStatusWire(b *testing.B) {
 	b.StopTimer()
 	cdc := amino.NewCodec()
 	ctypes.RegisterAmino(cdc)
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	status := &ctypes.ResultStatus{
 		NodeInfo: p2p.NodeInfo{
 			ID:         nodeKey.ID(),
@@ -52,7 +52,7 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) {
 	b.StopTimer()
 	cdc := amino.NewCodec()
 	ctypes.RegisterAmino(cdc)
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	nodeInfo := p2p.NodeInfo{
 		ID:         nodeKey.ID(),
 		Moniker:    "SOMENAME",
@@ -77,7 +77,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 	b.StopTimer()
 	cdc := amino.NewCodec()
 	ctypes.RegisterAmino(cdc)
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	nodeInfo := p2p.NodeInfo{
 		ID:         nodeKey.ID(),
 		Moniker:    "SOMENAME",
@@ -98,7 +98,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoProto(b *testing.B) {
 	b.StopTimer()
-	nodeKey := p2p.NodeKey{PrivKey: crypto.GenPrivKeyEd25519()}
+	nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()}
 	nodeID := string(nodeKey.ID())
 	someName := "SOMENAME"
 	someAddr := "SOMEADDR"

blockchain/pool_test.go

@@ -1,7 +1,6 @@
 package blockchain
 
 import (
-	"math/rand"
 	"testing"
 	"time"
 
@@ -25,7 +24,7 @@ func makePeers(numPeers int, minHeight, maxHeight int64) map[p2p.ID]testPeer {
 	peers := make(map[p2p.ID]testPeer, numPeers)
 	for i := 0; i < numPeers; i++ {
 		peerID := p2p.ID(cmn.RandStr(12))
-		height := minHeight + rand.Int63n(maxHeight-minHeight)
+		height := minHeight + cmn.RandInt63n(maxHeight-minHeight)
 		peers[peerID] = testPeer{peerID, height}
 	}
 	return peers
@@ -80,7 +79,7 @@ func TestBasic(t *testing.T) {
 			}
 			// Request desired, pretend like we got the block immediately.
 			go func() {
-				block := &types.Block{Header: &types.Header{Height: request.Height}}
+				block := &types.Block{Header: types.Header{Height: request.Height}}
 				pool.AddBlock(request.PeerID, block, 123)
 				t.Logf("Added block from peer %v (height: %v)", request.PeerID, request.Height)
 			}()

blockchain/reactor_test.go

@@ -156,7 +156,7 @@ func makeTxs(height int64) (txs []types.Tx) {
 }
 
 func makeBlock(height int64, state sm.State) *types.Block {
-	block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit))
+	block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit), nil)
 	return block
 }
 
@@ -206,3 +206,4 @@ func (tp *bcrTestPeer) IsPersistent() bool                   { return true }
 func (tp *bcrTestPeer) Get(s string) interface{}             { return s }
 func (tp *bcrTestPeer) Set(string, interface{})              {}
 func (tp *bcrTestPeer) RemoteIP() net.IP                     { return []byte{127, 0, 0, 1} }
+func (tp *bcrTestPeer) OriginalAddr() *p2p.NetAddress        { return nil }

blockchain/store_test.go

@@ -126,7 +126,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		eraseSeenCommitInDB   bool
 	}{
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 		},
@@ -137,19 +137,19 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:     newBlock(&header2, commitAtH10),
+			block:     newBlock(header2, commitAtH10),
 			parts:     uncontiguousPartSet,
 			wantPanic: "only save contiguous blocks", // and incomplete and uncontiguous parts
 		},
 
 		{
-			block:     newBlock(&header1, commitAtH10),
+			block:     newBlock(header1, commitAtH10),
 			parts:     incompletePartSet,
 			wantPanic: "only save complete block", // incomplete parts
 		},
 
 		{
-			block:             newBlock(&header1, commitAtH10),
+			block:             newBlock(header1, commitAtH10),
 			parts:             validPartSet,
 			seenCommit:        seenCommit1,
 			corruptCommitInDB: true, // Corrupt the DB's commit entry
@@ -157,7 +157,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:            newBlock(&header1, commitAtH10),
+			block:            newBlock(header1, commitAtH10),
 			parts:            validPartSet,
 			seenCommit:       seenCommit1,
 			wantPanic:        "unmarshal to types.BlockMeta failed",
@@ -165,7 +165,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 
@@ -174,7 +174,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 
@@ -183,7 +183,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
 		},
 
 		{
-			block:      newBlock(&header1, commitAtH10),
+			block:      newBlock(header1, commitAtH10),
 			parts:      validPartSet,
 			seenCommit: seenCommit1,
 
@@ -375,7 +375,7 @@ func doFn(fn func() (interface{}, error)) (res interface{}, err error, panicErr
 	return res, err, panicErr
 }
 
-func newBlock(hdr *types.Header, lastCommit *types.Commit) *types.Block {
+func newBlock(hdr types.Header, lastCommit *types.Commit) *types.Block {
 	return &types.Block{
 		Header:     hdr,
 		LastCommit: lastCommit,

blockchain/wire.go

@@ -2,12 +2,14 @@ package blockchain
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
+	"github.com/tendermint/tendermint/types"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
 	RegisterBlockchainMessages(cdc)
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
+	types.RegisterEvidences(cdc)
 }

cmd/priv_val_server/main.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"os"
 
-	crypto "github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
 
@@ -37,7 +37,7 @@ func main() {
 		*chainID,
 		*addr,
 		pv,
-		crypto.GenPrivKeyEd25519(),
+		ed25519.GenPrivKey(),
 	)
 	err := rs.Start()
 	if err != nil {

cmd/tendermint/commands/wire.go

@@ -2,11 +2,11 @@ package commands
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 }

codecov.yml

@@ -21,3 +21,4 @@ ignore:
   - "docs"
   - "DOCKER"
   - "scripts"
+  - "**/*.pb.go"

config/config.go

@@ -284,7 +284,6 @@ type P2PConfig struct {
 	Seeds string `mapstructure:"seeds"`
 
 	// Comma separated list of nodes to keep persistent connections to
-	// Do not add private peers to this list if you don't want them advertised
 	PersistentPeers string `mapstructure:"persistent_peers"`
 
 	// UPNP port forwarding
@@ -349,9 +348,9 @@ func DefaultP2PConfig() *P2PConfig {
 		AddrBookStrict:          true,
 		MaxNumPeers:             50,
 		FlushThrottleTimeout:    100,
-		MaxPacketMsgPayloadSize: 1024,   // 1 kB
-		SendRate:                512000, // 500 kB/s
-		RecvRate:                512000, // 500 kB/s
+		MaxPacketMsgPayloadSize: 1024,    // 1 kB
+		SendRate:                5120000, // 5 mB/s
+		RecvRate:                5120000, // 5 mB/s
 		PexReactor:              true,
 		SeedMode:                false,
 		AllowDuplicateIP:        true, // so non-breaking yet

config/toml.go

@@ -152,7 +152,6 @@ external_address = "{{ .P2P.ExternalAddress }}"
 seeds = "{{ .P2P.Seeds }}"
 
 # Comma separated list of nodes to keep persistent connections to
-# Do not add private peers to this list if you don't want them advertised
 persistent_peers = "{{ .P2P.PersistentPeers }}"
 
 # UPNP port forwarding

consensus/reactor.go

@@ -80,6 +80,9 @@ func (conR *ConsensusReactor) OnStop() {
 	conR.BaseReactor.OnStop()
 	conR.unsubscribeFromBroadcastEvents()
 	conR.conS.Stop()
+	if !conR.FastSync() {
+		conR.conS.Wait()
+	}
 }
 
 // SwitchToConsensus switches from fast_sync mode to consensus mode.

consensus/reactor_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
+	sm "github.com/tendermint/tendermint/state"
 
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/p2p"
@@ -91,6 +92,51 @@ func TestReactorBasic(t *testing.T) {
 	}, css)
 }
 
+// Ensure we can process blocks with evidence
+func TestReactorWithEvidence(t *testing.T) {
+	N := 4
+	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	evpool := mockEvidencePool{
+		t:  t,
+		ev: []types.Evidence{types.NewMockGoodEvidence(1, 1, []byte("somone"))},
+	}
+	for i := 0; i < N; i++ {
+		css[i].evpool = evpool
+	}
+	reactors, eventChans, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+	// wait till everyone makes the first new block
+	timeoutWaitGroup(t, N, func(j int) {
+		<-eventChans[j]
+	}, css)
+
+	// second block should have evidence
+	timeoutWaitGroup(t, N, func(j int) {
+		<-eventChans[j]
+	}, css)
+}
+
+type mockEvidencePool struct {
+	height int
+	ev     []types.Evidence
+	t      *testing.T
+}
+
+func (m mockEvidencePool) PendingEvidence() []types.Evidence {
+	if m.height > 0 {
+		return m.ev
+	}
+	return nil
+}
+func (m mockEvidencePool) AddEvidence(types.Evidence) error { return nil }
+func (m mockEvidencePool) Update(block *types.Block, state sm.State) {
+	m.height += 1
+
+	if m.height > 0 {
+		require.True(m.t, len(block.Evidence.Evidence) > 0)
+	}
+}
+
 // Ensure a testnet sends proposal heartbeats and makes blocks when there are txs
 func TestReactorProposalHeartbeats(t *testing.T) {
 	N := 4

consensus/replay.go

@@ -273,7 +273,7 @@ func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight
 			ChainId:         h.genDoc.ChainID,
 			ConsensusParams: csParams,
 			Validators:      validators,
-			AppStateBytes:   h.genDoc.AppStateJSON,
+			AppStateBytes:   h.genDoc.AppState,
 		}
 		res, err := proxyApp.Consensus().InitChainSync(req)
 		if err != nil {

consensus/state.go

@@ -81,7 +81,7 @@ type ConsensusState struct {
 	evpool     sm.EvidencePool
 
 	// internal state
-	mtx sync.Mutex
+	mtx sync.RWMutex
 	cstypes.RoundState
 	state sm.State // State until height-1.
 
@@ -192,15 +192,15 @@ func (cs *ConsensusState) String() string {
 
 // GetState returns a copy of the chain state.
 func (cs *ConsensusState) GetState() sm.State {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	return cs.state.Copy()
 }
 
 // GetRoundState returns a shallow copy of the internal consensus state.
 func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 
 	rs := cs.RoundState // copy
 	return &rs
@@ -208,24 +208,24 @@ func (cs *ConsensusState) GetRoundState() *cstypes.RoundState {
 
 // GetRoundStateJSON returns a json of RoundState, marshalled using go-amino.
 func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 
 	return cdc.MarshalJSON(cs.RoundState)
 }
 
 // GetRoundStateSimpleJSON returns a json of RoundStateSimple, marshalled using go-amino.
 func (cs *ConsensusState) GetRoundStateSimpleJSON() ([]byte, error) {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 
 	return cdc.MarshalJSON(cs.RoundState.RoundStateSimple())
 }
 
 // GetValidators returns a copy of the current validators.
 func (cs *ConsensusState) GetValidators() (int64, []*types.Validator) {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators
 }
 
@@ -245,8 +245,8 @@ func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) {
 
 // LoadCommit loads the commit for a given height.
 func (cs *ConsensusState) LoadCommit(height int64) *types.Commit {
-	cs.mtx.Lock()
-	defer cs.mtx.Unlock()
+	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
 	if height == cs.blockStore.Height() {
 		return cs.blockStore.LoadSeenCommit(height)
 	}
@@ -571,8 +571,8 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 		var mi msgInfo
 
 		select {
-		case height := <-cs.mempool.TxsAvailable():
-			cs.handleTxsAvailable(height)
+		case <-cs.mempool.TxsAvailable():
+			cs.handleTxsAvailable()
 		case mi = <-cs.peerMsgQueue:
 			cs.wal.Write(mi)
 			// handles proposals, block parts, votes
@@ -683,11 +683,11 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) {
 
 }
 
-func (cs *ConsensusState) handleTxsAvailable(height int64) {
+func (cs *ConsensusState) handleTxsAvailable() {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
 	// we only need to do this for round 0
-	cs.enterPropose(height, 0)
+	cs.enterPropose(cs.Height, 0)
 }
 
 //-----------------------------------------------------------------------------
@@ -907,6 +907,8 @@ func (cs *ConsensusState) isProposalComplete() bool {
 }
 
 // Create the next block to propose and return it.
+// We really only need to return the parts, but the block
+// is returned for convenience so we can log the proposal block.
 // Returns nil block upon error.
 // NOTE: keep it side-effect free for clarity.
 func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
@@ -926,9 +928,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 
 	// Mempool validated transactions
 	txs := cs.mempool.Reap(cs.state.ConsensusParams.BlockSize.MaxTxs)
-	block, parts := cs.state.MakeBlock(cs.Height, txs, commit)
 	evidence := cs.evpool.PendingEvidence()
-	block.AddEvidence(evidence)
+	block, parts := cs.state.MakeBlock(cs.Height, txs, commit, evidence)
 	return block, parts
 }
 

consensus/types/round_state_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
-	"github.com/tendermint/tendermint/types"
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
+	"github.com/tendermint/tendermint/types"
 )
 
 func BenchmarkRoundStateDeepCopy(b *testing.B) {
@@ -23,7 +23,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 			Hash: cmn.RandBytes(20),
 		},
 	}
-	sig := crypto.SignatureEd25519{}
+	sig := ed25519.SignatureEd25519{}
 	for i := 0; i < nval; i++ {
 		precommits[i] = &types.Vote{
 			ValidatorAddress: types.Address(cmn.RandBytes(20)),
@@ -38,7 +38,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 	}
 	// Random block
 	block := &types.Block{
-		Header: &types.Header{
+		Header: types.Header{
 			ChainID:         cmn.RandStr(12),
 			Time:            time.Now(),
 			LastBlockID:     blockID,
@@ -50,7 +50,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 			LastResultsHash: cmn.RandBytes(20),
 			EvidenceHash:    cmn.RandBytes(20),
 		},
-		Data: &types.Data{
+		Data: types.Data{
 			Txs: txs,
 		},
 		Evidence: types.EvidenceData{},

consensus/types/wire.go

@@ -2,11 +2,13 @@ package types
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
+	"github.com/tendermint/tendermint/types"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
+	types.RegisterEvidences(cdc)
 }

consensus/wire.go

@@ -2,7 +2,7 @@ package consensus
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 var cdc = amino.NewCodec()
@@ -10,5 +10,5 @@ var cdc = amino.NewCodec()
 func init() {
 	RegisterConsensusMessages(cdc)
 	RegisterWALMessages(cdc)
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 }

crypto/README.md

@@ -3,8 +3,15 @@
 crypto is the cryptographic package adapted for Tendermint's uses
 
 ## Importing it
+To get the interfaces,
 `import "github.com/tendermint/tendermint/crypto"`
 
+For any specific algorithm, use its specific module e.g.
+`import "github.com/tendermint/tendermint/crypto/ed25519"`
+
+If you want to decode bytes into one of the types, but don't care about the specific algorithm, use
+`import "github.com/tendermint/tendermint/crypto/amino"`
+
 ## Binary encoding
 
 For Binary encoding, please refer to the [Tendermint encoding spec](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/encoding.md).
@@ -16,9 +23,9 @@ crypto `.Bytes()` uses Amino:binary encoding, but Amino:JSON is also supported.
 ```go
 Example Amino:JSON encodings:
 
-crypto.PrivKeyEd25519     - {"type":"954568A3288910","value":"EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="}
+ed25519.PrivKeyEd25519     - {"type":"954568A3288910","value":"EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ=="}
 crypto.SignatureEd25519   - {"type":"6BF5903DA1DB28","value":"77sQNZOrf7ltExpf7AV1WaYPCHbyRLgjBsoWVzcduuLk+jIGmYk+s5R6Emm29p12HeiNAuhUJgdFGmwkpeGJCA=="}
-crypto.PubKeyEd25519      - {"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="}
+ed25519.PubKeyEd25519      - {"type":"AC26791624DE60","value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="}
 crypto.PrivKeySecp256k1   - {"type":"019E82E1B0F798","value":"zx4Pnh67N+g2V+5vZbQzEyRerX9c4ccNZOVzM9RvJ0Y="}
 crypto.SignatureSecp256k1 - {"type":"6D1EA416E1FEE8","value":"MEUCIQCIg5TqS1l7I+MKTrSPIuUN2+4m5tA29dcauqn3NhEJ2wIgICaZ+lgRc5aOTVahU/XoLopXKn8BZcl0bnuYWLvohR8="}
 crypto.PubKeySecp256k1    - {"type":"F8CCEAEB5AE980","value":"A8lPKJXcNl5VHt1FK8a244K9EJuS4WX1hFBnwisi0IJx"}

crypto/amino.go

@@ -1,37 +0,0 @@
-package crypto
-
-import (
-	amino "github.com/tendermint/go-amino"
-)
-
-var cdc = amino.NewCodec()
-
-func init() {
-	// NOTE: It's important that there be no conflicts here,
-	// as that would change the canonical representations,
-	// and therefore change the address.
-	// TODO: Add feature to go-amino to ensure that there
-	// are no conflicts.
-	RegisterAmino(cdc)
-}
-
-// RegisterAmino registers all crypto related types in the given (amino) codec.
-func RegisterAmino(cdc *amino.Codec) {
-	cdc.RegisterInterface((*PubKey)(nil), nil)
-	cdc.RegisterConcrete(PubKeyEd25519{},
-		"tendermint/PubKeyEd25519", nil)
-	cdc.RegisterConcrete(PubKeySecp256k1{},
-		"tendermint/PubKeySecp256k1", nil)
-
-	cdc.RegisterInterface((*PrivKey)(nil), nil)
-	cdc.RegisterConcrete(PrivKeyEd25519{},
-		"tendermint/PrivKeyEd25519", nil)
-	cdc.RegisterConcrete(PrivKeySecp256k1{},
-		"tendermint/PrivKeySecp256k1", nil)
-
-	cdc.RegisterInterface((*Signature)(nil), nil)
-	cdc.RegisterConcrete(SignatureEd25519{},
-		"tendermint/SignatureEd25519", nil)
-	cdc.RegisterConcrete(SignatureSecp256k1{},
-		"tendermint/SignatureSecp256k1", nil)
-}

crypto/armor/armor.go

@@ -1,4 +1,4 @@
-package crypto
+package armor
 
 import (
 	"bytes"

crypto/armor/armor_test.go

@@ -1,4 +1,4 @@
-package crypto
+package armor
 
 import (
 	"testing"

crypto/crypto.go

@@ -0,0 +1,36 @@
+package crypto
+
+import (
+	cmn "github.com/tendermint/tendermint/libs/common"
+)
+
+type PrivKey interface {
+	Bytes() []byte
+	Sign(msg []byte) (Signature, error)
+	PubKey() PubKey
+	Equals(PrivKey) bool
+}
+
+// An address is a []byte, but hex-encoded even in JSON.
+// []byte leaves us the option to change the address length.
+// Use an alias so Unmarshal methods (with ptr receivers) are available too.
+type Address = cmn.HexBytes
+
+type PubKey interface {
+	Address() Address
+	Bytes() []byte
+	VerifyBytes(msg []byte, sig Signature) bool
+	Equals(PubKey) bool
+}
+
+type Signature interface {
+	Bytes() []byte
+	IsZero() bool
+	Equals(Signature) bool
+}
+
+type Symmetric interface {
+	Keygen() []byte
+	Encrypt(plaintext []byte, secret []byte) (ciphertext []byte)
+	Decrypt(ciphertext []byte, secret []byte) (plaintext []byte, err error)
+}

crypto/doc.go

@@ -22,7 +22,7 @@
 //     pubKey := key.PubKey()
 
 // For example:
-//     privKey, err := crypto.GenPrivKeyEd25519()
+//     privKey, err := ed25519.GenPrivKey()
 //     if err != nil {
 // 	...
 //     }

crypto/ed25519/ed25519.go

@@ -0,0 +1,227 @@
+package ed25519
+
+import (
+	"bytes"
+	"crypto/subtle"
+	"fmt"
+
+	"github.com/tendermint/ed25519"
+	"github.com/tendermint/ed25519/extra25519"
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	cmn "github.com/tendermint/tendermint/libs/common"
+)
+
+//-------------------------------------
+
+var _ crypto.PrivKey = PrivKeyEd25519{}
+
+const (
+	Ed25519PrivKeyAminoRoute   = "tendermint/PrivKeyEd25519"
+	Ed25519PubKeyAminoRoute    = "tendermint/PubKeyEd25519"
+	Ed25519SignatureAminoRoute = "tendermint/SignatureEd25519"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
+	cdc.RegisterConcrete(PubKeyEd25519{},
+		Ed25519PubKeyAminoRoute, nil)
+
+	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
+	cdc.RegisterConcrete(PrivKeyEd25519{},
+		Ed25519PrivKeyAminoRoute, nil)
+
+	cdc.RegisterInterface((*crypto.Signature)(nil), nil)
+	cdc.RegisterConcrete(SignatureEd25519{},
+		Ed25519SignatureAminoRoute, nil)
+}
+
+// PrivKeyEd25519 implements crypto.PrivKey.
+type PrivKeyEd25519 [64]byte
+
+// Bytes marshals the privkey using amino encoding.
+func (privKey PrivKeyEd25519) Bytes() []byte {
+	return cdc.MustMarshalBinaryBare(privKey)
+}
+
+// Sign produces a signature on the provided message.
+func (privKey PrivKeyEd25519) Sign(msg []byte) (crypto.Signature, error) {
+	privKeyBytes := [64]byte(privKey)
+	signatureBytes := ed25519.Sign(&privKeyBytes, msg)
+	return SignatureEd25519(*signatureBytes), nil
+}
+
+// PubKey gets the corresponding public key from the private key.
+func (privKey PrivKeyEd25519) PubKey() crypto.PubKey {
+	privKeyBytes := [64]byte(privKey)
+	initialized := false
+	// If the latter 32 bytes of the privkey are all zero, compute the pubkey
+	// otherwise privkey is initialized and we can use the cached value inside
+	// of the private key.
+	for _, v := range privKeyBytes[32:] {
+		if v != 0 {
+			initialized = true
+			break
+		}
+	}
+	if initialized {
+		var pubkeyBytes [PubKeyEd25519Size]byte
+		copy(pubkeyBytes[:], privKeyBytes[32:])
+		return PubKeyEd25519(pubkeyBytes)
+	}
+
+	pubBytes := *ed25519.MakePublicKey(&privKeyBytes)
+	return PubKeyEd25519(pubBytes)
+}
+
+// Equals - you probably don't need to use this.
+// Runs in constant time based on length of the keys.
+func (privKey PrivKeyEd25519) Equals(other crypto.PrivKey) bool {
+	if otherEd, ok := other.(PrivKeyEd25519); ok {
+		return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1
+	} else {
+		return false
+	}
+}
+
+// ToCurve25519 takes a private key and returns its representation on
+// Curve25519. Curve25519 is birationally equivalent to Edwards25519,
+// which Ed25519 uses internally. This method is intended for use in
+// an X25519 Diffie Hellman key exchange.
+func (privKey PrivKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte {
+	keyCurve25519 := new([32]byte)
+	privKeyBytes := [64]byte(privKey)
+	extra25519.PrivateKeyToCurve25519(keyCurve25519, &privKeyBytes)
+	return keyCurve25519
+}
+
+// GenPrivKey generates a new ed25519 private key.
+// It uses OS randomness in conjunction with the current global random seed
+// in tendermint/libs/common to generate the private key.
+func GenPrivKey() PrivKeyEd25519 {
+	privKey := new([64]byte)
+	copy(privKey[:32], crypto.CRandBytes(32))
+	// ed25519.MakePublicKey(privKey) alters the last 32 bytes of privKey.
+	// It places the pubkey in the last 32 bytes of privKey, and returns the
+	// public key.
+	ed25519.MakePublicKey(privKey)
+	return PrivKeyEd25519(*privKey)
+}
+
+// GenPrivKeyFromSecret hashes the secret with SHA2, and uses
+// that 32 byte output to create the private key.
+// NOTE: secret should be the output of a KDF like bcrypt,
+// if it's derived from user input.
+func GenPrivKeyFromSecret(secret []byte) PrivKeyEd25519 {
+	privKey32 := crypto.Sha256(secret) // Not Ripemd160 because we want 32 bytes.
+	privKey := new([64]byte)
+	copy(privKey[:32], privKey32)
+	// ed25519.MakePublicKey(privKey) alters the last 32 bytes of privKey.
+	// It places the pubkey in the last 32 bytes of privKey, and returns the
+	// public key.
+	ed25519.MakePublicKey(privKey)
+	return PrivKeyEd25519(*privKey)
+}
+
+//-------------------------------------
+
+var _ crypto.PubKey = PubKeyEd25519{}
+
+// PubKeyEd25519Size is the number of bytes in an Ed25519 signature.
+const PubKeyEd25519Size = 32
+
+// PubKeyEd25519 implements crypto.PubKey for the Ed25519 signature scheme.
+type PubKeyEd25519 [PubKeyEd25519Size]byte
+
+// Address is the SHA256-20 of the raw pubkey bytes.
+func (pubKey PubKeyEd25519) Address() crypto.Address {
+	return crypto.Address(tmhash.Sum(pubKey[:]))
+}
+
+// Bytes marshals the PubKey using amino encoding.
+func (pubKey PubKeyEd25519) Bytes() []byte {
+	bz, err := cdc.MarshalBinaryBare(pubKey)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (pubKey PubKeyEd25519) VerifyBytes(msg []byte, sig_ crypto.Signature) bool {
+	// make sure we use the same algorithm to sign
+	sig, ok := sig_.(SignatureEd25519)
+	if !ok {
+		return false
+	}
+	pubKeyBytes := [PubKeyEd25519Size]byte(pubKey)
+	sigBytes := [SignatureEd25519Size]byte(sig)
+	return ed25519.Verify(&pubKeyBytes, msg, &sigBytes)
+}
+
+// ToCurve25519 takes a public key and returns its representation on
+// Curve25519. Curve25519 is birationally equivalent to Edwards25519,
+// which Ed25519 uses internally. This method is intended for use in
+// an X25519 Diffie Hellman key exchange.
+//
+// If there is an error, then this function returns nil.
+func (pubKey PubKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte {
+	keyCurve25519, pubKeyBytes := new([PubKeyEd25519Size]byte), [PubKeyEd25519Size]byte(pubKey)
+	ok := extra25519.PublicKeyToCurve25519(keyCurve25519, &pubKeyBytes)
+	if !ok {
+		return nil
+	}
+	return keyCurve25519
+}
+
+func (pubKey PubKeyEd25519) String() string {
+	return fmt.Sprintf("PubKeyEd25519{%X}", pubKey[:])
+}
+
+// nolint: golint
+func (pubKey PubKeyEd25519) Equals(other crypto.PubKey) bool {
+	if otherEd, ok := other.(PubKeyEd25519); ok {
+		return bytes.Equal(pubKey[:], otherEd[:])
+	} else {
+		return false
+	}
+}
+
+//-------------------------------------
+
+var _ crypto.Signature = SignatureEd25519{}
+
+// Size of an Edwards25519 signature. Namely the size of a compressed
+// Edwards25519 point, and a field element. Both of which are 32 bytes.
+const SignatureEd25519Size = 64
+
+// SignatureEd25519 implements crypto.Signature
+type SignatureEd25519 [SignatureEd25519Size]byte
+
+func (sig SignatureEd25519) Bytes() []byte {
+	bz, err := cdc.MarshalBinaryBare(sig)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (sig SignatureEd25519) IsZero() bool { return len(sig) == 0 }
+
+func (sig SignatureEd25519) String() string { return fmt.Sprintf("/%X.../", cmn.Fingerprint(sig[:])) }
+
+func (sig SignatureEd25519) Equals(other crypto.Signature) bool {
+	if otherEd, ok := other.(SignatureEd25519); ok {
+		return subtle.ConstantTimeCompare(sig[:], otherEd[:]) == 1
+	} else {
+		return false
+	}
+}
+
+func SignatureEd25519FromBytes(data []byte) crypto.Signature {
+	var sig SignatureEd25519
+	copy(sig[:], data)
+	return sig
+}

crypto/ed25519/ed25519_test.go

@@ -0,0 +1,31 @@
+package ed25519_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+)
+
+func TestSignAndValidateEd25519(t *testing.T) {
+
+	privKey := ed25519.GenPrivKey()
+	pubKey := privKey.PubKey()
+
+	msg := crypto.CRandBytes(128)
+	sig, err := privKey.Sign(msg)
+	require.Nil(t, err)
+
+	// Test the signature
+	assert.True(t, pubKey.VerifyBytes(msg, sig))
+
+	// Mutate the signature, just one bit.
+	// TODO: Replace this with a much better fuzzer, tendermint/ed25519/issues/10
+	sigEd := sig.(ed25519.SignatureEd25519)
+	sigEd[7] ^= byte(0x01)
+	sig = sigEd
+
+	assert.False(t, pubKey.VerifyBytes(msg, sig))
+}

crypto/encoding/amino/amino.go

@@ -0,0 +1,57 @@
+package cryptoAmino
+
+import (
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	// NOTE: It's important that there be no conflicts here,
+	// as that would change the canonical representations,
+	// and therefore change the address.
+	// TODO: Remove above note when
+	// https://github.com/tendermint/go-amino/issues/9
+	// is resolved
+	RegisterAmino(cdc)
+}
+
+// RegisterAmino registers all crypto related types in the given (amino) codec.
+func RegisterAmino(cdc *amino.Codec) {
+	// These are all written here instead of
+	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
+	cdc.RegisterConcrete(ed25519.PubKeyEd25519{},
+		"tendermint/PubKeyEd25519", nil)
+	cdc.RegisterConcrete(secp256k1.PubKeySecp256k1{},
+		"tendermint/PubKeySecp256k1", nil)
+
+	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
+	cdc.RegisterConcrete(ed25519.PrivKeyEd25519{},
+		"tendermint/PrivKeyEd25519", nil)
+	cdc.RegisterConcrete(secp256k1.PrivKeySecp256k1{},
+		"tendermint/PrivKeySecp256k1", nil)
+
+	cdc.RegisterInterface((*crypto.Signature)(nil), nil)
+	cdc.RegisterConcrete(ed25519.SignatureEd25519{},
+		"tendermint/SignatureEd25519", nil)
+	cdc.RegisterConcrete(secp256k1.SignatureSecp256k1{},
+		"tendermint/SignatureSecp256k1", nil)
+}
+
+func PrivKeyFromBytes(privKeyBytes []byte) (privKey crypto.PrivKey, err error) {
+	err = cdc.UnmarshalBinaryBare(privKeyBytes, &privKey)
+	return
+}
+
+func PubKeyFromBytes(pubKeyBytes []byte) (pubKey crypto.PubKey, err error) {
+	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
+	return
+}
+
+func SignatureFromBytes(pubKeyBytes []byte) (pubKey crypto.Signature, err error) {
+	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
+	return
+}

crypto/encoding/amino/encode_test.go

@@ -1,4 +1,4 @@
-package crypto
+package cryptoAmino
 
 import (
 	"os"
@@ -6,6 +6,9 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
 )
 
 type byter interface {
@@ -56,64 +59,70 @@ func ExamplePrintRegisteredTypes() {
 
 func TestKeyEncodings(t *testing.T) {
 	cases := []struct {
-		privKey           PrivKey
+		privKey           crypto.PrivKey
 		privSize, pubSize int // binary sizes
 	}{
 		{
-			privKey:  GenPrivKeyEd25519(),
+			privKey:  ed25519.GenPrivKey(),
 			privSize: 69,
 			pubSize:  37,
 		},
 		{
-			privKey:  GenPrivKeySecp256k1(),
+			privKey:  secp256k1.GenPrivKey(),
 			privSize: 37,
 			pubSize:  38,
 		},
 	}
 
-	for _, tc := range cases {
+	for tcIndex, tc := range cases {
 
 		// Check (de/en)codings of PrivKeys.
-		var priv2, priv3 PrivKey
+		var priv2, priv3 crypto.PrivKey
 		checkAminoBinary(t, tc.privKey, &priv2, tc.privSize)
-		assert.EqualValues(t, tc.privKey, priv2)
+		assert.EqualValues(t, tc.privKey, priv2, "tc #%d", tcIndex)
 		checkAminoJSON(t, tc.privKey, &priv3, false) // TODO also check Prefix bytes.
-		assert.EqualValues(t, tc.privKey, priv3)
+		assert.EqualValues(t, tc.privKey, priv3, "tc #%d", tcIndex)
 
 		// Check (de/en)codings of Signatures.
-		var sig1, sig2, sig3 Signature
+		var sig1, sig2, sig3 crypto.Signature
 		sig1, err := tc.privKey.Sign([]byte("something"))
-		assert.NoError(t, err)
+		assert.NoError(t, err, "tc #%d", tcIndex)
 		checkAminoBinary(t, sig1, &sig2, -1) // Signature size changes for Secp anyways.
-		assert.EqualValues(t, sig1, sig2)
+		assert.EqualValues(t, sig1, sig2, "tc #%d", tcIndex)
 		checkAminoJSON(t, sig1, &sig3, false) // TODO also check Prefix bytes.
-		assert.EqualValues(t, sig1, sig3)
+		assert.EqualValues(t, sig1, sig3, "tc #%d", tcIndex)
 
 		// Check (de/en)codings of PubKeys.
 		pubKey := tc.privKey.PubKey()
-		var pub2, pub3 PubKey
+		var pub2, pub3 crypto.PubKey
 		checkAminoBinary(t, pubKey, &pub2, tc.pubSize)
-		assert.EqualValues(t, pubKey, pub2)
+		assert.EqualValues(t, pubKey, pub2, "tc #%d", tcIndex)
 		checkAminoJSON(t, pubKey, &pub3, false) // TODO also check Prefix bytes.
-		assert.EqualValues(t, pubKey, pub3)
+		assert.EqualValues(t, pubKey, pub3, "tc #%d", tcIndex)
 	}
 }
 
 func TestNilEncodings(t *testing.T) {
 
 	// Check nil Signature.
-	var a, b Signature
+	var a, b crypto.Signature
 	checkAminoJSON(t, &a, &b, true)
 	assert.EqualValues(t, a, b)
 
 	// Check nil PubKey.
-	var c, d PubKey
+	var c, d crypto.PubKey
 	checkAminoJSON(t, &c, &d, true)
 	assert.EqualValues(t, c, d)
 
 	// Check nil PrivKey.
-	var e, f PrivKey
+	var e, f crypto.PrivKey
 	checkAminoJSON(t, &e, &f, true)
 	assert.EqualValues(t, e, f)
 
 }
+
+func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) {
+	pk, err := PubKeyFromBytes([]byte("foo"))
+	require.NotNil(t, err, "expecting a non-nil error")
+	require.Nil(t, pk, "expecting an empty public key on error")
+}

crypto/hash.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"crypto/sha256"
+
 	"golang.org/x/crypto/ripemd160"
 )
 

crypto/hkdfchacha20poly1305/hkdfchachapoly.go

@@ -14,6 +14,7 @@ import (
 	"golang.org/x/crypto/hkdf"
 )
 
+// Implements crypto.AEAD
 type hkdfchacha20poly1305 struct {
 	key [KeySize]byte
 }

crypto/priv_key.go

@@ -1,164 +0,0 @@
-package crypto
-
-import (
-	"crypto/subtle"
-
-	secp256k1 "github.com/btcsuite/btcd/btcec"
-	"github.com/tendermint/ed25519"
-	"github.com/tendermint/ed25519/extra25519"
-)
-
-func PrivKeyFromBytes(privKeyBytes []byte) (privKey PrivKey, err error) {
-	err = cdc.UnmarshalBinaryBare(privKeyBytes, &privKey)
-	return
-}
-
-//----------------------------------------
-
-type PrivKey interface {
-	Bytes() []byte
-	Sign(msg []byte) (Signature, error)
-	PubKey() PubKey
-	Equals(PrivKey) bool
-}
-
-//-------------------------------------
-
-var _ PrivKey = PrivKeyEd25519{}
-
-// Implements PrivKey
-type PrivKeyEd25519 [64]byte
-
-func (privKey PrivKeyEd25519) Bytes() []byte {
-	return cdc.MustMarshalBinaryBare(privKey)
-}
-
-func (privKey PrivKeyEd25519) Sign(msg []byte) (Signature, error) {
-	privKeyBytes := [64]byte(privKey)
-	signatureBytes := ed25519.Sign(&privKeyBytes, msg)
-	return SignatureEd25519(*signatureBytes), nil
-}
-
-func (privKey PrivKeyEd25519) PubKey() PubKey {
-	privKeyBytes := [64]byte(privKey)
-	pubBytes := *ed25519.MakePublicKey(&privKeyBytes)
-	return PubKeyEd25519(pubBytes)
-}
-
-// Equals - you probably don't need to use this.
-// Runs in constant time based on length of the keys.
-func (privKey PrivKeyEd25519) Equals(other PrivKey) bool {
-	if otherEd, ok := other.(PrivKeyEd25519); ok {
-		return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1
-	} else {
-		return false
-	}
-}
-
-func (privKey PrivKeyEd25519) ToCurve25519() *[32]byte {
-	keyCurve25519 := new([32]byte)
-	privKeyBytes := [64]byte(privKey)
-	extra25519.PrivateKeyToCurve25519(keyCurve25519, &privKeyBytes)
-	return keyCurve25519
-}
-
-// Deterministically generates new priv-key bytes from key.
-func (privKey PrivKeyEd25519) Generate(index int) PrivKeyEd25519 {
-	bz, err := cdc.MarshalBinaryBare(struct {
-		PrivKey [64]byte
-		Index   int
-	}{privKey, index})
-	if err != nil {
-		panic(err)
-	}
-	newBytes := Sha256(bz)
-	newKey := new([64]byte)
-	copy(newKey[:32], newBytes)
-	ed25519.MakePublicKey(newKey)
-	return PrivKeyEd25519(*newKey)
-}
-
-func GenPrivKeyEd25519() PrivKeyEd25519 {
-	privKeyBytes := new([64]byte)
-	copy(privKeyBytes[:32], CRandBytes(32))
-	ed25519.MakePublicKey(privKeyBytes)
-	return PrivKeyEd25519(*privKeyBytes)
-}
-
-// NOTE: secret should be the output of a KDF like bcrypt,
-// if it's derived from user input.
-func GenPrivKeyEd25519FromSecret(secret []byte) PrivKeyEd25519 {
-	privKey32 := Sha256(secret) // Not Ripemd160 because we want 32 bytes.
-	privKeyBytes := new([64]byte)
-	copy(privKeyBytes[:32], privKey32)
-	ed25519.MakePublicKey(privKeyBytes)
-	return PrivKeyEd25519(*privKeyBytes)
-}
-
-//-------------------------------------
-
-var _ PrivKey = PrivKeySecp256k1{}
-
-// Implements PrivKey
-type PrivKeySecp256k1 [32]byte
-
-func (privKey PrivKeySecp256k1) Bytes() []byte {
-	return cdc.MustMarshalBinaryBare(privKey)
-}
-
-func (privKey PrivKeySecp256k1) Sign(msg []byte) (Signature, error) {
-	priv__, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
-	sig__, err := priv__.Sign(Sha256(msg))
-	if err != nil {
-		return nil, err
-	}
-	return SignatureSecp256k1(sig__.Serialize()), nil
-}
-
-func (privKey PrivKeySecp256k1) PubKey() PubKey {
-	_, pub__ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
-	var pub PubKeySecp256k1
-	copy(pub[:], pub__.SerializeCompressed())
-	return pub
-}
-
-// Equals - you probably don't need to use this.
-// Runs in constant time based on length of the keys.
-func (privKey PrivKeySecp256k1) Equals(other PrivKey) bool {
-	if otherSecp, ok := other.(PrivKeySecp256k1); ok {
-		return subtle.ConstantTimeCompare(privKey[:], otherSecp[:]) == 1
-	} else {
-		return false
-	}
-}
-
-/*
-// Deterministically generates new priv-key bytes from key.
-func (key PrivKeySecp256k1) Generate(index int) PrivKeySecp256k1 {
-	newBytes := cdc.BinarySha256(struct {
-		PrivKey [64]byte
-		Index   int
-	}{key, index})
-	var newKey [64]byte
-	copy(newKey[:], newBytes)
-	return PrivKeySecp256k1(newKey)
-}
-*/
-
-func GenPrivKeySecp256k1() PrivKeySecp256k1 {
-	privKeyBytes := [32]byte{}
-	copy(privKeyBytes[:], CRandBytes(32))
-	priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKeyBytes[:])
-	copy(privKeyBytes[:], priv.Serialize())
-	return PrivKeySecp256k1(privKeyBytes)
-}
-
-// NOTE: secret should be the output of a KDF like bcrypt,
-// if it's derived from user input.
-func GenPrivKeySecp256k1FromSecret(secret []byte) PrivKeySecp256k1 {
-	privKey32 := Sha256(secret) // Not Ripemd160 because we want 32 bytes.
-	priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey32)
-	privKeyBytes := [32]byte{}
-	copy(privKeyBytes[:], priv.Serialize())
-	return PrivKeySecp256k1(privKeyBytes)
-}

crypto/priv_key_test.go

@@ -1,60 +0,0 @@
-package crypto_test
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/tendermint/tendermint/crypto"
-)
-
-func TestGeneratePrivKey(t *testing.T) {
-	testPriv := crypto.GenPrivKeyEd25519()
-	testGenerate := testPriv.Generate(1)
-	signBytes := []byte("something to sign")
-	pub := testGenerate.PubKey()
-	sig, err := testGenerate.Sign(signBytes)
-	assert.NoError(t, err)
-	assert.True(t, pub.VerifyBytes(signBytes, sig))
-}
-
-/*
-
-type BadKey struct {
-	PrivKeyEd25519
-}
-
-func TestReadPrivKey(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	// garbage in, garbage out
-	garbage := []byte("hjgewugfbiewgofwgewr")
-	XXX This test wants to register BadKey globally to crypto,
-	but we don't want to support that.
-	_, err := PrivKeyFromBytes(garbage)
-	require.Error(err)
-
-	edKey := GenPrivKeyEd25519()
-	badKey := BadKey{edKey}
-
-	cases := []struct {
-		key   PrivKey
-		valid bool
-	}{
-		{edKey, true},
-		{badKey, false},
-	}
-
-	for i, tc := range cases {
-		data := tc.key.Bytes()
-		fmt.Println(">>>", data)
-		key, err := PrivKeyFromBytes(data)
-		fmt.Printf("!!! %#v\n", key, err)
-		if tc.valid {
-			assert.NoError(err, "%d", i)
-			assert.Equal(tc.key, key, "%d", i)
-		} else {
-			assert.Error(err, "%d: %#v", i, key)
-		}
-	}
-}
-*/

crypto/pub_key.go

@@ -1,153 +0,0 @@
-package crypto
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"fmt"
-
-	"golang.org/x/crypto/ripemd160"
-
-	secp256k1 "github.com/btcsuite/btcd/btcec"
-
-	"github.com/tendermint/ed25519"
-	"github.com/tendermint/ed25519/extra25519"
-
-	cmn "github.com/tendermint/tendermint/libs/common"
-
-	"github.com/tendermint/tendermint/crypto/tmhash"
-)
-
-// An address is a []byte, but hex-encoded even in JSON.
-// []byte leaves us the option to change the address length.
-// Use an alias so Unmarshal methods (with ptr receivers) are available too.
-type Address = cmn.HexBytes
-
-func PubKeyFromBytes(pubKeyBytes []byte) (pubKey PubKey, err error) {
-	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
-	return
-}
-
-//----------------------------------------
-
-type PubKey interface {
-	Address() Address
-	Bytes() []byte
-	VerifyBytes(msg []byte, sig Signature) bool
-	Equals(PubKey) bool
-}
-
-//-------------------------------------
-
-var _ PubKey = PubKeyEd25519{}
-
-const PubKeyEd25519Size = 32
-
-// Implements PubKeyInner
-type PubKeyEd25519 [PubKeyEd25519Size]byte
-
-// Address is the SHA256-20 of the raw pubkey bytes.
-func (pubKey PubKeyEd25519) Address() Address {
-	return Address(tmhash.Sum(pubKey[:]))
-}
-
-func (pubKey PubKeyEd25519) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(pubKey)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (pubKey PubKeyEd25519) VerifyBytes(msg []byte, sig_ Signature) bool {
-	// make sure we use the same algorithm to sign
-	sig, ok := sig_.(SignatureEd25519)
-	if !ok {
-		return false
-	}
-	pubKeyBytes := [PubKeyEd25519Size]byte(pubKey)
-	sigBytes := [SignatureEd25519Size]byte(sig)
-	return ed25519.Verify(&pubKeyBytes, msg, &sigBytes)
-}
-
-// For use with golang/crypto/nacl/box
-// If error, returns nil.
-func (pubKey PubKeyEd25519) ToCurve25519() *[PubKeyEd25519Size]byte {
-	keyCurve25519, pubKeyBytes := new([PubKeyEd25519Size]byte), [PubKeyEd25519Size]byte(pubKey)
-	ok := extra25519.PublicKeyToCurve25519(keyCurve25519, &pubKeyBytes)
-	if !ok {
-		return nil
-	}
-	return keyCurve25519
-}
-
-func (pubKey PubKeyEd25519) String() string {
-	return fmt.Sprintf("PubKeyEd25519{%X}", pubKey[:])
-}
-
-func (pubKey PubKeyEd25519) Equals(other PubKey) bool {
-	if otherEd, ok := other.(PubKeyEd25519); ok {
-		return bytes.Equal(pubKey[:], otherEd[:])
-	} else {
-		return false
-	}
-}
-
-//-------------------------------------
-
-var _ PubKey = PubKeySecp256k1{}
-
-const PubKeySecp256k1Size = 33
-
-// Implements PubKey.
-// Compressed pubkey (just the x-cord),
-// prefixed with 0x02 or 0x03, depending on the y-cord.
-type PubKeySecp256k1 [PubKeySecp256k1Size]byte
-
-// Implements Bitcoin style addresses: RIPEMD160(SHA256(pubkey))
-func (pubKey PubKeySecp256k1) Address() Address {
-	hasherSHA256 := sha256.New()
-	hasherSHA256.Write(pubKey[:]) // does not error
-	sha := hasherSHA256.Sum(nil)
-
-	hasherRIPEMD160 := ripemd160.New()
-	hasherRIPEMD160.Write(sha) // does not error
-	return Address(hasherRIPEMD160.Sum(nil))
-}
-
-func (pubKey PubKeySecp256k1) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(pubKey)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sig_ Signature) bool {
-	// and assert same algorithm to sign and verify
-	sig, ok := sig_.(SignatureSecp256k1)
-	if !ok {
-		return false
-	}
-
-	pub__, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256())
-	if err != nil {
-		return false
-	}
-	sig__, err := secp256k1.ParseDERSignature(sig[:], secp256k1.S256())
-	if err != nil {
-		return false
-	}
-	return sig__.Verify(Sha256(msg), pub__)
-}
-
-func (pubKey PubKeySecp256k1) String() string {
-	return fmt.Sprintf("PubKeySecp256k1{%X}", pubKey[:])
-}
-
-func (pubKey PubKeySecp256k1) Equals(other PubKey) bool {
-	if otherSecp, ok := other.(PubKeySecp256k1); ok {
-		return bytes.Equal(pubKey[:], otherSecp[:])
-	} else {
-		return false
-	}
-}

crypto/pub_key_test.go

@@ -1,50 +0,0 @@
-package crypto
-
-import (
-	"encoding/hex"
-	"testing"
-
-	"github.com/btcsuite/btcutil/base58"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type keyData struct {
-	priv string
-	pub  string
-	addr string
-}
-
-var secpDataTable = []keyData{
-	{
-		priv: "a96e62ed3955e65be32703f12d87b6b5cf26039ecfa948dc5107a495418e5330",
-		pub:  "02950e1cdfcb133d6024109fd489f734eeb4502418e538c28481f22bce276f248c",
-		addr: "1CKZ9Nx4zgds8tU7nJHotKSDr4a9bYJCa3",
-	},
-}
-
-func TestPubKeySecp256k1Address(t *testing.T) {
-	for _, d := range secpDataTable {
-		privB, _ := hex.DecodeString(d.priv)
-		pubB, _ := hex.DecodeString(d.pub)
-		addrBbz, _, _ := base58.CheckDecode(d.addr)
-		addrB := Address(addrBbz)
-
-		var priv PrivKeySecp256k1
-		copy(priv[:], privB)
-
-		pubKey := priv.PubKey()
-		pubT, _ := pubKey.(PubKeySecp256k1)
-		pub := pubT[:]
-		addr := pubKey.Address()
-
-		assert.Equal(t, pub, pubB, "Expected pub keys to match")
-		assert.Equal(t, addr, addrB, "Expected addresses to match")
-	}
-}
-
-func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) {
-	pk, err := PubKeyFromBytes([]byte("foo"))
-	require.NotNil(t, err, "expecting a non-nil error")
-	require.Nil(t, pk, "expecting an empty public key on error")
-}

crypto/secp256k1/secp256k1.go

@@ -0,0 +1,198 @@
+package secp256k1
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/subtle"
+	"fmt"
+
+	secp256k1 "github.com/btcsuite/btcd/btcec"
+	amino "github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/libs/common"
+	"golang.org/x/crypto/ripemd160"
+)
+
+//-------------------------------------
+const (
+	Secp256k1PrivKeyAminoRoute   = "tendermint/PrivKeySecp256k1"
+	Secp256k1PubKeyAminoRoute    = "tendermint/PubKeySecp256k1"
+	Secp256k1SignatureAminoRoute = "tendermint/SignatureSecp256k1"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	cdc.RegisterInterface((*crypto.PubKey)(nil), nil)
+	cdc.RegisterConcrete(PubKeySecp256k1{},
+		Secp256k1PubKeyAminoRoute, nil)
+
+	cdc.RegisterInterface((*crypto.PrivKey)(nil), nil)
+	cdc.RegisterConcrete(PrivKeySecp256k1{},
+		Secp256k1PrivKeyAminoRoute, nil)
+
+	cdc.RegisterInterface((*crypto.Signature)(nil), nil)
+	cdc.RegisterConcrete(SignatureSecp256k1{},
+		Secp256k1SignatureAminoRoute, nil)
+}
+
+//-------------------------------------
+
+var _ crypto.PrivKey = PrivKeySecp256k1{}
+
+// PrivKeySecp256k1 implements PrivKey.
+type PrivKeySecp256k1 [32]byte
+
+// Bytes marshalls the private key using amino encoding.
+func (privKey PrivKeySecp256k1) Bytes() []byte {
+	return cdc.MustMarshalBinaryBare(privKey)
+}
+
+// Sign creates an ECDSA signature on curve Secp256k1, using SHA256 on the msg.
+func (privKey PrivKeySecp256k1) Sign(msg []byte) (crypto.Signature, error) {
+	priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
+	sig, err := priv.Sign(crypto.Sha256(msg))
+	if err != nil {
+		return nil, err
+	}
+	return SignatureSecp256k1(sig.Serialize()), nil
+}
+
+// PubKey performs the point-scalar multiplication from the privKey on the
+// generator point to get the pubkey.
+func (privKey PrivKeySecp256k1) PubKey() crypto.PubKey {
+	_, pubkeyObject := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:])
+	var pubkeyBytes PubKeySecp256k1
+	copy(pubkeyBytes[:], pubkeyObject.SerializeCompressed())
+	return pubkeyBytes
+}
+
+// Equals - you probably don't need to use this.
+// Runs in constant time based on length of the keys.
+func (privKey PrivKeySecp256k1) Equals(other crypto.PrivKey) bool {
+	if otherSecp, ok := other.(PrivKeySecp256k1); ok {
+		return subtle.ConstantTimeCompare(privKey[:], otherSecp[:]) == 1
+	}
+	return false
+}
+
+// GenPrivKey generates a new ECDSA private key on curve secp256k1 private key.
+// It uses OS randomness in conjunction with the current global random seed
+// in tendermint/libs/common to generate the private key.
+func GenPrivKey() PrivKeySecp256k1 {
+	privKeyBytes := [32]byte{}
+	copy(privKeyBytes[:], crypto.CRandBytes(32))
+	// crypto.CRandBytes is guaranteed to be 32 bytes long, so it can be
+	// casted to PrivKeySecp256k1.
+	return PrivKeySecp256k1(privKeyBytes)
+}
+
+// GenPrivKeySecp256k1 hashes the secret with SHA2, and uses
+// that 32 byte output to create the private key.
+// NOTE: secret should be the output of a KDF like bcrypt,
+// if it's derived from user input.
+func GenPrivKeySecp256k1(secret []byte) PrivKeySecp256k1 {
+	privKey32 := sha256.Sum256(secret)
+	// sha256.Sum256() is guaranteed to be 32 bytes long, so it can be
+	// casted to PrivKeySecp256k1.
+	return PrivKeySecp256k1(privKey32)
+}
+
+//-------------------------------------
+
+var _ crypto.PubKey = PubKeySecp256k1{}
+
+// PubKeySecp256k1Size is comprised of 32 bytes for one field element
+// (the x-coordinate), plus one byte for the parity of the y-coordinate.
+const PubKeySecp256k1Size = 33
+
+// PubKeySecp256k1 implements crypto.PubKey.
+// It is the compressed form of the pubkey. The first byte depends is a 0x02 byte
+// if the y-coordinate is the lexicographically largest of the two associated with
+// the x-coordinate. Otherwise the first byte is a 0x03.
+// This prefix is followed with the x-coordinate.
+type PubKeySecp256k1 [PubKeySecp256k1Size]byte
+
+// Address returns a Bitcoin style addresses: RIPEMD160(SHA256(pubkey))
+func (pubKey PubKeySecp256k1) Address() crypto.Address {
+	hasherSHA256 := sha256.New()
+	hasherSHA256.Write(pubKey[:]) // does not error
+	sha := hasherSHA256.Sum(nil)
+
+	hasherRIPEMD160 := ripemd160.New()
+	hasherRIPEMD160.Write(sha) // does not error
+	return crypto.Address(hasherRIPEMD160.Sum(nil))
+}
+
+// Bytes returns the pubkey marshalled with amino encoding.
+func (pubKey PubKeySecp256k1) Bytes() []byte {
+	bz, err := cdc.MarshalBinaryBare(pubKey)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, interfaceSig crypto.Signature) bool {
+	// and assert same algorithm to sign and verify
+	sig, ok := interfaceSig.(SignatureSecp256k1)
+	if !ok {
+		return false
+	}
+
+	pub, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256())
+	if err != nil {
+		return false
+	}
+	parsedSig, err := secp256k1.ParseDERSignature(sig[:], secp256k1.S256())
+	if err != nil {
+		return false
+	}
+	return parsedSig.Verify(crypto.Sha256(msg), pub)
+}
+
+func (pubKey PubKeySecp256k1) String() string {
+	return fmt.Sprintf("PubKeySecp256k1{%X}", pubKey[:])
+}
+
+func (pubKey PubKeySecp256k1) Equals(other crypto.PubKey) bool {
+	if otherSecp, ok := other.(PubKeySecp256k1); ok {
+		return bytes.Equal(pubKey[:], otherSecp[:])
+	}
+	return false
+}
+
+//-------------------------------------
+
+var _ crypto.Signature = SignatureSecp256k1{}
+
+// SignatureSecp256k1 implements crypto.Signature
+type SignatureSecp256k1 []byte
+
+func (sig SignatureSecp256k1) Bytes() []byte {
+	bz, err := cdc.MarshalBinaryBare(sig)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (sig SignatureSecp256k1) IsZero() bool { return len(sig) == 0 }
+
+func (sig SignatureSecp256k1) String() string {
+	return fmt.Sprintf("/%X.../", common.Fingerprint(sig[:]))
+}
+
+func (sig SignatureSecp256k1) Equals(other crypto.Signature) bool {
+	if otherSecp, ok := other.(SignatureSecp256k1); ok {
+		return subtle.ConstantTimeCompare(sig[:], otherSecp[:]) == 1
+	} else {
+		return false
+	}
+}
+
+func SignatureSecp256k1FromBytes(data []byte) crypto.Signature {
+	sig := make(SignatureSecp256k1, len(data))
+	copy(sig[:], data)
+	return sig
+}

crypto/secp256k1/secpk256k1_test.go

@@ -0,0 +1,88 @@
+package secp256k1_test
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/btcsuite/btcutil/base58"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
+
+	underlyingSecp256k1 "github.com/btcsuite/btcd/btcec"
+)
+
+type keyData struct {
+	priv string
+	pub  string
+	addr string
+}
+
+var secpDataTable = []keyData{
+	{
+		priv: "a96e62ed3955e65be32703f12d87b6b5cf26039ecfa948dc5107a495418e5330",
+		pub:  "02950e1cdfcb133d6024109fd489f734eeb4502418e538c28481f22bce276f248c",
+		addr: "1CKZ9Nx4zgds8tU7nJHotKSDr4a9bYJCa3",
+	},
+}
+
+func TestPubKeySecp256k1Address(t *testing.T) {
+	for _, d := range secpDataTable {
+		privB, _ := hex.DecodeString(d.priv)
+		pubB, _ := hex.DecodeString(d.pub)
+		addrBbz, _, _ := base58.CheckDecode(d.addr)
+		addrB := crypto.Address(addrBbz)
+
+		var priv secp256k1.PrivKeySecp256k1
+		copy(priv[:], privB)
+
+		pubKey := priv.PubKey()
+		pubT, _ := pubKey.(secp256k1.PubKeySecp256k1)
+		pub := pubT[:]
+		addr := pubKey.Address()
+
+		assert.Equal(t, pub, pubB, "Expected pub keys to match")
+		assert.Equal(t, addr, addrB, "Expected addresses to match")
+	}
+}
+
+func TestSignAndValidateSecp256k1(t *testing.T) {
+	privKey := secp256k1.GenPrivKey()
+	pubKey := privKey.PubKey()
+
+	msg := crypto.CRandBytes(128)
+	sig, err := privKey.Sign(msg)
+	require.Nil(t, err)
+
+	assert.True(t, pubKey.VerifyBytes(msg, sig))
+
+	// Mutate the signature, just one bit.
+	sigEd := sig.(secp256k1.SignatureSecp256k1)
+	sigEd[3] ^= byte(0x01)
+	sig = sigEd
+
+	assert.False(t, pubKey.VerifyBytes(msg, sig))
+}
+
+// This test is intended to justify the removal of calls to the underlying library
+// in creating the privkey.
+func TestSecp256k1LoadPrivkeyAndSerializeIsIdentity(t *testing.T) {
+	numberOfTests := 256
+	for i := 0; i < numberOfTests; i++ {
+		// Seed the test case with some random bytes
+		privKeyBytes := [32]byte{}
+		copy(privKeyBytes[:], crypto.CRandBytes(32))
+
+		// This function creates a private and public key in the underlying libraries format.
+		// The private key is basically calling new(big.Int).SetBytes(pk), which removes leading zero bytes
+		priv, _ := underlyingSecp256k1.PrivKeyFromBytes(underlyingSecp256k1.S256(), privKeyBytes[:])
+		// this takes the bytes returned by `(big int).Bytes()`, and if the length is less than 32 bytes,
+		// pads the bytes from the left with zero bytes. Therefore these two functions composed
+		// result in the identity function on privKeyBytes, hence the following equality check
+		// always returning true.
+		serializedBytes := priv.Serialize()
+		require.Equal(t, privKeyBytes[:], serializedBytes)
+	}
+}

crypto/signature.go

@@ -1,90 +0,0 @@
-package crypto
-
-import (
-	"fmt"
-
-	"crypto/subtle"
-
-	. "github.com/tendermint/tendermint/libs/common"
-)
-
-func SignatureFromBytes(pubKeyBytes []byte) (pubKey Signature, err error) {
-	err = cdc.UnmarshalBinaryBare(pubKeyBytes, &pubKey)
-	return
-}
-
-//----------------------------------------
-
-type Signature interface {
-	Bytes() []byte
-	IsZero() bool
-	Equals(Signature) bool
-}
-
-//-------------------------------------
-
-var _ Signature = SignatureEd25519{}
-
-const SignatureEd25519Size = 64
-
-// Implements Signature
-type SignatureEd25519 [SignatureEd25519Size]byte
-
-func (sig SignatureEd25519) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(sig)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (sig SignatureEd25519) IsZero() bool { return len(sig) == 0 }
-
-func (sig SignatureEd25519) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }
-
-func (sig SignatureEd25519) Equals(other Signature) bool {
-	if otherEd, ok := other.(SignatureEd25519); ok {
-		return subtle.ConstantTimeCompare(sig[:], otherEd[:]) == 1
-	} else {
-		return false
-	}
-}
-
-func SignatureEd25519FromBytes(data []byte) Signature {
-	var sig SignatureEd25519
-	copy(sig[:], data)
-	return sig
-}
-
-//-------------------------------------
-
-var _ Signature = SignatureSecp256k1{}
-
-// Implements Signature
-type SignatureSecp256k1 []byte
-
-func (sig SignatureSecp256k1) Bytes() []byte {
-	bz, err := cdc.MarshalBinaryBare(sig)
-	if err != nil {
-		panic(err)
-	}
-	return bz
-}
-
-func (sig SignatureSecp256k1) IsZero() bool { return len(sig) == 0 }
-
-func (sig SignatureSecp256k1) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }
-
-func (sig SignatureSecp256k1) Equals(other Signature) bool {
-	if otherSecp, ok := other.(SignatureSecp256k1); ok {
-		return subtle.ConstantTimeCompare(sig[:], otherSecp[:]) == 1
-	} else {
-		return false
-	}
-}
-
-func SignatureSecp256k1FromBytes(data []byte) Signature {
-	sig := make(SignatureSecp256k1, len(data))
-	copy(sig[:], data)
-	return sig
-}

crypto/signature_test.go

@@ -1,46 +0,0 @@
-package crypto
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSignAndValidateEd25519(t *testing.T) {
-
-	privKey := GenPrivKeyEd25519()
-	pubKey := privKey.PubKey()
-
-	msg := CRandBytes(128)
-	sig, err := privKey.Sign(msg)
-	require.Nil(t, err)
-
-	// Test the signature
-	assert.True(t, pubKey.VerifyBytes(msg, sig))
-
-	// Mutate the signature, just one bit.
-	sigEd := sig.(SignatureEd25519)
-	sigEd[7] ^= byte(0x01)
-	sig = sigEd
-
-	assert.False(t, pubKey.VerifyBytes(msg, sig))
-}
-
-func TestSignAndValidateSecp256k1(t *testing.T) {
-	privKey := GenPrivKeySecp256k1()
-	pubKey := privKey.PubKey()
-
-	msg := CRandBytes(128)
-	sig, err := privKey.Sign(msg)
-	require.Nil(t, err)
-
-	assert.True(t, pubKey.VerifyBytes(msg, sig))
-
-	// Mutate the signature, just one bit.
-	sigEd := sig.(SignatureSecp256k1)
-	sigEd[3] ^= byte(0x01)
-	sig = sigEd
-
-	assert.False(t, pubKey.VerifyBytes(msg, sig))
-}

crypto/xsalsa20symmetric/symmetric.go

@@ -1,12 +1,15 @@
-package crypto
+package xsalsa20symmetric
 
 import (
 	"errors"
 
-	. "github.com/tendermint/tendermint/libs/common"
+	"github.com/tendermint/tendermint/crypto"
+	cmn "github.com/tendermint/tendermint/libs/common"
 	"golang.org/x/crypto/nacl/secretbox"
 )
 
+// TODO, make this into a struct that implements crypto.Symmetric.
+
 const nonceLen = 24
 const secretLen = 32
 
@@ -15,9 +18,9 @@ const secretLen = 32
 // NOTE: call crypto.MixEntropy() first.
 func EncryptSymmetric(plaintext []byte, secret []byte) (ciphertext []byte) {
 	if len(secret) != secretLen {
-		PanicSanity(Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
+		cmn.PanicSanity(cmn.Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
 	}
-	nonce := CRandBytes(nonceLen)
+	nonce := crypto.CRandBytes(nonceLen)
 	nonceArr := [nonceLen]byte{}
 	copy(nonceArr[:], nonce)
 	secretArr := [secretLen]byte{}
@@ -32,7 +35,7 @@ func EncryptSymmetric(plaintext []byte, secret []byte) (ciphertext []byte) {
 // The ciphertext is (secretbox.Overhead + 24) bytes longer than the plaintext.
 func DecryptSymmetric(ciphertext []byte, secret []byte) (plaintext []byte, err error) {
 	if len(secret) != secretLen {
-		PanicSanity(Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
+		cmn.PanicSanity(cmn.Fmt("Secret must be 32 bytes long, got len %v", len(secret)))
 	}
 	if len(ciphertext) <= secretbox.Overhead+nonceLen {
 		return nil, errors.New("Ciphertext is too short")

crypto/xsalsa20symmetric/symmetric_test.go

@@ -1,4 +1,4 @@
-package crypto
+package xsalsa20symmetric
 
 import (
 	"testing"
@@ -6,12 +6,13 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/tendermint/tendermint/crypto"
 	"golang.org/x/crypto/bcrypt"
 )
 
 func TestSimple(t *testing.T) {
 
-	MixEntropy([]byte("someentropy"))
+	crypto.MixEntropy([]byte("someentropy"))
 
 	plaintext := []byte("sometext")
 	secret := []byte("somesecretoflengththirtytwo===32")
@@ -24,7 +25,7 @@ func TestSimple(t *testing.T) {
 
 func TestSimpleWithKDF(t *testing.T) {
 
-	MixEntropy([]byte("someentropy"))
+	crypto.MixEntropy([]byte("someentropy"))
 
 	plaintext := []byte("sometext")
 	secretPass := []byte("somesecret")
@@ -32,7 +33,7 @@ func TestSimpleWithKDF(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	secret = Sha256(secret)
+	secret = crypto.Sha256(secret)
 
 	ciphertext := EncryptSymmetric(plaintext, secret)
 	plaintext2, err := DecryptSymmetric(ciphertext, secret)

docs/DOCS_README.md

@@ -9,7 +9,9 @@ and built using [VuePress](https://vuepress.vuejs.org/) from the tendermint webs
 - https://github.com/tendermint/tendermint.com
 
 which has a [configuration file](https://github.com/tendermint/tendermint.com/blob/develop/docs/.vuepress/config.js) for displaying
-the Table of Contents that lists all the documentation.
+the Table of Contents that lists all the documentation. 
+
+Under the hood, Jenkins listens for changes in ./docs then pushes a `docs-staging` branch to the tendermint.com repo with the latest documentation. That branch must be manually PR'd to `develop` then `master` for staging then production. This process should happen in synchrony with a release.
 
 The `README.md` in this directory is the landing page for
 website documentation and the following folders are intentionally

docs/networks/fast-sync.md

@@ -1,8 +1,4 @@
-Fast Sync
-=========
-
-Background
-----------
+# Fast Sync
 
 In a proof of work blockchain, syncing with the chain is the same
 process as staying up-to-date with the consensus: download blocks, and
@@ -14,21 +10,19 @@ scratch can take a very long time. It's much faster to just download
 blocks and check the merkle tree of validators than to run the real-time
 consensus gossip protocol.
 
-Fast Sync
----------
+## Using Fast Sync
 
-To support faster syncing, tendermint offers a ``fast-sync`` mode, which
-is enabled by default, and can be toggled in the ``config.toml`` or via
-``--fast_sync=false``.
+To support faster syncing, tendermint offers a `fast-sync` mode, which
+is enabled by default, and can be toggled in the `config.toml` or via
+`--fast_sync=false`.
 
 In this mode, the tendermint daemon will sync hundreds of times faster
 than if it used the real-time consensus process. Once caught up, the
 daemon will switch out of fast sync and into the normal consensus mode.
-After running for some time, the node is considered ``caught up`` if it
+After running for some time, the node is considered `caught up` if it
 has at least one peer and it's height is at least as high as the max
-reported peer height. See `the IsCaughtUp
-method <https://github.com/tendermint/tendermint/blob/b467515719e686e4678e6da4e102f32a491b85a0/blockchain/pool.go#L128>`__.
+reported peer height. See [the IsCaughtUp
+method](https://github.com/tendermint/tendermint/blob/b467515719e686e4678e6da4e102f32a491b85a0/blockchain/pool.go#L128).
 
 If we're lagging sufficiently, we should go back to fast syncing, but
-this is an open issue:
-https://github.com/tendermint/tendermint/issues/129
+this is an [open issue](https://github.com/tendermint/tendermint/issues/129).

docs/spec/blockchain/encoding.md

@@ -149,7 +149,33 @@ func MakeParts(obj interface{}, partSize int) []Part
 
 ## Merkle Trees
 
-Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure.
+For an overview of Merkle trees, see
+[wikipedia](https://en.wikipedia.org/wiki/Merkle_tree)
+
+
+A Simple Tree is a simple compact binary tree for a static list of items. Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. In a Simple Tree, the transactions and validation signatures of a block are hashed using this simple merkle tree logic.
+
+If the number of items is not a power of two, the tree will not be full
+and some leaf nodes will be at different levels. Simple Tree tries to
+keep both sides of the tree the same size, but the left side may be one
+greater, for example: 
+
+```
+   Simple Tree with 6 items           Simple Tree with 7 items 
+                                                        
+              *                                  *             
+             / \                                / \            
+           /     \                            /     \          
+         /         \                        /         \        
+       /             \                    /             \      
+      *               *                  *               *     
+     / \             / \                / \             / \    
+    /   \           /   \              /   \           /   \   
+   /     \         /     \            /     \         /     \  
+  *       h2      *       h5         *       *       *       h6
+ / \             / \                / \     / \     / \        
+h0  h1          h3  h4             h0  h1  h2  h3  h4  h5
+```
 
 Tendermint always uses the `TMHASH` hash function, which is the first 20-bytes
 of the SHA256:
@@ -235,6 +261,18 @@ func computeHashFromAunts(index, total int, leafHash []byte, innerHashes [][]byt
 }
 ```
 
+### Simple Tree with Dictionaries
+
+The Simple Tree is used to merkelize a list of items, so to merkelize a
+(short) dictionary of key-value pairs, encode the dictionary as an
+ordered list of ``KVPair`` structs. The block hash is such a hash
+derived from all the fields of the block ``Header``. The state hash is
+similarly derived.
+
+### IAVL+ Tree
+
+Because Tendermint only uses a Simple Merkle Tree, application developers are expect to use their own Merkle tree in their applications. For example, the IAVL+ Tree - an immutable self-balancing binary tree for persisting application state is used by the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk/blob/develop/docs/core/multistore.md)
+
 ## JSON
 
 ### Amino

docs/spec/consensus/consensus.md

@@ -1,9 +1,329 @@
-We are working to finalize an updated Tendermint specification with formal
-proofs of safety and liveness.
+# Byzantine Consensus Algorithm
 
-In the meantime, see the [description in the
-docs](http://tendermint.readthedocs.io/en/master/specification/byzantine-consensus-algorithm.html).
+## Terms
 
-There are also relevant but somewhat outdated descriptions in Jae Kwon's [original
-whitepaper](https://tendermint.com/static/docs/tendermint.pdf) and Ethan Buchman's [master's
-thesis](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769).
+-   The network is composed of optionally connected *nodes*. Nodes
+    directly connected to a particular node are called *peers*.
+-   The consensus process in deciding the next block (at some *height*
+    `H`) is composed of one or many *rounds*.
+-   `NewHeight`, `Propose`, `Prevote`, `Precommit`, and `Commit`
+    represent state machine states of a round. (aka `RoundStep` or
+    just "step").
+-   A node is said to be *at* a given height, round, and step, or at
+    `(H,R,S)`, or at `(H,R)` in short to omit the step.
+-   To *prevote* or *precommit* something means to broadcast a [prevote
+    vote](https://godoc.org/github.com/tendermint/tendermint/types#Vote)
+    or [first precommit
+    vote](https://godoc.org/github.com/tendermint/tendermint/types#FirstPrecommit)
+    for something.
+-   A vote *at* `(H,R)` is a vote signed with the bytes for `H` and `R`
+    included in its [sign-bytes](block-structure.html#vote-sign-bytes).
+-   *+2/3* is short for "more than 2/3"
+-   *1/3+* is short for "1/3 or more"
+-   A set of +2/3 of prevotes for a particular block or `<nil>` at
+    `(H,R)` is called a *proof-of-lock-change* or *PoLC* for short.
+
+## State Machine Overview
+
+At each height of the blockchain a round-based protocol is run to
+determine the next block. Each round is composed of three *steps*
+(`Propose`, `Prevote`, and `Precommit`), along with two special steps
+`Commit` and `NewHeight`.
+
+In the optimal scenario, the order of steps is:
+
+```
+NewHeight -> (Propose -> Prevote -> Precommit)+ -> Commit -> NewHeight ->...
+```
+
+The sequence `(Propose -> Prevote -> Precommit)` is called a *round*.
+There may be more than one round required to commit a block at a given
+height. Examples for why more rounds may be required include:
+
+-   The designated proposer was not online.
+-   The block proposed by the designated proposer was not valid.
+-   The block proposed by the designated proposer did not propagate
+    in time.
+-   The block proposed was valid, but +2/3 of prevotes for the proposed
+    block were not received in time for enough validator nodes by the
+    time they reached the `Precommit` step. Even though +2/3 of prevotes
+    are necessary to progress to the next step, at least one validator
+    may have voted `<nil>` or maliciously voted for something else.
+-   The block proposed was valid, and +2/3 of prevotes were received for
+    enough nodes, but +2/3 of precommits for the proposed block were not
+    received for enough validator nodes.
+
+Some of these problems are resolved by moving onto the next round &
+proposer. Others are resolved by increasing certain round timeout
+parameters over each successive round.
+
+## State Machine Diagram
+
+```
+                         +-------------------------------------+
+                         v                                     |(Wait til `CommmitTime+timeoutCommit`)
+                   +-----------+                         +-----+-----+
+      +----------> |  Propose  +--------------+          | NewHeight |
+      |            +-----------+              |          +-----------+
+      |                                       |                ^
+      |(Else, after timeoutPrecommit)         v                |
++-----+-----+                           +-----------+          |
+| Precommit |  <------------------------+  Prevote  |          |
++-----+-----+                           +-----------+          |
+      |(When +2/3 Precommits for block found)                  |
+      v                                                        |
++--------------------------------------------------------------------+
+       |  Commit                                                            |
+       |                                                                    |
+       |  * Set CommitTime = now;                                           |
+       |  * Wait for block, then stage/save/commit block;                   |
+       +--------------------------------------------------------------------+
+```
+
+Background Gossip
+=================
+
+A node may not have a corresponding validator private key, but it
+nevertheless plays an active role in the consensus process by relaying
+relevant meta-data, proposals, blocks, and votes to its peers. A node
+that has the private keys of an active validator and is engaged in
+signing votes is called a *validator-node*. All nodes (not just
+validator-nodes) have an associated state (the current height, round,
+and step) and work to make progress.
+
+Between two nodes there exists a `Connection`, and multiplexed on top of
+this connection are fairly throttled `Channel`s of information. An
+epidemic gossip protocol is implemented among some of these channels to
+bring peers up to speed on the most recent state of consensus. For
+example,
+
+-   Nodes gossip `PartSet` parts of the current round's proposer's
+    proposed block. A LibSwift inspired algorithm is used to quickly
+    broadcast blocks across the gossip network.
+-   Nodes gossip prevote/precommit votes. A node `NODE_A` that is ahead
+    of `NODE_B` can send `NODE_B` prevotes or precommits for `NODE_B`'s
+    current (or future) round to enable it to progress forward.
+-   Nodes gossip prevotes for the proposed PoLC (proof-of-lock-change)
+    round if one is proposed.
+-   Nodes gossip to nodes lagging in blockchain height with block
+    [commits](https://godoc.org/github.com/tendermint/tendermint/types#Commit)
+    for older blocks.
+-   Nodes opportunistically gossip `HasVote` messages to hint peers what
+    votes it already has.
+-   Nodes broadcast their current state to all neighboring peers. (but
+    is not gossiped further)
+
+There's more, but let's not get ahead of ourselves here.
+
+## Proposals
+
+A proposal is signed and published by the designated proposer at each
+round. The proposer is chosen by a deterministic and non-choking round
+robin selection algorithm that selects proposers in proportion to their
+voting power (see
+[implementation](https://github.com/tendermint/tendermint/blob/develop/types/validator_set.go)).
+
+A proposal at `(H,R)` is composed of a block and an optional latest
+`PoLC-Round < R` which is included iff the proposer knows of one. This
+hints the network to allow nodes to unlock (when safe) to ensure the
+liveness property.
+
+## State Machine Spec
+
+### Propose Step (height:H,round:R)
+
+Upon entering `Propose`: - The designated proposer proposes a block at
+`(H,R)`.
+
+The `Propose` step ends: - After `timeoutProposeR` after entering
+`Propose`. --> goto `Prevote(H,R)` - After receiving proposal block
+and all prevotes at `PoLC-Round`. --> goto `Prevote(H,R)` - After
+[common exit conditions](#common-exit-conditions)
+
+### Prevote Step (height:H,round:R)
+
+Upon entering `Prevote`, each validator broadcasts its prevote vote.
+
+-   First, if the validator is locked on a block since `LastLockRound`
+    but now has a PoLC for something else at round `PoLC-Round` where
+    `LastLockRound < PoLC-Round < R`, then it unlocks.
+-   If the validator is still locked on a block, it prevotes that.
+-   Else, if the proposed block from `Propose(H,R)` is good, it
+    prevotes that.
+-   Else, if the proposal is invalid or wasn't received on time, it
+    prevotes `<nil>`.
+
+The `Prevote` step ends: - After +2/3 prevotes for a particular block or
+`<nil>`. -->; goto `Precommit(H,R)` - After `timeoutPrevote` after
+receiving any +2/3 prevotes. --> goto `Precommit(H,R)` - After
+[common exit conditions](#common-exit-conditions)
+
+### Precommit Step (height:H,round:R)
+
+Upon entering `Precommit`, each validator broadcasts its precommit vote.
+- If the validator has a PoLC at `(H,R)` for a particular block `B`, it
+(re)locks (or changes lock to) and precommits `B` and sets
+`LastLockRound = R`. - Else, if the validator has a PoLC at `(H,R)` for
+`<nil>`, it unlocks and precommits `<nil>`. - Else, it keeps the lock
+unchanged and precommits `<nil>`.
+
+A precommit for `<nil>` means "I didn’t see a PoLC for this round, but I
+did get +2/3 prevotes and waited a bit".
+
+The Precommit step ends: - After +2/3 precommits for `<nil>`. -->
+goto `Propose(H,R+1)` - After `timeoutPrecommit` after receiving any
++2/3 precommits. --> goto `Propose(H,R+1)` - After [common exit
+conditions](#common-exit-conditions)
+
+### Common exit conditions
+
+-   After +2/3 precommits for a particular block. --> goto
+    `Commit(H)`
+-   After any +2/3 prevotes received at `(H,R+x)`. --> goto
+    `Prevote(H,R+x)`
+-   After any +2/3 precommits received at `(H,R+x)`. --> goto
+    `Precommit(H,R+x)`
+
+### Commit Step (height:H)
+
+-   Set `CommitTime = now()`
+-   Wait until block is received. --> goto `NewHeight(H+1)`
+
+### NewHeight Step (height:H)
+
+-   Move `Precommits` to `LastCommit` and increment height.
+-   Set `StartTime = CommitTime+timeoutCommit`
+-   Wait until `StartTime` to receive straggler commits. --> goto
+    `Propose(H,0)`
+
+## Proofs
+
+### Proof of Safety
+
+Assume that at most -1/3 of the voting power of validators is byzantine.
+If a validator commits block `B` at round `R`, it's because it saw +2/3
+of precommits at round `R`. This implies that 1/3+ of honest nodes are
+still locked at round `R' > R`. These locked validators will remain
+locked until they see a PoLC at `R' > R`, but this won't happen because
+1/3+ are locked and honest, so at most -2/3 are available to vote for
+anything other than `B`.
+
+### Proof of Liveness
+
+If 1/3+ honest validators are locked on two different blocks from
+different rounds, a proposers' `PoLC-Round` will eventually cause nodes
+locked from the earlier round to unlock. Eventually, the designated
+proposer will be one that is aware of a PoLC at the later round. Also,
+`timeoutProposalR` increments with round `R`, while the size of a
+proposal are capped, so eventually the network is able to "fully gossip"
+the whole proposal (e.g. the block & PoLC).
+
+### Proof of Fork Accountability
+
+Define the JSet (justification-vote-set) at height `H` of a validator
+`V1` to be all the votes signed by the validator at `H` along with
+justification PoLC prevotes for each lock change. For example, if `V1`
+signed the following precommits: `Precommit(B1 @ round 0)`,
+`Precommit(<nil> @ round 1)`, `Precommit(B2 @ round 4)` (note that no
+precommits were signed for rounds 2 and 3, and that's ok),
+`Precommit(B1 @ round 0)` must be justified by a PoLC at round 0, and
+`Precommit(B2 @ round 4)` must be justified by a PoLC at round 4; but
+the precommit for `<nil>` at round 1 is not a lock-change by definition
+so the JSet for `V1` need not include any prevotes at round 1, 2, or 3
+(unless `V1` happened to have prevoted for those rounds).
+
+Further, define the JSet at height `H` of a set of validators `VSet` to
+be the union of the JSets for each validator in `VSet`. For a given
+commit by honest validators at round `R` for block `B` we can construct
+a JSet to justify the commit for `B` at `R`. We say that a JSet
+*justifies* a commit at `(H,R)` if all the committers (validators in the
+commit-set) are each justified in the JSet with no duplicitous vote
+signatures (by the committers).
+
+-   **Lemma**: When a fork is detected by the existence of two
+    conflicting [commits](./validators.html#commiting-a-block), the
+    union of the JSets for both commits (if they can be compiled) must
+    include double-signing by at least 1/3+ of the validator set.
+    **Proof**: The commit cannot be at the same round, because that
+    would immediately imply double-signing by 1/3+. Take the union of
+    the JSets of both commits. If there is no double-signing by at least
+    1/3+ of the validator set in the union, then no honest validator
+    could have precommitted any different block after the first commit.
+    Yet, +2/3 did. Reductio ad absurdum.
+
+As a corollary, when there is a fork, an external process can determine
+the blame by requiring each validator to justify all of its round votes.
+Either we will find 1/3+ who cannot justify at least one of their votes,
+and/or, we will find 1/3+ who had double-signed.
+
+### Alternative algorithm
+
+Alternatively, we can take the JSet of a commit to be the "full commit".
+That is, if light clients and validators do not consider a block to be
+committed unless the JSet of the commit is also known, then we get the
+desirable property that if there ever is a fork (e.g. there are two
+conflicting "full commits"), then 1/3+ of the validators are immediately
+punishable for double-signing.
+
+There are many ways to ensure that the gossip network efficiently share
+the JSet of a commit. One solution is to add a new message type that
+tells peers that this node has (or does not have) a +2/3 majority for B
+(or) at (H,R), and a bitarray of which votes contributed towards that
+majority. Peers can react by responding with appropriate votes.
+
+We will implement such an algorithm for the next iteration of the
+Tendermint consensus protocol.
+
+Other potential improvements include adding more data in votes such as
+the last known PoLC round that caused a lock change, and the last voted
+round/step (or, we may require that validators not skip any votes). This
+may make JSet verification/gossip logic easier to implement.
+
+### Censorship Attacks
+
+Due to the definition of a block
+[commit](../../tendermint-core/validator.md#commiting-a-block), any 1/3+ coalition of
+validators can halt the blockchain by not broadcasting their votes. Such
+a coalition can also censor particular transactions by rejecting blocks
+that include these transactions, though this would result in a
+significant proportion of block proposals to be rejected, which would
+slow down the rate of block commits of the blockchain, reducing its
+utility and value. The malicious coalition might also broadcast votes in
+a trickle so as to grind blockchain block commits to a near halt, or
+engage in any combination of these attacks.
+
+If a global active adversary were also involved, it can partition the
+network in such a way that it may appear that the wrong subset of
+validators were responsible for the slowdown. This is not just a
+limitation of Tendermint, but rather a limitation of all consensus
+protocols whose network is potentially controlled by an active
+adversary.
+
+### Overcoming Forks and Censorship Attacks
+
+For these types of attacks, a subset of the validators through external
+means should coordinate to sign a reorg-proposal that chooses a fork
+(and any evidence thereof) and the initial subset of validators with
+their signatures. Validators who sign such a reorg-proposal forego its
+collateral on all other forks. Clients should verify the signatures on
+the reorg-proposal, verify any evidence, and make a judgement or prompt
+the end-user for a decision. For example, a phone wallet app may prompt
+the user with a security warning, while a refrigerator may accept any
+reorg-proposal signed by +1/2 of the original validators.
+
+No non-synchronous Byzantine fault-tolerant algorithm can come to
+consensus when 1/3+ of validators are dishonest, yet a fork assumes that
+1/3+ of validators have already been dishonest by double-signing or
+lock-changing without justification. So, signing the reorg-proposal is a
+coordination problem that cannot be solved by any non-synchronous
+protocol (i.e. automatically, and without making assumptions about the
+reliability of the underlying network). It must be provided by means
+external to the weakly-synchronous Tendermint consensus algorithm. For
+now, we leave the problem of reorg-proposal coordination to human
+coordination via internet media. Validators must take care to ensure
+that there are no significant network partitions, to avoid situations
+where two conflicting reorg-proposals are signed.
+
+Assuming that the external coordination medium and protocol is robust,
+it follows that forks are less of a concern than [censorship
+attacks](#censorship-attacks).

docs/spec/scripts/crypto.go

@@ -5,12 +5,12 @@ import (
 	"os"
 
 	amino "github.com/tendermint/go-amino"
-	crypto "github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 func main() {
 	cdc := amino.NewCodec()
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 	cdc.PrintTypes(os.Stdout)
 	fmt.Println("")
 }

docs/specification/block-structure.rst

@@ -1,218 +0,0 @@
-Block Structure
-===============
-
-The tendermint consensus engine records all agreements by a
-supermajority of nodes into a blockchain, which is replicated among all
-nodes. This blockchain is accessible via various rpc endpoints, mainly
-``/block?height=`` to get the full block, as well as
-``/blockchain?minHeight=_&maxHeight=_`` to get a list of headers. But
-what exactly is stored in these blocks?
-
-Block
-~~~~~
-
-A
-`Block <https://godoc.org/github.com/tendermint/tendermint/types#Block>`__
-contains:
-
--  a `Header <#header>`__ contains merkle hashes for various chain
-   states
--  the
-   `Data <https://godoc.org/github.com/tendermint/tendermint/types#Data>`__
-   is all transactions which are to be processed
--  the `LastCommit <#commit>`__ > 2/3 signatures for the last block
-
-The signatures returned along with block ``H`` are those validating
-block ``H-1``. This can be a little confusing, but we must also consider
-that the ``Header`` also contains the ``LastCommitHash``. It would be
-impossible for a Header to include the commits that sign it, as it would
-cause an infinite loop here. But when we get block ``H``, we find
-``Header.LastCommitHash``, which must match the hash of ``LastCommit``.
-
-Header
-~~~~~~
-
-The
-`Header <https://godoc.org/github.com/tendermint/tendermint/types#Header>`__
-contains lots of information (follow link for up-to-date info). Notably,
-it maintains the ``Height``, the ``LastBlockID`` (to make it a chain),
-and hashes of the data, the app state, and the validator set. This is
-important as the only item that is signed by the validators is the
-``Header``, and all other data must be validated against one of the
-merkle hashes in the ``Header``.
-
-The ``DataHash`` can provide a nice check on the
-`Data <https://godoc.org/github.com/tendermint/tendermint/types#Data>`__
-returned in this same block. If you are subscribed to new blocks, via
-tendermint RPC, in order to display or process the new transactions you
-should at least validate that the ``DataHash`` is valid. If it is
-important to verify autheniticity, you must wait for the ``LastCommit``
-from the next block to make sure the block header (including
-``DataHash``) was properly signed.
-
-The ``ValidatorHash`` contains a hash of the current
-`Validators <https://godoc.org/github.com/tendermint/tendermint/types#Validator>`__.
-Tracking all changes in the validator set is complex, but a client can
-quickly compare this hash with the `hash of the currently known
-validators <https://godoc.org/github.com/tendermint/tendermint/types#ValidatorSet.Hash>`__
-to see if there have been changes.
-
-The ``AppHash`` serves as the basis for validating any merkle proofs
-that come from the ABCI application. It represents the
-state of the actual application, rather that the state of the blockchain
-itself. This means it's necessary in order to perform any business
-logic, such as verifying an account balance.
-
-**Note** After the transactions are committed to a block, they still
-need to be processed in a separate step, which happens between the
-blocks. If you find a given transaction in the block at height ``H``,
-the effects of running that transaction will be first visible in the
-``AppHash`` from the block header at height ``H+1``.
-
-Like the ``LastCommit`` issue, this is a requirement of the immutability
-of the block chain, as the application only applies transactions *after*
-they are commited to the chain.
-
-Commit
-~~~~~~
-
-The
-`Commit <https://godoc.org/github.com/tendermint/tendermint/types#Commit>`__
-contains a set of
-`Votes <https://godoc.org/github.com/tendermint/tendermint/types#Vote>`__
-that were made by the validator set to reach consensus on this block.
-This is the key to the security in any PoS system, and actually no data
-that cannot be traced back to a block header with a valid set of Votes
-can be trusted. Thus, getting the Commit data and verifying the votes is
-extremely important.
-
-As mentioned above, in order to find the ``precommit votes`` for block
-header ``H``, we need to query block ``H+1``. Then we need to check the
-votes, make sure they really are for that block, and properly formatted.
-Much of this code is implemented in Go in the
-`light-client <https://github.com/tendermint/light-client>`__ package.
-If you look at the code, you will notice that we need to provide the
-``chainID`` of the blockchain in order to properly calculate the votes.
-This is to protect anyone from swapping votes between chains to fake (or
-frame) a validator. Also note that this ``chainID`` is in the
-``genesis.json`` from *Tendermint*, not the ``genesis.json`` from the
-basecoin app (`that is a different
-chainID... <https://github.com/cosmos/cosmos-sdk/issues/32>`__).
-
-Once we have those votes, and we calculated the proper `sign
-bytes <https://godoc.org/github.com/tendermint/tendermint/types#Vote.WriteSignBytes>`__
-using the chainID and a `nice helper
-function <https://godoc.org/github.com/tendermint/tendermint/types#SignBytes>`__,
-we can verify them. The light client is responsible for maintaining a
-set of validators that we trust. Each vote only stores the validators
-``Address``, as well as the ``Signature``. Assuming we have a local copy
-of the trusted validator set, we can look up the ``Public Key`` of the
-validator given its ``Address``, then verify that the ``Signature``
-matches the ``SignBytes`` and ``Public Key``. Then we sum up the total
-voting power of all validators, whose votes fulfilled all these
-stringent requirements. If the total number of voting power for a single
-block is greater than 2/3 of all voting power, then we can finally trust
-the block header, the AppHash, and the proof we got from the ABCI
-application.
-
-Vote Sign Bytes
-^^^^^^^^^^^^^^^
-
-The ``sign-bytes`` of a vote is produced by taking a
-`stable-json <https://github.com/substack/json-stable-stringify>`__-like
-deterministic JSON `wire <./wire-protocol.html>`__ encoding of
-the vote (excluding the ``Signature`` field), and wrapping it with
-``{"chain_id":"my_chain","vote":...}``.
-
-For example, a precommit vote might have the following ``sign-bytes``:
-
-.. code:: json
-
-    {"chain_id":"my_chain","vote":{"block_hash":"611801F57B4CE378DF1A3FFF1216656E89209A99","block_parts_header":{"hash":"B46697379DBE0774CC2C3B656083F07CA7E0F9CE","total":123},"height":1234,"round":1,"type":2}}
-
-Block Hash
-~~~~~~~~~~
-
-The `block
-hash <https://godoc.org/github.com/tendermint/tendermint/types#Block.Hash>`__
-is the `Simple Tree hash <./merkle.html#simple-tree-with-dictionaries>`__
-of the fields of the block ``Header`` encoded as a list of
-``KVPair``\ s.
-
-Transaction
-~~~~~~~~~~~
-
-A transaction is any sequence of bytes. It is up to your
-ABCI application to accept or reject transactions.
-
-BlockID
-~~~~~~~
-
-Many of these data structures refer to the
-`BlockID <https://godoc.org/github.com/tendermint/tendermint/types#BlockID>`__,
-which is the ``BlockHash`` (hash of the block header, also referred to
-by the next block) along with the ``PartSetHeader``. The
-``PartSetHeader`` is explained below and is used internally to
-orchestrate the p2p propogation. For clients, it is basically opaque
-bytes, but they must match for all votes.
-
-PartSetHeader
-~~~~~~~~~~~~~
-
-The
-`PartSetHeader <https://godoc.org/github.com/tendermint/tendermint/types#PartSetHeader>`__
-contains the total number of pieces in a
-`PartSet <https://godoc.org/github.com/tendermint/tendermint/types#PartSet>`__,
-and the Merkle root hash of those pieces.
-
-PartSet
-~~~~~~~
-
-PartSet is used to split a byteslice of data into parts (pieces) for
-transmission. By splitting data into smaller parts and computing a
-Merkle root hash on the list, you can verify that a part is legitimately
-part of the complete data, and the part can be forwarded to other peers
-before all the parts are known. In short, it's a fast way to securely
-propagate a large chunk of data (like a block) over a gossip network.
-
-PartSet was inspired by the LibSwift project.
-
-Usage:
-
-.. code:: go
-
-    data := RandBytes(2 << 20) // Something large
-
-    partSet := NewPartSetFromData(data)
-    partSet.Total()     // Total number of 4KB parts
-    partSet.Count()     // Equal to the Total, since we already have all the parts
-    partSet.Hash()      // The Merkle root hash
-    partSet.BitArray()  // A BitArray of partSet.Total() 1's
-
-    header := partSet.Header() // Send this to the peer
-    header.Total        // Total number of parts
-    header.Hash         // The merkle root hash
-
-    // Now we'll reconstruct the data from the parts
-    partSet2 := NewPartSetFromHeader(header)
-    partSet2.Total()    // Same total as partSet.Total()
-    partSet2.Count()    // Zero, since this PartSet doesn't have any parts yet.
-    partSet2.Hash()     // Same hash as in partSet.Hash()
-    partSet2.BitArray() // A BitArray of partSet.Total() 0's
-
-    // In a gossip network the parts would arrive in arbitrary order, perhaps
-    // in response to explicit requests for parts, or optimistically in response
-    // to the receiving peer's partSet.BitArray().
-    for !partSet2.IsComplete() {
-        part := receivePartFromGossipNetwork()
-        added, err := partSet2.AddPart(part)
-        if err != nil {
-        // A wrong part,
-            // the merkle trail does not hash to partSet2.Hash()
-        } else if !added {
-            // A duplicate part already received
-        }
-    }
-
-    data2, _ := ioutil.ReadAll(partSet2.GetReader())
-    bytes.Equal(data, data2) // true

docs/specification/byzantine-consensus-algorithm.rst

@@ -1,349 +0,0 @@
-Byzantine Consensus Algorithm
-=============================
-
-Terms
------
-
--  The network is composed of optionally connected *nodes*. Nodes
-   directly connected to a particular node are called *peers*.
--  The consensus process in deciding the next block (at some *height*
-   ``H``) is composed of one or many *rounds*.
--  ``NewHeight``, ``Propose``, ``Prevote``, ``Precommit``, and
-   ``Commit`` represent state machine states of a round. (aka
-   ``RoundStep`` or just "step").
--  A node is said to be *at* a given height, round, and step, or at
-   ``(H,R,S)``, or at ``(H,R)`` in short to omit the step.
--  To *prevote* or *precommit* something means to broadcast a `prevote
-   vote <https://godoc.org/github.com/tendermint/tendermint/types#Vote>`__
-   or `first precommit
-   vote <https://godoc.org/github.com/tendermint/tendermint/types#FirstPrecommit>`__
-   for something.
--  A vote *at* ``(H,R)`` is a vote signed with the bytes for ``H`` and
-   ``R`` included in its
-   `sign-bytes <block-structure.html#vote-sign-bytes>`__.
--  *+2/3* is short for "more than 2/3"
--  *1/3+* is short for "1/3 or more"
--  A set of +2/3 of prevotes for a particular block or ``<nil>`` at
-   ``(H,R)`` is called a *proof-of-lock-change* or *PoLC* for short.
-
-State Machine Overview
-----------------------
-
-At each height of the blockchain a round-based protocol is run to
-determine the next block. Each round is composed of three *steps*
-(``Propose``, ``Prevote``, and ``Precommit``), along with two special
-steps ``Commit`` and ``NewHeight``.
-
-In the optimal scenario, the order of steps is:
-
-::
-
-    NewHeight -> (Propose -> Prevote -> Precommit)+ -> Commit -> NewHeight ->...
-
-The sequence ``(Propose -> Prevote -> Precommit)`` is called a *round*.
-There may be more than one round required to commit a block at a given
-height. Examples for why more rounds may be required include:
-
--  The designated proposer was not online.
--  The block proposed by the designated proposer was not valid.
--  The block proposed by the designated proposer did not propagate in
-   time.
--  The block proposed was valid, but +2/3 of prevotes for the proposed
-   block were not received in time for enough validator nodes by the
-   time they reached the ``Precommit`` step. Even though +2/3 of
-   prevotes are necessary to progress to the next step, at least one
-   validator may have voted ``<nil>`` or maliciously voted for something
-   else.
--  The block proposed was valid, and +2/3 of prevotes were received for
-   enough nodes, but +2/3 of precommits for the proposed block were not
-   received for enough validator nodes.
-
-Some of these problems are resolved by moving onto the next round &
-proposer. Others are resolved by increasing certain round timeout
-parameters over each successive round.
-
-State Machine Diagram
----------------------
-
-::
-
-                                +-------------------------------------+
-                                v                                     |(Wait til `CommmitTime+timeoutCommit`)
-                          +-----------+                         +-----+-----+
-             +----------> |  Propose  +--------------+          | NewHeight |
-             |            +-----------+              |          +-----------+
-             |                                       |                ^
-             |(Else, after timeoutPrecommit)         v                |
-       +-----+-----+                           +-----------+          |
-       | Precommit |  <------------------------+  Prevote  |          |
-       +-----+-----+                           +-----------+          |
-             |(When +2/3 Precommits for block found)                  |
-             v                                                        |
-       +--------------------------------------------------------------------+
-       |  Commit                                                            |
-       |                                                                    |
-       |  * Set CommitTime = now;                                           |
-       |  * Wait for block, then stage/save/commit block;                   |
-       +--------------------------------------------------------------------+
-
-Background Gossip
------------------
-
-A node may not have a corresponding validator private key, but it
-nevertheless plays an active role in the consensus process by relaying
-relevant meta-data, proposals, blocks, and votes to its peers. A node
-that has the private keys of an active validator and is engaged in
-signing votes is called a *validator-node*. All nodes (not just
-validator-nodes) have an associated state (the current height, round,
-and step) and work to make progress.
-
-Between two nodes there exists a ``Connection``, and multiplexed on top
-of this connection are fairly throttled ``Channel``\ s of information.
-An epidemic gossip protocol is implemented among some of these channels
-to bring peers up to speed on the most recent state of consensus. For
-example,
-
--  Nodes gossip ``PartSet`` parts of the current round's proposer's
-   proposed block. A LibSwift inspired algorithm is used to quickly
-   broadcast blocks across the gossip network.
--  Nodes gossip prevote/precommit votes. A node NODE\_A that is ahead of
-   NODE\_B can send NODE\_B prevotes or precommits for NODE\_B's current
-   (or future) round to enable it to progress forward.
--  Nodes gossip prevotes for the proposed PoLC (proof-of-lock-change)
-   round if one is proposed.
--  Nodes gossip to nodes lagging in blockchain height with block
-   `commits <https://godoc.org/github.com/tendermint/tendermint/types#Commit>`__
-   for older blocks.
--  Nodes opportunistically gossip ``HasVote`` messages to hint peers
-   what votes it already has.
--  Nodes broadcast their current state to all neighboring peers. (but is
-   not gossiped further)
-
-There's more, but let's not get ahead of ourselves here.
-
-Proposals
----------
-
-A proposal is signed and published by the designated proposer at each
-round. The proposer is chosen by a deterministic and non-choking round
-robin selection algorithm that selects proposers in proportion to their
-voting power. (see
-`implementation <https://github.com/tendermint/tendermint/blob/develop/types/validator_set.go>`__)
-
-A proposal at ``(H,R)`` is composed of a block and an optional latest
-``PoLC-Round < R`` which is included iff the proposer knows of one. This
-hints the network to allow nodes to unlock (when safe) to ensure the
-liveness property.
-
-State Machine Spec
-------------------
-
-Propose Step (height:H,round:R)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upon entering ``Propose``: - The designated proposer proposes a block at
-``(H,R)``.
-
-The ``Propose`` step ends: - After ``timeoutProposeR`` after entering
-``Propose``. --> goto ``Prevote(H,R)`` - After receiving proposal block
-and all prevotes at ``PoLC-Round``. --> goto ``Prevote(H,R)`` - After
-`common exit conditions <#common-exit-conditions>`__
-
-Prevote Step (height:H,round:R)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upon entering ``Prevote``, each validator broadcasts its prevote vote.
-
--  First, if the validator is locked on a block since ``LastLockRound``
-   but now has a PoLC for something else at round ``PoLC-Round`` where
-   ``LastLockRound < PoLC-Round < R``, then it unlocks.
--  If the validator is still locked on a block, it prevotes that.
--  Else, if the proposed block from ``Propose(H,R)`` is good, it
-   prevotes that.
--  Else, if the proposal is invalid or wasn't received on time, it
-   prevotes ``<nil>``.
-
-The ``Prevote`` step ends: - After +2/3 prevotes for a particular block
-or ``<nil>``. --> goto ``Precommit(H,R)`` - After ``timeoutPrevote``
-after receiving any +2/3 prevotes. --> goto ``Precommit(H,R)`` - After
-`common exit conditions <#common-exit-conditions>`__
-
-Precommit Step (height:H,round:R)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upon entering ``Precommit``, each validator broadcasts its precommit
-vote. - If the validator has a PoLC at ``(H,R)`` for a particular block
-``B``, it (re)locks (or changes lock to) and precommits ``B`` and sets
-``LastLockRound = R``. - Else, if the validator has a PoLC at ``(H,R)``
-for ``<nil>``, it unlocks and precommits ``<nil>``. - Else, it keeps the
-lock unchanged and precommits ``<nil>``.
-
-A precommit for ``<nil>`` means "I didn’t see a PoLC for this round, but
-I did get +2/3 prevotes and waited a bit".
-
-The Precommit step ends: - After +2/3 precommits for ``<nil>``. --> goto
-``Propose(H,R+1)`` - After ``timeoutPrecommit`` after receiving any +2/3
-precommits. --> goto ``Propose(H,R+1)`` - After `common exit
-conditions <#common-exit-conditions>`__
-
-common exit conditions
-^^^^^^^^^^^^^^^^^^^^^^
-
--  After +2/3 precommits for a particular block. --> goto ``Commit(H)``
--  After any +2/3 prevotes received at ``(H,R+x)``. --> goto
-   ``Prevote(H,R+x)``
--  After any +2/3 precommits received at ``(H,R+x)``. --> goto
-   ``Precommit(H,R+x)``
-
-Commit Step (height:H)
-~~~~~~~~~~~~~~~~~~~~~~
-
--  Set ``CommitTime = now()``
--  Wait until block is received. --> goto ``NewHeight(H+1)``
-
-NewHeight Step (height:H)
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
--  Move ``Precommits`` to ``LastCommit`` and increment height.
--  Set ``StartTime = CommitTime+timeoutCommit``
--  Wait until ``StartTime`` to receive straggler commits. --> goto
-   ``Propose(H,0)``
-
-Proofs
-------
-
-Proof of Safety
-~~~~~~~~~~~~~~~
-
-Assume that at most -1/3 of the voting power of validators is byzantine.
-If a validator commits block ``B`` at round ``R``, it's because it saw
-+2/3 of precommits at round ``R``. This implies that 1/3+ of honest
-nodes are still locked at round ``R' > R``. These locked validators will
-remain locked until they see a PoLC at ``R' > R``, but this won't happen
-because 1/3+ are locked and honest, so at most -2/3 are available to
-vote for anything other than ``B``.
-
-Proof of Liveness
-~~~~~~~~~~~~~~~~~
-
-If 1/3+ honest validators are locked on two different blocks from
-different rounds, a proposers' ``PoLC-Round`` will eventually cause
-nodes locked from the earlier round to unlock. Eventually, the
-designated proposer will be one that is aware of a PoLC at the later
-round. Also, ``timeoutProposalR`` increments with round ``R``, while the
-size of a proposal are capped, so eventually the network is able to
-"fully gossip" the whole proposal (e.g. the block & PoLC).
-
-Proof of Fork Accountability
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Define the JSet (justification-vote-set) at height ``H`` of a validator
-``V1`` to be all the votes signed by the validator at ``H`` along with
-justification PoLC prevotes for each lock change. For example, if ``V1``
-signed the following precommits: ``Precommit(B1 @ round 0)``,
-``Precommit(<nil> @ round 1)``, ``Precommit(B2 @ round 4)`` (note that
-no precommits were signed for rounds 2 and 3, and that's ok),
-``Precommit(B1 @ round 0)`` must be justified by a PoLC at round 0, and
-``Precommit(B2 @ round 4)`` must be justified by a PoLC at round 4; but
-the precommit for ``<nil>`` at round 1 is not a lock-change by
-definition so the JSet for ``V1`` need not include any prevotes at round
-1, 2, or 3 (unless ``V1`` happened to have prevoted for those rounds).
-
-Further, define the JSet at height ``H`` of a set of validators ``VSet``
-to be the union of the JSets for each validator in ``VSet``. For a given
-commit by honest validators at round ``R`` for block ``B`` we can
-construct a JSet to justify the commit for ``B`` at ``R``. We say that a
-JSet *justifies* a commit at ``(H,R)`` if all the committers (validators
-in the commit-set) are each justified in the JSet with no duplicitous
-vote signatures (by the committers).
-
--  **Lemma**: When a fork is detected by the existence of two
-   conflicting `commits <./validators.html#commiting-a-block>`__,
-   the union of the JSets for both commits (if they can be compiled)
-   must include double-signing by at least 1/3+ of the validator set.
-   **Proof**: The commit cannot be at the same round, because that would
-   immediately imply double-signing by 1/3+. Take the union of the JSets
-   of both commits. If there is no double-signing by at least 1/3+ of
-   the validator set in the union, then no honest validator could have
-   precommitted any different block after the first commit. Yet, +2/3
-   did. Reductio ad absurdum.
-
-As a corollary, when there is a fork, an external process can determine
-the blame by requiring each validator to justify all of its round votes.
-Either we will find 1/3+ who cannot justify at least one of their votes,
-and/or, we will find 1/3+ who had double-signed.
-
-Alternative algorithm
-~~~~~~~~~~~~~~~~~~~~~
-
-Alternatively, we can take the JSet of a commit to be the "full commit".
-That is, if light clients and validators do not consider a block to be
-committed unless the JSet of the commit is also known, then we get the
-desirable property that if there ever is a fork (e.g. there are two
-conflicting "full commits"), then 1/3+ of the validators are immediately
-punishable for double-signing.
-
-There are many ways to ensure that the gossip network efficiently share
-the JSet of a commit. One solution is to add a new message type that
-tells peers that this node has (or does not have) a +2/3 majority for B
-(or ) at (H,R), and a bitarray of which votes contributed towards that
-majority. Peers can react by responding with appropriate votes.
-
-We will implement such an algorithm for the next iteration of the
-Tendermint consensus protocol.
-
-Other potential improvements include adding more data in votes such as
-the last known PoLC round that caused a lock change, and the last voted
-round/step (or, we may require that validators not skip any votes). This
-may make JSet verification/gossip logic easier to implement.
-
-Censorship Attacks
-~~~~~~~~~~~~~~~~~~
-
-Due to the definition of a block
-`commit <validators.html#commiting-a-block>`__, any 1/3+
-coalition of validators can halt the blockchain by not broadcasting
-their votes. Such a coalition can also censor particular transactions by
-rejecting blocks that include these transactions, though this would
-result in a significant proportion of block proposals to be rejected,
-which would slow down the rate of block commits of the blockchain,
-reducing its utility and value. The malicious coalition might also
-broadcast votes in a trickle so as to grind blockchain block commits to
-a near halt, or engage in any combination of these attacks.
-
-If a global active adversary were also involved, it can partition the
-network in such a way that it may appear that the wrong subset of
-validators were responsible for the slowdown. This is not just a
-limitation of Tendermint, but rather a limitation of all consensus
-protocols whose network is potentially controlled by an active
-adversary.
-
-Overcoming Forks and Censorship Attacks
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-For these types of attacks, a subset of the validators through external
-means should coordinate to sign a reorg-proposal that chooses a fork
-(and any evidence thereof) and the initial subset of validators with
-their signatures. Validators who sign such a reorg-proposal forego its
-collateral on all other forks. Clients should verify the signatures on
-the reorg-proposal, verify any evidence, and make a judgement or prompt
-the end-user for a decision. For example, a phone wallet app may prompt
-the user with a security warning, while a refrigerator may accept any
-reorg-proposal signed by +1/2 of the original validators.
-
-No non-synchronous Byzantine fault-tolerant algorithm can come to
-consensus when 1/3+ of validators are dishonest, yet a fork assumes that
-1/3+ of validators have already been dishonest by double-signing or
-lock-changing without justification. So, signing the reorg-proposal is a
-coordination problem that cannot be solved by any non-synchronous
-protocol (i.e. automatically, and without making assumptions about the
-reliability of the underlying network). It must be provided by means
-external to the weakly-synchronous Tendermint consensus algorithm. For
-now, we leave the problem of reorg-proposal coordination to human
-coordination via internet media. Validators must take care to ensure
-that there are no significant network partitions, to avoid situations
-where two conflicting reorg-proposals are signed.
-
-Assuming that the external coordination medium and protocol is robust,
-it follows that forks are less of a concern than `censorship
-attacks <#censorship-attacks>`__.

docs/specification/corruption.rst

@@ -1,70 +0,0 @@
-Corruption
-==========
-
-Important step
---------------
-
-Make sure you have a backup of the Tendermint data directory.
-
-Possible causes
----------------
-
-Remember that most corruption is caused by hardware issues:
-
-- RAID controllers with faulty / worn out battery backup, and an unexpected power loss
-- Hard disk drives with write-back cache enabled, and an unexpected power loss
-- Cheap SSDs with insufficient power-loss protection, and an unexpected power-loss
-- Defective RAM
-- Defective or overheating CPU(s)
-
-Other causes can be:
-
-- Database systems configured with fsync=off and an OS crash or power loss
-- Filesystems configured to use write barriers plus a storage layer that ignores write barriers. LVM is a particular culprit.
-- Tendermint bugs
-- Operating system bugs
-- Admin error
-  - directly modifying Tendermint data-directory contents
-
-(Source: https://wiki.postgresql.org/wiki/Corruption)
-
-WAL Corruption
---------------
-
-If consensus WAL is corrupted at the lastest height and you are trying to start
-Tendermint, replay will fail with panic.
-
-Recovering from data corruption can be hard and time-consuming. Here are two approaches you can take:
-
-1) Delete the WAL file and restart Tendermint. It will attempt to sync with other peers.
-2) Try to repair the WAL file manually:
-
-  1. Create a backup of the corrupted WAL file:
-
-  .. code:: bash
-
-      cp "$TMHOME/data/cs.wal/wal" > /tmp/corrupted_wal_backup
-
-  2. Use ./scripts/wal2json to create a human-readable version
-
-  .. code:: bash
-
-      ./scripts/wal2json/wal2json "$TMHOME/data/cs.wal/wal" > /tmp/corrupted_wal
-
-  3. Search for a "CORRUPTED MESSAGE" line.
-  4. By looking at the previous message and the message after the corrupted one
-       and looking at the logs, try to rebuild the message. If the consequent
-       messages are marked as corrupted too (this may happen if length header
-       got corrupted or some writes did not make it to the WAL ~ truncation),
-       then remove all the lines starting from the corrupted one and restart
-       Tendermint.
-
-  .. code:: bash
-
-      $EDITOR /tmp/corrupted_wal
-
-  5. After editing, convert this file back into binary form by running:
-
-  .. code:: bash
-
-      ./scripts/json2wal/json2wal /tmp/corrupted_wal > "$TMHOME/data/cs.wal/wal"

docs/specification/genesis.rst

@@ -1,71 +0,0 @@
-Genesis
-=======
-
-The genesis.json file in ``$TMHOME/config`` defines the initial TendermintCore
-state upon genesis of the blockchain (`see
-definition <https://github.com/tendermint/tendermint/blob/master/types/genesis.go>`__).
-
-Fields
-~~~~~~
-
--  ``genesis_time``: Official time of blockchain start.
--  ``chain_id``: ID of the blockchain. This must be unique for every
-   blockchain. If your testnet blockchains do not have unique chain IDs,
-   you will have a bad time.
--  ``validators``:
--  ``pub_key``: The first element specifies the pub\_key type. 1 ==
-   Ed25519. The second element are the pubkey bytes.
--  ``power``: The validator's voting power.
--  ``name``: Name of the validator (optional).
--  ``app_hash``: The expected application hash (as returned by the
-   ``ResponseInfo`` ABCI message) upon genesis. If the app's hash does not
-   match, Tendermint will panic.
--  ``app_state``: The application state (e.g. initial distribution of tokens).
-
-Sample genesis.json
-~~~~~~~~~~~~~~~~~~~
-
-.. code:: json
-
-    {
-      "genesis_time": "2016-02-05T06:02:31.526Z",
-      "chain_id": "chain-tTH4mi",
-      "validators": [
-        {
-          "pub_key": [
-            1,
-            "9BC5112CB9614D91CE423FA8744885126CD9D08D9FC9D1F42E552D662BAA411E"
-          ],
-          "power": 1,
-          "name": "mach1"
-        },
-        {
-          "pub_key": [
-            1,
-            "F46A5543D51F31660D9F59653B4F96061A740FF7433E0DC1ECBC30BE8494DE06"
-          ],
-          "power": 1,
-          "name": "mach2"
-        },
-        {
-          "pub_key": [
-            1,
-            "0E7B423C1635FD07C0FC3603B736D5D27953C1C6CA865BB9392CD79DE1A682BB"
-          ],
-          "power": 1,
-          "name": "mach3"
-        },
-        {
-          "pub_key": [
-            1,
-            "4F49237B9A32EB50682EDD83C48CE9CDB1D02A7CFDADCFF6EC8C1FAADB358879"
-          ],
-          "power": 1,
-          "name": "mach4"
-        }
-      ],
-      "app_hash": "15005165891224E721CB664D15CB972240F5703F",
-      "app_state": {
-        {"account": "Bob", "coins": 5000}
-      }
-    }

docs/specification/light-client-protocol.rst

@@ -1,33 +0,0 @@
-Light Client Protocol
-=====================
-
-Light clients are an important part of the complete blockchain system
-for most applications. Tendermint provides unique speed and security
-properties for light client applications.
-
-See our `lite package
-<https://godoc.org/github.com/tendermint/tendermint/lite>`__.
-
-Overview
---------
-
-The objective of the light client protocol is to get a
-`commit <./validators.html#committing-a-block>`__ for a recent
-`block hash <./block-structure.html#block-hash>`__ where the commit
-includes a majority of signatures from the last known validator set.
-From there, all the application state is verifiable with `merkle
-proofs <./merkle.html#iavl-tree>`__.
-
-Properties
-----------
-
--  You get the full collateralized security benefits of Tendermint; No
-   need to wait for confirmations.
--  You get the full speed benefits of Tendermint; transactions commit
-   instantly.
--  You can get the most recent version of the application state
-   non-interactively (without committing anything to the blockchain).
-   For example, this means that you can get the most recent value of a
-   name from the name-registry without worrying about fork censorship
-   attacks, without posting a commit and waiting for confirmations. It's
-   fast, secure, and free!

docs/specification/merkle.rst

@@ -1,88 +0,0 @@
-Merkle
-======
-
-For an overview of Merkle trees, see
-`wikipedia <https://en.wikipedia.org/wiki/Merkle_tree>`__.
-
-There are two types of Merkle trees used in Tendermint.
-
--  **IAVL+ Tree**: An immutable self-balancing binary
-   tree for persistent application state
--  **Simple Tree**: A simple compact binary tree for
-   a static list of items
-
-IAVL+ Tree
-----------
-
-The purpose of this data structure is to provide persistent storage for
-key-value pairs (e.g. account state, name-registrar data, and
-per-contract data) such that a deterministic merkle root hash can be
-computed. The tree is balanced using a variant of the `AVL
-algorithm <http://en.wikipedia.org/wiki/AVL_tree>`__ so all operations
-are O(log(n)).
-
-Nodes of this tree are immutable and indexed by its hash. Thus any node
-serves as an immutable snapshot which lets us stage uncommitted
-transactions from the mempool cheaply, and we can instantly roll back to
-the last committed state to process transactions of a newly committed
-block (which may not be the same set of transactions as those from the
-mempool).
-
-In an AVL tree, the heights of the two child subtrees of any node differ
-by at most one. Whenever this condition is violated upon an update, the
-tree is rebalanced by creating O(log(n)) new nodes that point to
-unmodified nodes of the old tree. In the original AVL algorithm, inner
-nodes can also hold key-value pairs. The AVL+ algorithm (note the plus)
-modifies the AVL algorithm to keep all values on leaf nodes, while only
-using branch-nodes to store keys. This simplifies the algorithm while
-minimizing the size of merkle proofs
-
-In Ethereum, the analog is the `Patricia
-trie <http://en.wikipedia.org/wiki/Radix_tree>`__. There are tradeoffs.
-Keys do not need to be hashed prior to insertion in IAVL+ trees, so this
-provides faster iteration in the key space which may benefit some
-applications. The logic is simpler to implement, requiring only two
-types of nodes -- inner nodes and leaf nodes. The IAVL+ tree is a binary
-tree, so merkle proofs are much shorter than the base 16 Patricia trie.
-On the other hand, while IAVL+ trees provide a deterministic merkle root
-hash, it depends on the order of updates. In practice this shouldn't be
-a problem, since you can efficiently encode the tree structure when
-serializing the tree contents.
-
-Simple Tree
------------
-
-For merkelizing smaller static lists, use the Simple Tree. The
-transactions and validation signatures of a block are hashed using this
-simple merkle tree logic.
-
-If the number of items is not a power of two, the tree will not be full
-and some leaf nodes will be at different levels. Simple Tree tries to
-keep both sides of the tree the same size, but the left side may be one
-greater.
-
-::
-
-        Simple Tree with 6 items           Simple Tree with 7 items 
-                                                             
-                   *                                  *             
-                  / \                                / \            
-                /     \                            /     \          
-              /         \                        /         \        
-            /             \                    /             \      
-           *               *                  *               *     
-          / \             / \                / \             / \    
-         /   \           /   \              /   \           /   \   
-        /     \         /     \            /     \         /     \  
-       *       h2      *       h5         *       *       *       h6
-      / \             / \                / \     / \     / \        
-     h0  h1          h3  h4             h0  h1  h2  h3  h4  h5      
-
-Simple Tree with Dictionaries
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The Simple Tree is used to merkelize a list of items, so to merkelize a
-(short) dictionary of key-value pairs, encode the dictionary as an
-ordered list of ``KVPair`` structs. The block hash is such a hash
-derived from all the fields of the block ``Header``. The state hash is
-similarly derived.

docs/specification/new-spec/README.md

@@ -1 +0,0 @@
-Spec moved to [docs/spec](https://github.com/tendermint/tendermint/tree/master/docs/spec).

docs/specification/wire-protocol.rst

@@ -1,172 +0,0 @@
-Wire Protocol
-=============
-
-The `Tendermint wire protocol <https://github.com/tendermint/go-wire>`__
-encodes data in `c-style binary <#binary>`__ and `JSON <#json>`__ form.
-
-Supported types
----------------
-
--  Primitive types
--  ``uint8`` (aka ``byte``), ``uint16``, ``uint32``, ``uint64``
--  ``int8``, ``int16``, ``int32``, ``int64``
--  ``uint``, ``int``: variable length (un)signed integers
--  ``string``, ``[]byte``
--  ``time``
--  Derived types
--  structs
--  var-length arrays of a particular type
--  fixed-length arrays of a particular type
--  interfaces: registered union types preceded by a ``type byte``
--  pointers
-
-Binary
-------
-
-**Fixed-length primitive types** are encoded with 1,2,3, or 4 big-endian
-bytes. - ``uint8`` (aka ``byte``), ``uint16``, ``uint32``, ``uint64``:
-takes 1,2,3, and 4 bytes respectively - ``int8``, ``int16``, ``int32``,
-``int64``: takes 1,2,3, and 4 bytes respectively - ``time``: ``int64``
-representation of nanoseconds since epoch
-
-**Variable-length integers** are encoded with a single leading byte
-representing the length of the following big-endian bytes. For signed
-negative integers, the most significant bit of the leading byte is a 1.
-
--  ``uint``: 1-byte length prefixed variable-size (0 ~ 255 bytes)
-   unsigned integers
--  ``int``: 1-byte length prefixed variable-size (0 ~ 127 bytes) signed
-   integers
-
-NOTE: While the number 0 (zero) is encoded with a single byte ``x00``,
-the number 1 (one) takes two bytes to represent: ``x0101``. This isn't
-the most efficient representation, but the rules are easier to remember.
-
-+---------------+----------------+----------------+
-| number        | binary         | binary ``int`` |
-|               | ``uint``       |                |
-+===============+================+================+
-| 0             | ``x00``        | ``x00``        |
-+---------------+----------------+----------------+
-| 1             | ``x0101``      | ``x0101``      |
-+---------------+----------------+----------------+
-| 2             | ``x0102``      | ``x0102``      |
-+---------------+----------------+----------------+
-| 256           | ``x020100``    | ``x020100``    |
-+---------------+----------------+----------------+
-| 2^(127\ *8)-1 | ``x800100...`` | overflow       |
-| \|            |                |                |
-| ``x7FFFFF...` |                |                |
-| `             |                |                |
-| \|            |                |                |
-| ``x7FFFFF...` |                |                |
-| `             |                |                |
-| \| \|         |                |                |
-| 2^(127*\ 8)   |                |                |
-+---------------+----------------+----------------+
-| 2^(255\*8)-1  |
-| \|            |
-| ``xFFFFFF...` |
-| `             |
-| \| overflow   |
-| \| \| -1 \|   |
-| n/a \|        |
-| ``x8101`` \|  |
-| \| -2 \| n/a  |
-| \| ``x8102``  |
-| \| \| -256 \| |
-| n/a \|        |
-| ``x820100``   |
-| \|            |
-+---------------+----------------+----------------+
-
-**Structures** are encoded by encoding the field values in order of
-declaration.
-
-.. code:: go
-
-    type Foo struct {
-        MyString string
-        MyUint32 uint32
-    }
-    var foo = Foo{"626172", math.MaxUint32}
-
-    /* The binary representation of foo:
-     0103626172FFFFFFFF
-     0103:               `int` encoded length of string, here 3
-         626172:         3 bytes of string "bar"
-               FFFFFFFF: 4 bytes of uint32 MaxUint32
-    */
-
-**Variable-length arrays** are encoded with a leading ``int`` denoting
-the length of the array followed by the binary representation of the
-items. **Fixed-length arrays** are similar but aren't preceded by the
-leading ``int``.
-
-.. code:: go
-
-    foos := []Foo{foo, foo}
-
-    /* The binary representation of foos:
-     01020103626172FFFFFFFF0103626172FFFFFFFF
-     0102:                                     `int` encoded length of array, here 2
-         0103626172FFFFFFFF:                   the first `foo`
-                           0103626172FFFFFFFF: the second `foo`
-    */
-
-    foos := [2]Foo{foo, foo} // fixed-length array
-
-    /* The binary representation of foos:
-     0103626172FFFFFFFF0103626172FFFFFFFF
-     0103626172FFFFFFFF:                   the first `foo`
-                       0103626172FFFFFFFF: the second `foo`
-    */
-
-**Interfaces** can represent one of any number of concrete types. The
-concrete types of an interface must first be declared with their
-corresponding ``type byte``. An interface is then encoded with the
-leading ``type byte``, then the binary encoding of the underlying
-concrete type.
-
-NOTE: The byte ``x00`` is reserved for the ``nil`` interface value and
-``nil`` pointer values.
-
-.. code:: go
-
-    type Animal interface{}
-    type Dog uint32
-    type Cat string
-
-    RegisterInterface(
-        struct{ Animal }{},          // Convenience for referencing the 'Animal' interface
-        ConcreteType{Dog(0),  0x01}, // Register the byte 0x01 to denote a Dog
-        ConcreteType{Cat(""), 0x02}, // Register the byte 0x02 to denote a Cat
-    )
-
-    var animal Animal = Dog(02)
-
-    /* The binary representation of animal:
-     010102
-     01:     the type byte for a `Dog`
-       0102: the bytes of Dog(02)
-    */
-
-**Pointers** are encoded with a single leading byte ``x00`` for ``nil``
-pointers, otherwise encoded with a leading byte ``x01`` followed by the
-binary encoding of the value pointed to.
-
-NOTE: It's easy to convert pointer types into interface types, since the
-``type byte`` ``x00`` is always ``nil``.
-
-JSON
-----
-
-The JSON codec is compatible with the ```binary`` <#binary>`__ codec,
-and is fairly intuitive if you're already familiar with golang's JSON
-encoding. Some quirks are noted below:
-
--  variable-length and fixed-length bytes are encoded as uppercase
-   hexadecimal strings
--  interface values are encoded as an array of two items:
-   ``[type_byte, concrete_value]``
--  times are encoded as rfc2822 strings

docs/tendermint-core/block-structure.md

@@ -0,0 +1,206 @@
+# Block Structure
+
+The tendermint consensus engine records all agreements by a
+supermajority of nodes into a blockchain, which is replicated among all
+nodes. This blockchain is accessible via various rpc endpoints, mainly
+`/block?height=` to get the full block, as well as
+`/blockchain?minHeight=_&maxHeight=_` to get a list of headers. But what
+exactly is stored in these blocks?
+
+## Block
+
+A
+[Block](https://godoc.org/github.com/tendermint/tendermint/types#Block)
+contains:
+
+-   a [Header](#header) contains merkle hashes for various chain states
+-   the
+    [Data](https://godoc.org/github.com/tendermint/tendermint/types#Data)
+    is all transactions which are to be processed
+-   the [LastCommit](#commit) > 2/3 signatures for the last block
+
+The signatures returned along with block `H` are those validating block
+`H-1`. This can be a little confusing, but we must also consider that
+the `Header` also contains the `LastCommitHash`. It would be impossible
+for a Header to include the commits that sign it, as it would cause an
+infinite loop here. But when we get block `H`, we find
+`Header.LastCommitHash`, which must match the hash of `LastCommit`.
+
+## Header
+
+The
+[Header](https://godoc.org/github.com/tendermint/tendermint/types#Header)
+contains lots of information (follow link for up-to-date info). Notably,
+it maintains the `Height`, the `LastBlockID` (to make it a chain), and
+hashes of the data, the app state, and the validator set. This is
+important as the only item that is signed by the validators is the
+`Header`, and all other data must be validated against one of the merkle
+hashes in the `Header`.
+
+The `DataHash` can provide a nice check on the
+[Data](https://godoc.org/github.com/tendermint/tendermint/types#Data)
+returned in this same block. If you are subscribed to new blocks, via
+tendermint RPC, in order to display or process the new transactions you
+should at least validate that the `DataHash` is valid. If it is
+important to verify autheniticity, you must wait for the `LastCommit`
+from the next block to make sure the block header (including `DataHash`)
+was properly signed.
+
+The `ValidatorHash` contains a hash of the current
+[Validators](https://godoc.org/github.com/tendermint/tendermint/types#Validator).
+Tracking all changes in the validator set is complex, but a client can
+quickly compare this hash with the [hash of the currently known
+validators](https://godoc.org/github.com/tendermint/tendermint/types#ValidatorSet.Hash)
+to see if there have been changes.
+
+The `AppHash` serves as the basis for validating any merkle proofs that
+come from the ABCI application. It represents the state of the actual
+application, rather that the state of the blockchain itself. This means
+it's necessary in order to perform any business logic, such as verifying
+an account balance.
+
+**Note** After the transactions are committed to a block, they still
+need to be processed in a separate step, which happens between the
+blocks. If you find a given transaction in the block at height `H`, the
+effects of running that transaction will be first visible in the
+`AppHash` from the block header at height `H+1`.
+
+Like the `LastCommit` issue, this is a requirement of the immutability
+of the block chain, as the application only applies transactions *after*
+they are commited to the chain.
+
+## Commit
+
+The
+[Commit](https://godoc.org/github.com/tendermint/tendermint/types#Commit)
+contains a set of
+[Votes](https://godoc.org/github.com/tendermint/tendermint/types#Vote)
+that were made by the validator set to reach consensus on this block.
+This is the key to the security in any PoS system, and actually no data
+that cannot be traced back to a block header with a valid set of Votes
+can be trusted. Thus, getting the Commit data and verifying the votes is
+extremely important.
+
+As mentioned above, in order to find the `precommit votes` for block
+header `H`, we need to query block `H+1`. Then we need to check the
+votes, make sure they really are for that block, and properly formatted.
+Much of this code is implemented in Go in the
+[light-client](https://github.com/tendermint/light-client) package. If
+you look at the code, you will notice that we need to provide the
+`chainID` of the blockchain in order to properly calculate the votes.
+This is to protect anyone from swapping votes between chains to fake (or
+frame) a validator. Also note that this `chainID` is in the
+`genesis.json` from *Tendermint*, not the `genesis.json` from the
+basecoin app ([that is a different
+chainID...](https://github.com/cosmos/cosmos-sdk/issues/32)).
+
+Once we have those votes, and we calculated the proper [sign
+bytes](https://godoc.org/github.com/tendermint/tendermint/types#Vote.WriteSignBytes)
+using the chainID and a [nice helper
+function](https://godoc.org/github.com/tendermint/tendermint/types#SignBytes),
+we can verify them. The light client is responsible for maintaining a
+set of validators that we trust. Each vote only stores the validators
+`Address`, as well as the `Signature`. Assuming we have a local copy of
+the trusted validator set, we can look up the `Public Key` of the
+validator given its `Address`, then verify that the `Signature` matches
+the `SignBytes` and `Public Key`. Then we sum up the total voting power
+of all validators, whose votes fulfilled all these stringent
+requirements. If the total number of voting power for a single block is
+greater than 2/3 of all voting power, then we can finally trust the
+block header, the AppHash, and the proof we got from the ABCI
+application.
+
+### Vote Sign Bytes
+
+The `sign-bytes` of a vote is produced by taking a
+[stable-json](https://github.com/substack/json-stable-stringify)-like
+deterministic JSON [wire](./wire-protocol.html) encoding of the vote
+(excluding the `Signature` field), and wrapping it with
+`{"chain_id":"my_chain","vote":...}`.
+
+For example, a precommit vote might have the following `sign-bytes`:
+
+```
+{"chain_id":"my_chain","vote":{"block_hash":"611801F57B4CE378DF1A3FFF1216656E89209A99","block_parts_header":{"hash":"B46697379DBE0774CC2C3B656083F07CA7E0F9CE","total":123},"height":1234,"round":1,"type":2}}
+```
+
+## Block Hash
+
+The [block
+hash](https://godoc.org/github.com/tendermint/tendermint/types#Block.Hash)
+is the [Simple Tree hash](./merkle.html#simple-tree-with-dictionaries)
+of the fields of the block `Header` encoded as a list of `KVPair`s.
+
+## Transaction
+
+A transaction is any sequence of bytes. It is up to your ABCI
+application to accept or reject transactions.
+
+## BlockID
+
+Many of these data structures refer to the
+[BlockID](https://godoc.org/github.com/tendermint/tendermint/types#BlockID),
+which is the `BlockHash` (hash of the block header, also referred to by
+the next block) along with the `PartSetHeader`. The `PartSetHeader` is
+explained below and is used internally to orchestrate the p2p
+propogation. For clients, it is basically opaque bytes, but they must
+match for all votes.
+
+## PartSetHeader
+
+The
+[PartSetHeader](https://godoc.org/github.com/tendermint/tendermint/types#PartSetHeader)
+contains the total number of pieces in a
+[PartSet](https://godoc.org/github.com/tendermint/tendermint/types#PartSet),
+and the Merkle root hash of those pieces.
+
+## PartSet
+
+PartSet is used to split a byteslice of data into parts (pieces) for
+transmission. By splitting data into smaller parts and computing a
+Merkle root hash on the list, you can verify that a part is legitimately
+part of the complete data, and the part can be forwarded to other peers
+before all the parts are known. In short, it's a fast way to securely
+propagate a large chunk of data (like a block) over a gossip network.
+
+PartSet was inspired by the LibSwift project.
+
+Usage:
+
+```
+data := RandBytes(2 << 20) // Something large
+
+partSet := NewPartSetFromData(data)
+partSet.Total()     // Total number of 4KB parts
+partSet.Count()     // Equal to the Total, since we already have all the parts
+partSet.Hash()      // The Merkle root hash
+partSet.BitArray()  // A BitArray of partSet.Total() 1's
+
+header := partSet.Header() // Send this to the peer
+header.Total        // Total number of parts
+header.Hash         // The merkle root hash
+
+// Now we'll reconstruct the data from the parts
+partSet2 := NewPartSetFromHeader(header)
+partSet2.Total()    // Same total as partSet.Total()
+partSet2.Count()    // Zero, since this PartSet doesn't have any parts yet.
+partSet2.Hash()     // Same hash as in partSet.Hash()
+partSet2.BitArray() // A BitArray of partSet.Total() 0's
+
+// In a gossip network the parts would arrive in arbitrary order, perhaps
+// in response to explicit requests for parts, or optimistically in response
+// to the receiving peer's partSet.BitArray().
+for !partSet2.IsComplete() {
+    part := receivePartFromGossipNetwork()
+    added, err := partSet2.AddPart(part)
+    if err != nil {
+    // A wrong part,
+        // the merkle trail does not hash to partSet2.Hash()
+    } else if !added {
+        // A duplicate part already received
+    }
+}
+
+data2, _ := ioutil.ReadAll(partSet2.GetReader())
+bytes.Equal(data, data2) // true
+```

docs/tendermint-core/configuration.md

@@ -99,7 +99,6 @@ laddr = "tcp://0.0.0.0:26656"
 seeds = ""
 
 # Comma separated list of nodes to keep persistent connections to
-# Do not add private peers to this list if you don't want them advertised
 persistent_peers = ""
 
 # UPNP port forwarding
@@ -121,10 +120,10 @@ max_num_peers = 50
 max_packet_msg_payload_size = 1024
 
 # Rate at which packets can be sent, in bytes/second
-send_rate = 512000
+send_rate = 5120000
 
 # Rate at which packets can be received, in bytes/second
-recv_rate = 512000
+recv_rate = 5120000
 
 # Set true to enable the peer-exchange reactor
 pex = true

docs/tendermint-core/light-client-protocol.md

@@ -0,0 +1,30 @@
+# Light Client Protocol
+
+Light clients are an important part of the complete blockchain system
+for most applications. Tendermint provides unique speed and security
+properties for light client applications.
+
+See our [lite
+package](https://godoc.org/github.com/tendermint/tendermint/lite).
+
+## Overview
+
+The objective of the light client protocol is to get a
+[commit](./validators.md#committing-a-block) for a recent [block
+hash](../spec/consensus/consensus.md.md#block-hash) where the commit includes a
+majority of signatures from the last known validator set. From there,
+all the application state is verifiable with [merkle
+proofs](./merkle.md#iavl-tree).
+
+## Properties
+
+-   You get the full collateralized security benefits of Tendermint; No
+    need to wait for confirmations.
+-   You get the full speed benefits of Tendermint; transactions
+    commit instantly.
+-   You can get the most recent version of the application state
+    non-interactively (without committing anything to the blockchain).
+    For example, this means that you can get the most recent value of a
+    name from the name-registry without worrying about fork censorship
+    attacks, without posting a commit and waiting for confirmations.
+    It's fast, secure, and free!

docs/tendermint-core/running-in-production.md

@@ -104,6 +104,69 @@ signals we use the default behaviour in Go: [Default behavior of signals
 in Go
 programs](https://golang.org/pkg/os/signal/#hdr-Default_behavior_of_signals_in_Go_programs).
 
+## Corruption
+
+**NOTE:** Make sure you have a backup of the Tendermint data directory.
+
+### Possible causes
+
+Remember that most corruption is caused by hardware issues:
+
+- RAID controllers with faulty / worn out battery backup, and an unexpected power loss
+- Hard disk drives with write-back cache enabled, and an unexpected power loss
+- Cheap SSDs with insufficient power-loss protection, and an unexpected power-loss
+- Defective RAM
+- Defective or overheating CPU(s)
+
+Other causes can be:
+
+- Database systems configured with fsync=off and an OS crash or power loss
+- Filesystems configured to use write barriers plus a storage layer that ignores write barriers. LVM is a particular culprit.
+- Tendermint bugs
+- Operating system bugs
+- Admin error (e.g., directly modifying Tendermint data-directory contents)
+
+(Source: https://wiki.postgresql.org/wiki/Corruption)
+
+### WAL Corruption
+
+If consensus WAL is corrupted at the lastest height and you are trying to start
+Tendermint, replay will fail with panic.
+
+Recovering from data corruption can be hard and time-consuming. Here are two approaches you can take:
+
+1) Delete the WAL file and restart Tendermint. It will attempt to sync with other peers.
+2) Try to repair the WAL file manually:
+
+  1. Create a backup of the corrupted WAL file:
+
+```
+cp "$TMHOME/data/cs.wal/wal" > /tmp/corrupted_wal_backup
+```
+
+  2. Use `./scripts/wal2json` to create a human-readable version
+
+```
+./scripts/wal2json/wal2json "$TMHOME/data/cs.wal/wal" > /tmp/corrupted_wal
+```
+
+  3. Search for a "CORRUPTED MESSAGE" line.
+  4. By looking at the previous message and the message after the corrupted one
+       and looking at the logs, try to rebuild the message. If the consequent
+       messages are marked as corrupted too (this may happen if length header
+       got corrupted or some writes did not make it to the WAL ~ truncation),
+       then remove all the lines starting from the corrupted one and restart
+       Tendermint.
+
+```
+$EDITOR /tmp/corrupted_wal
+```
+  5. After editing, convert this file back into binary form by running:
+
+```
+./scripts/json2wal/json2wal /tmp/corrupted_wal > "$TMHOME/data/cs.wal/wal"
+```
+
 ## Hardware
 
 ### Processor and Memory

docs/tendermint-core/secure-p2p.md

@@ -1,12 +1,11 @@
-Secure P2P
-==========
+# Secure P2P
 
 The Tendermint p2p protocol uses an authenticated encryption scheme
-based on the `Station-to-Station
-Protocol <https://en.wikipedia.org/wiki/Station-to-Station_protocol>`__.
+based on the [Station-to-Station
+Protocol](https://en.wikipedia.org/wiki/Station-to-Station_protocol).
 The implementation uses
-`golang's <https://godoc.org/golang.org/x/crypto/nacl/box>`__ `nacl
-box <http://nacl.cr.yp.to/box.html>`__ for the actual authenticated
+[golang's](https://godoc.org/golang.org/x/crypto/nacl/box) [nacl
+box](http://nacl.cr.yp.to/box.html) for the actual authenticated
 encryption algorithm.
 
 Each peer generates an ED25519 key-pair to use as a persistent
@@ -19,10 +18,9 @@ their respective ephemeral public keys. This happens in the clear.
 They then each compute the shared secret. The shared secret is the
 multiplication of the peer's ephemeral private key by the other peer's
 ephemeral public key. The result is the same for both peers by the magic
-of `elliptic
-curves <https://en.wikipedia.org/wiki/Elliptic_curve_cryptography>`__.
-The shared secret is used as the symmetric key for the encryption
-algorithm.
+of [elliptic
+curves](https://en.wikipedia.org/wiki/Elliptic_curve_cryptography). The
+shared secret is used as the symmetric key for the encryption algorithm.
 
 The two ephemeral public keys are sorted to establish a canonical order.
 Then a 24-byte nonce is generated by concatenating the public keys and
@@ -52,8 +50,7 @@ time it is used. The communications maintain Perfect Forward Secrecy, as
 the persistent key pair was not used for generating secrets - only for
 authenticating.
 
-Caveat
-------
+## Caveat
 
 This system is still vulnerable to a Man-In-The-Middle attack if the
 persistent public key of the remote node is not known in advance. The
@@ -62,17 +59,15 @@ such as the Web-of-Trust or Certificate Authorities. In our case, we can
 use the blockchain itself as a certificate authority to ensure that we
 are connected to at least one validator.
 
-Config
-------
+## Config
 
 Authenticated encryption is enabled by default.
 
-Additional Reading
-------------------
+## Additional Reading
 
--  `Implementation <https://github.com/tendermint/go-p2p/blob/master/secret_connection.go#L49>`__
--  `Original STS paper by Whitfield Diffie, Paul C. van Oorschot and
-   Michael J.
-   Wiener <http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.216.6107&rep=rep1&type=pdf>`__
--  `Further work on secret
-   handshakes <https://dominictarr.github.io/secret-handshake-paper/shs.pdf>`__
+-   [Implementation](https://github.com/tendermint/tendermint/blob/64bae01d007b5bee0d0827ab53259ffd5910b4e6/p2p/conn/secret_connection.go#L47)
+-   [Original STS paper by Whitfield Diffie, Paul C. van Oorschot and
+    Michael J.
+    Wiener](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.216.6107&rep=rep1&type=pdf)
+-   [Further work on secret
+    handshakes](https://dominictarr.github.io/secret-handshake-paper/shs.pdf)

docs/tendermint-core/using-tendermint.md

@@ -31,6 +31,73 @@ For more elaborate initialization, see the tesnet command:
 tendermint testnet --help
 ```
 
+### Genesis
+
+The `genesis.json` file in `$TMHOME/config/` defines the initial
+TendermintCore state upon genesis of the blockchain ([see
+definition](https://github.com/tendermint/tendermint/blob/master/types/genesis.go)).
+
+#### Fields
+
+-   `genesis_time`: Official time of blockchain start.
+-   `chain_id`: ID of the blockchain. This must be unique for
+    every blockchain. If your testnet blockchains do not have unique
+    chain IDs, you will have a bad time.
+-   `validators`:
+-   `pub_key`: The first element specifies the `pub_key` type. 1
+    == Ed25519. The second element are the pubkey bytes.
+-   `power`: The validator's voting power.
+-   `name`: Name of the validator (optional).
+-   `app_hash`: The expected application hash (as returned by the
+    `ResponseInfo` ABCI message) upon genesis. If the app's hash does
+    not match, Tendermint will panic.
+-   `app_state`: The application state (e.g. initial distribution
+    of tokens).
+
+#### Sample genesis.json
+
+```
+{
+  "genesis_time": "2018-07-09T22:43:06.255718641Z",
+  "chain_id": "chain-IAkWsK",
+  "validators": [
+    {
+      "pub_key": {
+        "type": "tendermint/PubKeyEd25519",
+        "value": "oX8HhKsErMluxI0QWNSR8djQMSupDvHdAYrHwP7n73k="
+      },
+      "power": "1",
+      "name": "node0"
+    },
+    {
+      "pub_key": {
+        "type": "tendermint/PubKeyEd25519",
+        "value": "UZNSJA9zmeFQj36Rs296lY+WFQ4Rt6s7snPpuKypl5I="
+      },
+      "power": "1",
+      "name": "node1"
+    },
+    {
+      "pub_key": {
+        "type": "tendermint/PubKeyEd25519",
+        "value": "i9GrM6/MHB4zjCelMZBUYHNXYIzl4n0RkDCVmmLhS/o="
+      },
+      "power": "1",
+      "name": "node2"
+    },
+    {
+      "pub_key": {
+        "type": "tendermint/PubKeyEd25519",
+        "value": "0qq7954l87trEqbQV9c7d1gurnjTGMxreXc848ZZ5aw="
+      },
+      "power": "1",
+      "name": "node3"
+    }
+  ],
+  "app_hash": ""
+}
+```
+
 ## Run
 
 To run a Tendermint node, use

docs/tendermint-core/validators.md

@@ -1,5 +1,4 @@
-Validators
-==========
+# Validators
 
 Validators are responsible for committing new blocks in the blockchain.
 These validators participate in the consensus protocol by broadcasting
@@ -19,25 +18,22 @@ to post any collateral at all.
 Validators have a cryptographic key-pair and an associated amount of
 "voting power". Voting power need not be the same.
 
-Becoming a Validator
---------------------
+## Becoming a Validator
 
 There are two ways to become validator.
 
-1. They can be pre-established in the `genesis
-   state <./genesis.html>`__
-2. The ABCI app responds to the EndBlock message with changes to the
-   existing validator set.
+1.  They can be pre-established in the [genesis state](../../tendermint-core/using-tendermint.md#genesis)
+2.  The ABCI app responds to the EndBlock message with changes to the
+    existing validator set.
 
-Committing a Block
-------------------
+## Committing a Block
 
 *+2/3 is short for "more than 2/3"*
 
-A block is committed when +2/3 of the validator set sign `precommit
-votes <./block-structure.html#vote>`__ for that block at the same
-``round``. The +2/3 set of precommit votes is
-called a `*commit* <./block-structure.html#commit>`__. While any
-+2/3 set of precommits for the same block at the same height&round can
-serve as validation, the canonical commit is included in the next block
-(see `LastCommit <./block-structure.html>`__).
+A block is committed when +2/3 of the validator set sign [precommit
+votes](../spec/blockchain/blockchain.md#vote) for that block at the same `round`.
+The +2/3 set of precommit votes is called a
+[*commit*](../spec/blockchain/blockchain.md#commit). While any +2/3 set of
+precommits for the same block at the same height&round can serve as
+validation, the canonical commit is included in the next block (see
+[LastCommit](../spec/blockchain/blockchain.md#last-commit)).

evidence/wire.go

@@ -2,7 +2,7 @@ package evidence
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -10,16 +10,6 @@ var cdc = amino.NewCodec()
 
 func init() {
 	RegisterEvidenceMessages(cdc)
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 	types.RegisterEvidences(cdc)
-	RegisterMockEvidences(cdc) // For testing
-}
-
-//-------------------------------------------
-
-func RegisterMockEvidences(cdc *amino.Codec) {
-	cdc.RegisterConcrete(types.MockGoodEvidence{},
-		"tendermint/MockGoodEvidence", nil)
-	cdc.RegisterConcrete(types.MockBadEvidence{},
-		"tendermint/MockBadEvidence", nil)
 }

libs/clist/clist_test.go

@@ -2,11 +2,12 @@ package clist
 
 import (
 	"fmt"
-	"math/rand"
 	"runtime"
 	"sync/atomic"
 	"testing"
 	"time"
+
+	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
 func TestSmall(t *testing.T) {
@@ -131,7 +132,7 @@ func _TestGCRandom(t *testing.T) {
 		els = append(els, el)
 	}
 
-	for _, i := range rand.Perm(numElements) {
+	for _, i := range cmn.RandPerm(numElements) {
 		el := els[i]
 		l.Remove(el)
 		_ = el.Next()
@@ -189,7 +190,7 @@ func TestScanRightDeleteRandom(t *testing.T) {
 	// Remove an element, push back an element.
 	for i := 0; i < numTimes; i++ {
 		// Pick an element to remove
-		rmElIdx := rand.Intn(len(els))
+		rmElIdx := cmn.RandIntn(len(els))
 		rmEl := els[rmElIdx]
 
 		// Remove it
@@ -243,7 +244,7 @@ func TestWaitChan(t *testing.T) {
 		for i := 1; i < 100; i++ {
 			l.PushBack(i)
 			pushed++
-			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
+			time.Sleep(time.Duration(cmn.RandIntn(100)) * time.Millisecond)
 		}
 		close(done)
 	}()

libs/common/os_test.go

@@ -3,17 +3,14 @@ package common
 import (
 	"bytes"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"testing"
-	"time"
 )
 
 func TestWriteFileAtomic(t *testing.T) {
 	var (
-		seed             = rand.New(rand.NewSource(time.Now().UnixNano()))
-		data             = []byte(RandStr(seed.Intn(2048)))
-		old              = RandBytes(seed.Intn(2048))
+		data             = []byte(RandStr(RandIntn(2048)))
+		old              = RandBytes(RandIntn(2048))
 		perm os.FileMode = 0600
 	)
 

libs/common/random.go

@@ -11,9 +11,13 @@ const (
 	strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters
 )
 
-// pseudo random number generator.
-// seeded with OS randomness (crand)
-
+// Rand is a prng, that is seeded with OS randomness.
+// The OS randomness is obtained from crypto/rand, however none of the provided
+// methods are suitable for cryptographic usage.
+// They all utilize math/rand's prng internally.
+//
+// All of the methods here are suitable for concurrent use.
+// This is achieved by using a mutex lock on all of the provided methods.
 type Rand struct {
 	sync.Mutex
 	rand *mrand.Rand
@@ -105,18 +109,6 @@ func RandInt63n(n int64) int64 {
 	return grand.Int63n(n)
 }
 
-func RandUint16Exp() uint16 {
-	return grand.Uint16Exp()
-}
-
-func RandUint32Exp() uint32 {
-	return grand.Uint32Exp()
-}
-
-func RandUint64Exp() uint64 {
-	return grand.Uint64Exp()
-}
-
 func RandFloat32() float32 {
 	return grand.Float32()
 }
@@ -150,8 +142,7 @@ func (r *Rand) Seed(seed int64) {
 	r.Unlock()
 }
 
-// Constructs an alphanumeric string of given length.
-// It is not safe for cryptographic usage.
+// Str constructs a random alphanumeric string of given length.
 func (r *Rand) Str(length int) string {
 	chars := []byte{}
 MAIN_LOOP:
@@ -175,12 +166,10 @@ MAIN_LOOP:
 	return string(chars)
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Uint16() uint16 {
 	return uint16(r.Uint32() & (1<<16 - 1))
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Uint32() uint32 {
 	r.Lock()
 	u32 := r.rand.Uint32()
@@ -188,12 +177,10 @@ func (r *Rand) Uint32() uint32 {
 	return u32
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Uint64() uint64 {
 	return uint64(r.Uint32())<<32 + uint64(r.Uint32())
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Uint() uint {
 	r.Lock()
 	i := r.rand.Int()
@@ -201,22 +188,18 @@ func (r *Rand) Uint() uint {
 	return uint(i)
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int16() int16 {
 	return int16(r.Uint32() & (1<<16 - 1))
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int32() int32 {
 	return int32(r.Uint32())
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int64() int64 {
 	return int64(r.Uint64())
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int() int {
 	r.Lock()
 	i := r.rand.Int()
@@ -224,7 +207,6 @@ func (r *Rand) Int() int {
 	return i
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int31() int32 {
 	r.Lock()
 	i31 := r.rand.Int31()
@@ -232,7 +214,6 @@ func (r *Rand) Int31() int32 {
 	return i31
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int31n(n int32) int32 {
 	r.Lock()
 	i31n := r.rand.Int31n(n)
@@ -240,7 +221,6 @@ func (r *Rand) Int31n(n int32) int32 {
 	return i31n
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int63() int64 {
 	r.Lock()
 	i63 := r.rand.Int63()
@@ -248,7 +228,6 @@ func (r *Rand) Int63() int64 {
 	return i63
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Int63n(n int64) int64 {
 	r.Lock()
 	i63n := r.rand.Int63n(n)
@@ -256,43 +235,6 @@ func (r *Rand) Int63n(n int64) int64 {
 	return i63n
 }
 
-// Distributed pseudo-exponentially to test for various cases
-// It is not safe for cryptographic usage.
-func (r *Rand) Uint16Exp() uint16 {
-	bits := r.Uint32() % 16
-	if bits == 0 {
-		return 0
-	}
-	n := uint16(1 << (bits - 1))
-	n += uint16(r.Int31()) & ((1 << (bits - 1)) - 1)
-	return n
-}
-
-// Distributed pseudo-exponentially to test for various cases
-// It is not safe for cryptographic usage.
-func (r *Rand) Uint32Exp() uint32 {
-	bits := r.Uint32() % 32
-	if bits == 0 {
-		return 0
-	}
-	n := uint32(1 << (bits - 1))
-	n += uint32(r.Int31()) & ((1 << (bits - 1)) - 1)
-	return n
-}
-
-// Distributed pseudo-exponentially to test for various cases
-// It is not safe for cryptographic usage.
-func (r *Rand) Uint64Exp() uint64 {
-	bits := r.Uint32() % 64
-	if bits == 0 {
-		return 0
-	}
-	n := uint64(1 << (bits - 1))
-	n += uint64(r.Int63()) & ((1 << (bits - 1)) - 1)
-	return n
-}
-
-// It is not safe for cryptographic usage.
 func (r *Rand) Float32() float32 {
 	r.Lock()
 	f32 := r.rand.Float32()
@@ -300,7 +242,6 @@ func (r *Rand) Float32() float32 {
 	return f32
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Float64() float64 {
 	r.Lock()
 	f64 := r.rand.Float64()
@@ -308,13 +249,12 @@ func (r *Rand) Float64() float64 {
 	return f64
 }
 
-// It is not safe for cryptographic usage.
 func (r *Rand) Time() time.Time {
-	return time.Unix(int64(r.Uint64Exp()), 0)
+	return time.Unix(int64(r.Uint64()), 0)
 }
 
-// RandBytes returns n random bytes from the OS's source of entropy ie. via crypto/rand.
-// It is not safe for cryptographic usage.
+// Bytes returns n random bytes generated from the internal
+// prng.
 func (r *Rand) Bytes(n int) []byte {
 	// cRandBytes isn't guaranteed to be fast so instead
 	// use random bytes generated from the internal PRNG
@@ -325,9 +265,8 @@ func (r *Rand) Bytes(n int) []byte {
 	return bs
 }
 
-// RandIntn returns, as an int, a non-negative pseudo-random number in [0, n).
+// Intn returns, as an int, a uniform pseudo-random number in the range [0, n).
 // It panics if n <= 0.
-// It is not safe for cryptographic usage.
 func (r *Rand) Intn(n int) int {
 	r.Lock()
 	i := r.rand.Intn(n)
@@ -335,8 +274,7 @@ func (r *Rand) Intn(n int) int {
 	return i
 }
 
-// RandPerm returns a pseudo-random permutation of n integers in [0, n).
-// It is not safe for cryptographic usage.
+// Perm returns a pseudo-random permutation of n integers in [0, n).
 func (r *Rand) Perm(n int) []int {
 	r.Lock()
 	perm := r.rand.Perm(n)

libs/common/random_test.go

@@ -73,9 +73,6 @@ func testThemAll() string {
 	fmt.Fprintf(out, "randInt64: %d\n", RandInt64())
 	fmt.Fprintf(out, "randUint32: %d\n", RandUint32())
 	fmt.Fprintf(out, "randUint64: %d\n", RandUint64())
-	fmt.Fprintf(out, "randUint16Exp: %d\n", RandUint16Exp())
-	fmt.Fprintf(out, "randUint32Exp: %d\n", RandUint32Exp())
-	fmt.Fprintf(out, "randUint64Exp: %d\n", RandUint64Exp())
 	return out.String()
 }
 

libs/common/repeat_timer_test.go

@@ -1,7 +1,6 @@
 package common
 
 import (
-	"math/rand"
 	"sync"
 	"testing"
 	"time"
@@ -131,7 +130,7 @@ func TestRepeatTimerReset(t *testing.T) {
 
 	// just random calls
 	for i := 0; i < 100; i++ {
-		time.Sleep(time.Duration(rand.Intn(40)) * time.Millisecond)
+		time.Sleep(time.Duration(RandIntn(40)) * time.Millisecond)
 		timer.Reset()
 	}
 }

libs/common/types.pb.go

@@ -1,25 +1,32 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: common/types.proto
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: libs/common/types.proto
 
 /*
-Package common is a generated protocol buffer package.
+	Package common is a generated protocol buffer package.
 
-It is generated from these files:
-	common/types.proto
+	It is generated from these files:
+		libs/common/types.proto
 
-It has these top-level messages:
-	KVPair
-	KI64Pair
+	It has these top-level messages:
+		KVPair
+		KI64Pair
 */
-//nolint: gas
+//nolint
 package common
 
-import proto "github.com/golang/protobuf/proto"
+import proto "github.com/gogo/protobuf/proto"
+import golang_proto "github.com/golang/protobuf/proto"
 import fmt "fmt"
 import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import bytes "bytes"
+
+import io "io"
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
+var _ = golang_proto.Marshal
 var _ = fmt.Errorf
 var _ = math.Inf
 
@@ -27,7 +34,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
 
 // Define these here for compatibility but use tmlibs/common.KVPair.
 type KVPair struct {
@@ -38,7 +45,7 @@ type KVPair struct {
 func (m *KVPair) Reset()                    { *m = KVPair{} }
 func (m *KVPair) String() string            { return proto.CompactTextString(m) }
 func (*KVPair) ProtoMessage()               {}
-func (*KVPair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+func (*KVPair) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{0} }
 
 func (m *KVPair) GetKey() []byte {
 	if m != nil {
@@ -57,13 +64,13 @@ func (m *KVPair) GetValue() []byte {
 // Define these here for compatibility but use tmlibs/common.KI64Pair.
 type KI64Pair struct {
 	Key   []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Value int64  `protobuf:"varint,2,opt,name=value" json:"value,omitempty"`
+	Value int64  `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
 func (m *KI64Pair) Reset()                    { *m = KI64Pair{} }
 func (m *KI64Pair) String() string            { return proto.CompactTextString(m) }
 func (*KI64Pair) ProtoMessage()               {}
-func (*KI64Pair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+func (*KI64Pair) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{1} }
 
 func (m *KI64Pair) GetKey() []byte {
 	if m != nil {
@@ -81,18 +88,608 @@ func (m *KI64Pair) GetValue() int64 {
 
 func init() {
 	proto.RegisterType((*KVPair)(nil), "common.KVPair")
+	golang_proto.RegisterType((*KVPair)(nil), "common.KVPair")
 	proto.RegisterType((*KI64Pair)(nil), "common.KI64Pair")
+	golang_proto.RegisterType((*KI64Pair)(nil), "common.KI64Pair")
+}
+func (this *KVPair) Equal(that interface{}) bool {
+	if that == nil {
+		return this == nil
+	}
+
+	that1, ok := that.(*KVPair)
+	if !ok {
+		that2, ok := that.(KVPair)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return this == nil
+	} else if this == nil {
+		return false
+	}
+	if !bytes.Equal(this.Key, that1.Key) {
+		return false
+	}
+	if !bytes.Equal(this.Value, that1.Value) {
+		return false
+	}
+	return true
+}
+func (this *KI64Pair) Equal(that interface{}) bool {
+	if that == nil {
+		return this == nil
+	}
+
+	that1, ok := that.(*KI64Pair)
+	if !ok {
+		that2, ok := that.(KI64Pair)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return this == nil
+	} else if this == nil {
+		return false
+	}
+	if !bytes.Equal(this.Key, that1.Key) {
+		return false
+	}
+	if this.Value != that1.Value {
+		return false
+	}
+	return true
+}
+func (m *KVPair) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
 }
 
-func init() { proto.RegisterFile("common/types.proto", fileDescriptor0) }
-
-var fileDescriptor0 = []byte{
-	// 107 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0xce, 0xcf, 0xcd,
-	0xcd, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62,
-	0x83, 0x88, 0x29, 0x19, 0x70, 0xb1, 0x79, 0x87, 0x05, 0x24, 0x66, 0x16, 0x09, 0x09, 0x70, 0x31,
-	0x67, 0xa7, 0x56, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x04, 0x81, 0x98, 0x42, 0x22, 0x5c, 0xac,
-	0x65, 0x89, 0x39, 0xa5, 0xa9, 0x12, 0x4c, 0x60, 0x31, 0x08, 0x47, 0xc9, 0x88, 0x8b, 0xc3, 0xdb,
-	0xd3, 0xcc, 0x84, 0x18, 0x3d, 0xcc, 0x50, 0x3d, 0x49, 0x6c, 0x60, 0x4b, 0x8d, 0x01, 0x01, 0x00,
-	0x00, 0xff, 0xff, 0xd8, 0xf1, 0xc3, 0x8c, 0x8a, 0x00, 0x00, 0x00,
+func (m *KVPair) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Key) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Key)))
+		i += copy(dAtA[i:], m.Key)
+	}
+	if len(m.Value) > 0 {
+		dAtA[i] = 0x12
+		i++
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Value)))
+		i += copy(dAtA[i:], m.Value)
+	}
+	return i, nil
+}
+
+func (m *KI64Pair) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *KI64Pair) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Key) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Key)))
+		i += copy(dAtA[i:], m.Key)
+	}
+	if m.Value != 0 {
+		dAtA[i] = 0x10
+		i++
+		i = encodeVarintTypes(dAtA, i, uint64(m.Value))
+	}
+	return i, nil
+}
+
+func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return offset + 1
+}
+func NewPopulatedKVPair(r randyTypes, easy bool) *KVPair {
+	this := &KVPair{}
+	v1 := r.Intn(100)
+	this.Key = make([]byte, v1)
+	for i := 0; i < v1; i++ {
+		this.Key[i] = byte(r.Intn(256))
+	}
+	v2 := r.Intn(100)
+	this.Value = make([]byte, v2)
+	for i := 0; i < v2; i++ {
+		this.Value[i] = byte(r.Intn(256))
+	}
+	if !easy && r.Intn(10) != 0 {
+	}
+	return this
+}
+
+func NewPopulatedKI64Pair(r randyTypes, easy bool) *KI64Pair {
+	this := &KI64Pair{}
+	v3 := r.Intn(100)
+	this.Key = make([]byte, v3)
+	for i := 0; i < v3; i++ {
+		this.Key[i] = byte(r.Intn(256))
+	}
+	this.Value = int64(r.Int63())
+	if r.Intn(2) == 0 {
+		this.Value *= -1
+	}
+	if !easy && r.Intn(10) != 0 {
+	}
+	return this
+}
+
+type randyTypes interface {
+	Float32() float32
+	Float64() float64
+	Int63() int64
+	Int31() int32
+	Uint32() uint32
+	Intn(n int) int
+}
+
+func randUTF8RuneTypes(r randyTypes) rune {
+	ru := r.Intn(62)
+	if ru < 10 {
+		return rune(ru + 48)
+	} else if ru < 36 {
+		return rune(ru + 55)
+	}
+	return rune(ru + 61)
+}
+func randStringTypes(r randyTypes) string {
+	v4 := r.Intn(100)
+	tmps := make([]rune, v4)
+	for i := 0; i < v4; i++ {
+		tmps[i] = randUTF8RuneTypes(r)
+	}
+	return string(tmps)
+}
+func randUnrecognizedTypes(r randyTypes, maxFieldNumber int) (dAtA []byte) {
+	l := r.Intn(5)
+	for i := 0; i < l; i++ {
+		wire := r.Intn(4)
+		if wire == 3 {
+			wire = 5
+		}
+		fieldNumber := maxFieldNumber + r.Intn(100)
+		dAtA = randFieldTypes(dAtA, r, fieldNumber, wire)
+	}
+	return dAtA
+}
+func randFieldTypes(dAtA []byte, r randyTypes, fieldNumber int, wire int) []byte {
+	key := uint32(fieldNumber)<<3 | uint32(wire)
+	switch wire {
+	case 0:
+		dAtA = encodeVarintPopulateTypes(dAtA, uint64(key))
+		v5 := r.Int63()
+		if r.Intn(2) == 0 {
+			v5 *= -1
+		}
+		dAtA = encodeVarintPopulateTypes(dAtA, uint64(v5))
+	case 1:
+		dAtA = encodeVarintPopulateTypes(dAtA, uint64(key))
+		dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)))
+	case 2:
+		dAtA = encodeVarintPopulateTypes(dAtA, uint64(key))
+		ll := r.Intn(100)
+		dAtA = encodeVarintPopulateTypes(dAtA, uint64(ll))
+		for j := 0; j < ll; j++ {
+			dAtA = append(dAtA, byte(r.Intn(256)))
+		}
+	default:
+		dAtA = encodeVarintPopulateTypes(dAtA, uint64(key))
+		dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)))
+	}
+	return dAtA
+}
+func encodeVarintPopulateTypes(dAtA []byte, v uint64) []byte {
+	for v >= 1<<7 {
+		dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80))
+		v >>= 7
+	}
+	dAtA = append(dAtA, uint8(v))
+	return dAtA
+}
+func (m *KVPair) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	l = len(m.Value)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
+func (m *KI64Pair) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	if m.Value != 0 {
+		n += 1 + sovTypes(uint64(m.Value))
+	}
+	return n
+}
+
+func sovTypes(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozTypes(x uint64) (n int) {
+	return sovTypes(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *KVPair) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: KVPair: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: KVPair: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...)
+			if m.Key == nil {
+				m.Key = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...)
+			if m.Value == nil {
+				m.Value = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *KI64Pair) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: KI64Pair: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: KI64Pair: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...)
+			if m.Key == nil {
+				m.Key = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType)
+			}
+			m.Value = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Value |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipTypes(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthTypes
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowTypes
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipTypes(dAtA[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthTypes = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowTypes   = fmt.Errorf("proto: integer overflow")
+)
+
+func init() { proto.RegisterFile("libs/common/types.proto", fileDescriptorTypes) }
+func init() { golang_proto.RegisterFile("libs/common/types.proto", fileDescriptorTypes) }
+
+var fileDescriptorTypes = []byte{
+	// 174 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xcf, 0xc9, 0x4c, 0x2a,
+	0xd6, 0x4f, 0xce, 0xcf, 0xcd, 0xcd, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28,
+	0xca, 0x2f, 0xc9, 0x17, 0x62, 0x83, 0x88, 0x49, 0xe9, 0xa6, 0x67, 0x96, 0x64, 0x94, 0x26, 0xe9,
+	0x25, 0xe7, 0xe7, 0xea, 0xa7, 0xe7, 0xa7, 0xe7, 0xeb, 0x83, 0xa5, 0x93, 0x4a, 0xd3, 0xc0, 0x3c,
+	0x30, 0x07, 0xcc, 0x82, 0x68, 0x53, 0x32, 0xe0, 0x62, 0xf3, 0x0e, 0x0b, 0x48, 0xcc, 0x2c, 0x12,
+	0x12, 0xe0, 0x62, 0xce, 0x4e, 0xad, 0x94, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0x31, 0x85,
+	0x44, 0xb8, 0x58, 0xcb, 0x12, 0x73, 0x4a, 0x53, 0x25, 0x98, 0xc0, 0x62, 0x10, 0x8e, 0x92, 0x11,
+	0x17, 0x87, 0xb7, 0xa7, 0x99, 0x09, 0x31, 0x7a, 0x98, 0xa1, 0x7a, 0x9c, 0x64, 0x7e, 0x3c, 0x94,
+	0x63, 0x5c, 0xf1, 0x48, 0x8e, 0x71, 0xc7, 0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c,
+	0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0xf1, 0xc0, 0x63, 0x39, 0xc6, 0x24, 0x36, 0xb0, 0x53, 0x8c,
+	0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x39, 0xe1, 0xef, 0xdc, 0x00, 0x00, 0x00,
 }

libs/common/types.proto

@@ -1,6 +1,17 @@
 syntax = "proto3";
 package common;
 
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.goproto_registration) = true;
+// Generate tests
+option (gogoproto.populate_all) = true;
+option (gogoproto.equal_all) = true;
+option (gogoproto.testgen_all) = true;
+
 //----------------------------------------
 // Abstract types
 

libs/common/typespb_test.go

@@ -0,0 +1,280 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: libs/common/types.proto
+
+/*
+Package common is a generated protocol buffer package.
+
+It is generated from these files:
+	libs/common/types.proto
+
+It has these top-level messages:
+	KVPair
+	KI64Pair
+*/
+package common
+
+import testing "testing"
+import rand "math/rand"
+import time "time"
+import proto "github.com/gogo/protobuf/proto"
+import jsonpb "github.com/gogo/protobuf/jsonpb"
+import golang_proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = golang_proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+func TestKVPairProto(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKVPair(popr, false)
+	dAtA, err := proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &KVPair{}
+	if err := proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	littlefuzz := make([]byte, len(dAtA))
+	copy(littlefuzz, dAtA)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+	if len(littlefuzz) > 0 {
+		fuzzamount := 100
+		for i := 0; i < fuzzamount; i++ {
+			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+		}
+		// shouldn't panic
+		_ = proto.Unmarshal(littlefuzz, msg)
+	}
+}
+
+func TestKVPairMarshalTo(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKVPair(popr, false)
+	size := p.Size()
+	dAtA := make([]byte, size)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	_, err := p.MarshalTo(dAtA)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &KVPair{}
+	if err := proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestKI64PairProto(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKI64Pair(popr, false)
+	dAtA, err := proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &KI64Pair{}
+	if err := proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	littlefuzz := make([]byte, len(dAtA))
+	copy(littlefuzz, dAtA)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+	if len(littlefuzz) > 0 {
+		fuzzamount := 100
+		for i := 0; i < fuzzamount; i++ {
+			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+		}
+		// shouldn't panic
+		_ = proto.Unmarshal(littlefuzz, msg)
+	}
+}
+
+func TestKI64PairMarshalTo(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKI64Pair(popr, false)
+	size := p.Size()
+	dAtA := make([]byte, size)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	_, err := p.MarshalTo(dAtA)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &KI64Pair{}
+	if err := proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestKVPairJSON(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKVPair(popr, true)
+	marshaler := jsonpb.Marshaler{}
+	jsondata, err := marshaler.MarshalToString(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &KVPair{}
+	err = jsonpb.UnmarshalString(jsondata, msg)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+	}
+}
+func TestKI64PairJSON(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKI64Pair(popr, true)
+	marshaler := jsonpb.Marshaler{}
+	jsondata, err := marshaler.MarshalToString(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &KI64Pair{}
+	err = jsonpb.UnmarshalString(jsondata, msg)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+	}
+}
+func TestKVPairProtoText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKVPair(popr, true)
+	dAtA := proto.MarshalTextString(p)
+	msg := &KVPair{}
+	if err := proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestKVPairProtoCompactText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKVPair(popr, true)
+	dAtA := proto.CompactTextString(p)
+	msg := &KVPair{}
+	if err := proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestKI64PairProtoText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKI64Pair(popr, true)
+	dAtA := proto.MarshalTextString(p)
+	msg := &KI64Pair{}
+	if err := proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestKI64PairProtoCompactText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKI64Pair(popr, true)
+	dAtA := proto.CompactTextString(p)
+	msg := &KI64Pair{}
+	if err := proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestKVPairSize(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKVPair(popr, true)
+	size2 := proto.Size(p)
+	dAtA, err := proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	size := p.Size()
+	if len(dAtA) != size {
+		t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+	}
+	if size2 != size {
+		t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+	}
+	size3 := proto.Size(p)
+	if size3 != size {
+		t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+	}
+}
+
+func TestKI64PairSize(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := rand.New(rand.NewSource(seed))
+	p := NewPopulatedKI64Pair(popr, true)
+	size2 := proto.Size(p)
+	dAtA, err := proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	size := p.Size()
+	if len(dAtA) != size {
+		t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+	}
+	if size2 != size {
+		t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+	}
+	size3 := proto.Size(p)
+	if size3 != size {
+		t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+	}
+}
+
+//These tests are generated by github.com/gogo/protobuf/plugin/testgen

libs/events/events_test.go

@@ -2,11 +2,11 @@ package events
 
 import (
 	"fmt"
-	"math/rand"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
 // TestAddListenerForEventFireOnce sets up an EventSwitch, subscribes a single
@@ -306,8 +306,8 @@ func TestRemoveListenersAsync(t *testing.T) {
 	// collect received events for event2
 	go sumReceivedNumbers(numbers2, doneSum2)
 	addListenersStress := func() {
-		s1 := rand.NewSource(time.Now().UnixNano())
-		r1 := rand.New(s1)
+		r1 := cmn.NewRand()
+		r1.Seed(time.Now().UnixNano())
 		for k := uint16(0); k < 400; k++ {
 			listenerNumber := r1.Intn(100) + 3
 			eventNumber := r1.Intn(3) + 1
@@ -317,8 +317,8 @@ func TestRemoveListenersAsync(t *testing.T) {
 		}
 	}
 	removeListenersStress := func() {
-		s2 := rand.NewSource(time.Now().UnixNano())
-		r2 := rand.New(s2)
+		r2 := cmn.NewRand()
+		r2.Seed(time.Now().UnixNano())
 		for k := uint16(0); k < 80; k++ {
 			listenerNumber := r2.Intn(100) + 3
 			go evsw.RemoveListener(fmt.Sprintf("listener%v", listenerNumber))

libs/pubsub/pubsub.go

@@ -163,6 +163,8 @@ func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, ou
 		return nil
 	case <-ctx.Done():
 		return ctx.Err()
+	case <-s.Quit():
+		return nil
 	}
 }
 
@@ -190,6 +192,8 @@ func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query)
 		return nil
 	case <-ctx.Done():
 		return ctx.Err()
+	case <-s.Quit():
+		return nil
 	}
 }
 
@@ -211,6 +215,8 @@ func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error {
 		return nil
 	case <-ctx.Done():
 		return ctx.Err()
+	case <-s.Quit():
+		return nil
 	}
 }
 
@@ -229,6 +235,8 @@ func (s *Server) PublishWithTags(ctx context.Context, msg interface{}, tags TagM
 		return nil
 	case <-ctx.Done():
 		return ctx.Err()
+	case <-s.Quit():
+		return nil
 	}
 }
 

libs/pubsub/query/Makefile

@@ -1,10 +1,10 @@
 gen_query_parser:
-	@go get github.com/pointlander/peg
+	go get -u -v github.com/pointlander/peg
 	peg -inline -switch query.peg
 
 fuzzy_test:
-	@go get github.com/dvyukov/go-fuzz/go-fuzz
-	@go get github.com/dvyukov/go-fuzz/go-fuzz-build
+	go get -u -v github.com/dvyukov/go-fuzz/go-fuzz
+	go get -u -v github.com/dvyukov/go-fuzz/go-fuzz-build
 	go-fuzz-build github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test
 	go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output
 

libs/pubsub/query/query.peg.go

@@ -1,6 +1,8 @@
 // nolint
 package query
 
+//go:generate peg -inline -switch query.peg
+
 import (
 	"fmt"
 	"math"

lite/files/wire.go

@@ -2,11 +2,11 @@ package files
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 }

lite/helpers.go

@@ -4,6 +4,8 @@ import (
 	"time"
 
 	crypto "github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
 
 	"github.com/tendermint/tendermint/types"
 )
@@ -23,7 +25,7 @@ type ValKeys []crypto.PrivKey
 func GenValKeys(n int) ValKeys {
 	res := make(ValKeys, n)
 	for i := range res {
-		res[i] = crypto.GenPrivKeyEd25519()
+		res[i] = ed25519.GenPrivKey()
 	}
 	return res
 }
@@ -32,7 +34,7 @@ func GenValKeys(n int) ValKeys {
 func (v ValKeys) Change(i int) ValKeys {
 	res := make(ValKeys, len(v))
 	copy(res, v)
-	res[i] = crypto.GenPrivKeyEd25519()
+	res[i] = ed25519.GenPrivKey()
 	return res
 }
 
@@ -46,7 +48,7 @@ func (v ValKeys) Extend(n int) ValKeys {
 func GenSecpValKeys(n int) ValKeys {
 	res := make(ValKeys, n)
 	for i := range res {
-		res[i] = crypto.GenPrivKeySecp256k1()
+		res[i] = secp256k1.GenPrivKey()
 	}
 	return res
 }

lite/performance_test.go

@@ -2,13 +2,12 @@ package lite
 
 import (
 	"fmt"
-	"math/rand"
 	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
+	cmn "github.com/tendermint/tendermint/libs/common"
 	liteErr "github.com/tendermint/tendermint/lite/errors"
 )
 
@@ -280,7 +279,11 @@ func BenchmarkMemStoreProviderGetByHeightBinarySearch1000(b *testing.B) {
 	benchmarkMemStoreProvidergetByHeight(b, fcs1000, h1000, binarySearch)
 }
 
-var rng = rand.New(rand.NewSource(10))
+var rng = cmn.NewRand()
+
+func init() {
+	rng.Seed(10)
+}
 
 func benchmarkMemStoreProvidergetByHeight(b *testing.B, fcs []FullCommit, fHeights []int64, algo algo) {
 	lazyGenerateFullCommits(b)

lite/proxy/block.go

@@ -15,14 +15,14 @@ func ValidateBlockMeta(meta *types.BlockMeta, check lite.Commit) error {
 		return errors.New("expecting a non-nil BlockMeta")
 	}
 	// TODO: check the BlockID??
-	return ValidateHeader(meta.Header, check)
+	return ValidateHeader(&meta.Header, check)
 }
 
 func ValidateBlock(meta *types.Block, check lite.Commit) error {
 	if meta == nil {
 		return errors.New("expecting a non-nil Block")
 	}
-	err := ValidateHeader(meta.Header, check)
+	err := ValidateHeader(&meta.Header, check)
 	if err != nil {
 		return err
 	}

lite/proxy/validate_test.go

@@ -18,7 +18,7 @@ var (
 	testTime2    = time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC)
 )
 
-var hdrHeight11 = &types.Header{
+var hdrHeight11 = types.Header{
 	Height:         11,
 	Time:           testTime1,
 	ValidatorsHash: []byte("Tendermint"),
@@ -34,21 +34,18 @@ func TestValidateBlock(t *testing.T) {
 			block: nil, wantErr: "non-nil Block",
 		},
 		{
-			block: &types.Block{}, wantErr: "nil Header",
-		},
-		{
-			block: &types.Block{Header: new(types.Header)},
+			block: &types.Block{},
 		},
 
 		// Start Header.Height mismatch test
 		{
-			block:   &types.Block{Header: &types.Header{Height: 10}},
+			block:   &types.Block{Header: types.Header{Height: 10}},
 			commit:  lite.Commit{Header: &types.Header{Height: 11}},
 			wantErr: "don't match - 10 vs 11",
 		},
 
 		{
-			block:  &types.Block{Header: &types.Header{Height: 11}},
+			block:  &types.Block{Header: types.Header{Height: 11}},
 			commit: lite.Commit{Header: &types.Header{Height: 11}},
 		},
 		// End Header.Height mismatch test
@@ -62,15 +59,15 @@ func TestValidateBlock(t *testing.T) {
 
 		{
 			block:  &types.Block{Header: hdrHeight11},
-			commit: lite.Commit{Header: hdrHeight11},
+			commit: lite.Commit{Header: &hdrHeight11},
 		},
 		// End Header.Hash mismatch test
 
 		// Start Header.Data hash mismatch test
 		{
 			block: &types.Block{
-				Header: &types.Header{Height: 11},
-				Data:   &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}},
+				Header: types.Header{Height: 11},
+				Data:   types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}},
 			},
 			commit: lite.Commit{
 				Header: &types.Header{Height: 11},
@@ -80,8 +77,8 @@ func TestValidateBlock(t *testing.T) {
 		},
 		{
 			block: &types.Block{
-				Header: &types.Header{Height: 11, DataHash: deadBeefHash},
-				Data:   &types.Data{Txs: deadBeefTxs},
+				Header: types.Header{Height: 11, DataHash: deadBeefHash},
+				Data:   types.Data{Txs: deadBeefTxs},
 			},
 			commit: lite.Commit{
 				Header: &types.Header{Height: 11},
@@ -116,21 +113,18 @@ func TestValidateBlockMeta(t *testing.T) {
 			meta: nil, wantErr: "non-nil BlockMeta",
 		},
 		{
-			meta: &types.BlockMeta{}, wantErr: "non-nil Header",
-		},
-		{
-			meta: &types.BlockMeta{Header: new(types.Header)},
+			meta: &types.BlockMeta{},
 		},
 
 		// Start Header.Height mismatch test
 		{
-			meta:    &types.BlockMeta{Header: &types.Header{Height: 10}},
+			meta:    &types.BlockMeta{Header: types.Header{Height: 10}},
 			commit:  lite.Commit{Header: &types.Header{Height: 11}},
 			wantErr: "don't match - 10 vs 11",
 		},
 
 		{
-			meta:   &types.BlockMeta{Header: &types.Header{Height: 11}},
+			meta:   &types.BlockMeta{Header: types.Header{Height: 11}},
 			commit: lite.Commit{Header: &types.Header{Height: 11}},
 		},
 		// End Header.Height mismatch test
@@ -144,12 +138,12 @@ func TestValidateBlockMeta(t *testing.T) {
 
 		{
 			meta:   &types.BlockMeta{Header: hdrHeight11},
-			commit: lite.Commit{Header: hdrHeight11},
+			commit: lite.Commit{Header: &hdrHeight11},
 		},
 
 		{
 			meta: &types.BlockMeta{
-				Header: &types.Header{
+				Header: types.Header{
 					Height:         11,
 					ValidatorsHash: []byte("lite-test"),
 					// TODO: should be able to use empty time after Amino upgrade
@@ -164,7 +158,7 @@ func TestValidateBlockMeta(t *testing.T) {
 
 		{
 			meta: &types.BlockMeta{
-				Header: &types.Header{
+				Header: types.Header{
 					Height: 11, DataHash: deadBeefHash,
 					ValidatorsHash: []byte("Tendermint"),
 					Time:           testTime1,
@@ -183,7 +177,7 @@ func TestValidateBlockMeta(t *testing.T) {
 
 		{
 			meta: &types.BlockMeta{
-				Header: &types.Header{
+				Header: types.Header{
 					Height: 11, DataHash: deadBeefHash,
 					ValidatorsHash: []byte("Tendermint"),
 					Time:           testTime2,

mempool/mempool.go

@@ -78,7 +78,7 @@ type Mempool struct {
 	recheckCursor        *clist.CElement // next expected response
 	recheckEnd           *clist.CElement // re-checking stops here
 	notifiedTxsAvailable bool
-	txsAvailable         chan int64 // fires the next height once for each height, when the mempool is not empty
+	txsAvailable         chan struct{} // fires once for each height, when the mempool is not empty
 
 	// Keep a cache of already-seen txs.
 	// This reduces the pressure on the proxyApp.
@@ -130,7 +130,7 @@ func NewMempool(
 // ensuring it will trigger once every height when transactions are available.
 // NOTE: not thread safe - should only be called once, on startup
 func (mem *Mempool) EnableTxsAvailable() {
-	mem.txsAvailable = make(chan int64, 1)
+	mem.txsAvailable = make(chan struct{}, 1)
 }
 
 // SetLogger sets the Logger.
@@ -348,7 +348,7 @@ func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
 // TxsAvailable returns a channel which fires once for every height,
 // and only when transactions are available in the mempool.
 // NOTE: the returned channel may be nil if EnableTxsAvailable was not called.
-func (mem *Mempool) TxsAvailable() <-chan int64 {
+func (mem *Mempool) TxsAvailable() <-chan struct{} {
 	return mem.txsAvailable
 }
 
@@ -358,11 +358,11 @@ func (mem *Mempool) notifyTxsAvailable() {
 	}
 	if mem.txsAvailable != nil && !mem.notifiedTxsAvailable {
 		// channel cap is 1, so this will send once
+		mem.notifiedTxsAvailable = true
 		select {
-		case mem.txsAvailable <- mem.height + 1:
+		case mem.txsAvailable <- struct{}{}:
 		default:
 		}
-		mem.notifiedTxsAvailable = true
 	}
 }
 

mempool/mempool_test.go

@@ -38,7 +38,7 @@ func newMempoolWithApp(cc proxy.ClientCreator) *Mempool {
 	return mempool
 }
 
-func ensureNoFire(t *testing.T, ch <-chan int64, timeoutMS int) {
+func ensureNoFire(t *testing.T, ch <-chan struct{}, timeoutMS int) {
 	timer := time.NewTimer(time.Duration(timeoutMS) * time.Millisecond)
 	select {
 	case <-ch:
@@ -47,7 +47,7 @@ func ensureNoFire(t *testing.T, ch <-chan int64, timeoutMS int) {
 	}
 }
 
-func ensureFire(t *testing.T, ch <-chan int64, timeoutMS int) {
+func ensureFire(t *testing.T, ch <-chan struct{}, timeoutMS int) {
 	timer := time.NewTimer(time.Duration(timeoutMS) * time.Millisecond)
 	select {
 	case <-ch:

node/node.go

@@ -13,6 +13,7 @@ import (
 
 	amino "github.com/tendermint/go-amino"
 	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
@@ -20,7 +21,6 @@ import (
 	bc "github.com/tendermint/tendermint/blockchain"
 	cfg "github.com/tendermint/tendermint/config"
 	cs "github.com/tendermint/tendermint/consensus"
-	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/evidence"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
@@ -197,7 +197,7 @@ func NewNode(config *cfg.Config,
 		var (
 			// TODO: persist this key so external signer
 			// can actually authenticate us
-			privKey = crypto.GenPrivKeyEd25519()
+			privKey = ed25519.GenPrivKey()
 			pvsc    = privval.NewSocketPV(
 				logger.With("module", "privval"),
 				config.PrivValidatorListenAddr,
@@ -322,9 +322,9 @@ func NewNode(config *cfg.Config,
 		// TODO persistent peers ? so we can have their DNS addrs saved
 		pexReactor := pex.NewPEXReactor(addrBook,
 			&pex.PEXReactorConfig{
-				Seeds:          cmn.SplitAndTrim(config.P2P.Seeds, ",", " "),
-				SeedMode:       config.P2P.SeedMode,
-				PrivatePeerIDs: cmn.SplitAndTrim(config.P2P.PrivatePeerIDs, ",", " ")})
+				Seeds:    cmn.SplitAndTrim(config.P2P.Seeds, ",", " "),
+				SeedMode: config.P2P.SeedMode,
+			})
 		pexReactor.SetLogger(p2pLogger)
 		sw.AddReactor("PEX", pexReactor)
 	}
@@ -449,6 +449,9 @@ func (n *Node) OnStart() error {
 	// Add ourselves to addrbook to prevent dialing ourselves
 	n.addrBook.AddOurAddress(nodeInfo.NetAddress())
 
+	// Add private IDs to addrbook to block those peers being added
+	n.addrBook.AddPrivateIDs(cmn.SplitAndTrim(n.config.P2P.PrivatePeerIDs, ",", " "))
+
 	// Start the RPC server before the P2P server
 	// so we can eg. receive txs for the first block
 	if n.config.RPC.ListenAddress != "" {
@@ -486,9 +489,16 @@ func (n *Node) OnStop() {
 	n.BaseService.OnStop()
 
 	n.Logger.Info("Stopping Node")
+
+	// first stop the non-reactor services
+	n.eventBus.Stop()
+	n.indexerService.Stop()
+
+	// now stop the reactors
 	// TODO: gracefully disconnect from peers.
 	n.sw.Stop()
 
+	// finally stop the listeners / external services
 	for _, l := range n.rpcListeners {
 		n.Logger.Info("Closing rpc listener", "listener", l)
 		if err := l.Close(); err != nil {
@@ -496,9 +506,6 @@ func (n *Node) OnStop() {
 		}
 	}
 
-	n.eventBus.Stop()
-	n.indexerService.Stop()
-
 	if pvsc, ok := n.privValidator.(*privval.SocketPV); ok {
 		if err := pvsc.Stop(); err != nil {
 			n.Logger.Error("Error stopping priv validator socket client", "err", err)

node/node_test.go

@@ -2,6 +2,9 @@ package node
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"syscall"
 	"testing"
 	"time"
 
@@ -43,6 +46,13 @@ func TestNodeStartStop(t *testing.T) {
 	select {
 	case <-n.Quit():
 	case <-time.After(5 * time.Second):
+		pid := os.Getpid()
+		p, err := os.FindProcess(pid)
+		if err != nil {
+			panic(err)
+		}
+		err = p.Signal(syscall.SIGABRT)
+		fmt.Println(err)
 		t.Fatal("timed out waiting for shutdown")
 	}
 }

node/wire.go

@@ -2,11 +2,11 @@ package node
 
 import (
 	amino "github.com/tendermint/go-amino"
-	crypto "github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 var cdc = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 }

p2p/conn/secret_connection_test.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	crypto "github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
@@ -35,9 +35,9 @@ func makeKVStoreConnPair() (fooConn, barConn kvstoreConn) {
 func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection) {
 
 	var fooConn, barConn = makeKVStoreConnPair()
-	var fooPrvKey = crypto.GenPrivKeyEd25519()
+	var fooPrvKey = ed25519.GenPrivKey()
 	var fooPubKey = fooPrvKey.PubKey()
-	var barPrvKey = crypto.GenPrivKeyEd25519()
+	var barPrvKey = ed25519.GenPrivKey()
 	var barPubKey = barPrvKey.PubKey()
 
 	// Make connections from both sides in parallel.
@@ -105,7 +105,7 @@ func TestSecretConnectionReadWrite(t *testing.T) {
 	genNodeRunner := func(id string, nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) cmn.Task {
 		return func(_ int) (interface{}, error, bool) {
 			// Initiate cryptographic private key and secret connection trhough nodeConn.
-			nodePrvKey := crypto.GenPrivKeyEd25519()
+			nodePrvKey := ed25519.GenPrivKey()
 			nodeSecretConn, err := MakeSecretConnection(nodeConn, nodePrvKey)
 			if err != nil {
 				t.Errorf("Failed to establish SecretConnection for node: %v", err)

p2p/conn/wire.go

@@ -2,12 +2,12 @@ package conn
 
 import (
 	"github.com/tendermint/go-amino"
-	"github.com/tendermint/tendermint/crypto"
+	cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino"
 )
 
 var cdc *amino.Codec = amino.NewCodec()
 
 func init() {
-	crypto.RegisterAmino(cdc)
+	cryptoAmino.RegisterAmino(cdc)
 	RegisterPacket(cdc)
 }

p2p/dummy/peer.go

@@ -3,9 +3,9 @@ package dummy
 import (
 	"net"
 
+	cmn "github.com/tendermint/tendermint/libs/common"
 	p2p "github.com/tendermint/tendermint/p2p"
 	tmconn "github.com/tendermint/tendermint/p2p/conn"
-	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
 type peer struct {
@@ -78,3 +78,8 @@ func (p *peer) Get(key string) interface{} {
 	}
 	return nil
 }
+
+// OriginalAddr always returns nil.
+func (p *peer) OriginalAddr() *p2p.NetAddress {
+	return nil
+}

p2p/key.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 
 	crypto "github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/ed25519"
 	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
@@ -70,7 +71,7 @@ func LoadNodeKey(filePath string) (*NodeKey, error) {
 }
 
 func genNodeKey(filePath string) (*NodeKey, error) {
-	privKey := crypto.GenPrivKeyEd25519()
+	privKey := ed25519.GenPrivKey()
 	nodeKey := &NodeKey{
 		PrivKey: privKey,
 	}

p2p/listener.go

@@ -151,7 +151,7 @@ func (l *DefaultListener) OnStop() {
 	l.listener.Close() // nolint: errcheck
 }
 
-// Accept connections and pass on the channel
+// Accept connections and pass on the channel.
 func (l *DefaultListener) listenRoutine() {
 	for {
 		conn, err := l.listener.Accept()
@@ -178,6 +178,8 @@ func (l *DefaultListener) listenRoutine() {
 
 // Connections returns a channel of inbound connections.
 // It gets closed when the listener closes.
+// It is the callers responsibility to close any connections received
+// over this channel.
 func (l *DefaultListener) Connections() <-chan net.Conn {
 	return l.connections
 }

p2p/node_info.go

@@ -2,8 +2,9 @@ package p2p
 
 import (
 	"fmt"
-	cmn "github.com/tendermint/tendermint/libs/common"
 	"strings"
+
+	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
 const (
@@ -81,8 +82,8 @@ func (info NodeInfo) Validate() error {
 // CONTRACT: two nodes are compatible if the major version matches and network match
 // and they have at least one channel in common.
 func (info NodeInfo) CompatibleWith(other NodeInfo) error {
-	iMajor, iMinor, _, iErr := splitVersion(info.Version)
-	oMajor, oMinor, _, oErr := splitVersion(other.Version)
+	iMajor, _, _, iErr := splitVersion(info.Version)
+	oMajor, _, _, oErr := splitVersion(other.Version)
 
 	// if our own version number is not formatted right, we messed up
 	if iErr != nil {
@@ -99,11 +100,6 @@ func (info NodeInfo) CompatibleWith(other NodeInfo) error {
 		return fmt.Errorf("Peer is on a different major version. Got %v, expected %v", oMajor, iMajor)
 	}
 
-	// minor version can differ
-	if iMinor != oMinor {
-		// ok
-	}
-
 	// nodes must be on the same network
 	if info.Network != other.Network {
 		return fmt.Errorf("Peer is on a different network. Got %v, expected %v", other.Network, info.Network)