Merge pull request #559 from tendermint/release-v0.10.1

Release v0.10.1

.editorconfig

@@ -0,0 +1,15 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[Makefile]
+indent_style = tab
+
+[*.sh]
+indent_style = tab

.github/ISSUE_TEMPLATE

@@ -0,0 +1,41 @@
+<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.-->
+
+**Is this a BUG REPORT or FEATURE REQUEST?** (choose one):
+
+<!--
+If this is a BUG REPORT, please:
+  - Fill in as much of the template below as you can.
+
+If this is a FEATURE REQUEST, please:
+  - Describe *in detail* the feature/behavior/change you'd like to see.
+
+In both cases, be ready for followup questions, and please respond in a timely
+manner. We might ask you to provide additional logs and data (tendermint & app)
+in a case of bug.
+-->
+
+**Tendermint version** (use `tendermint version` or `git rev-parse --verify HEAD` if installed from source):
+
+
+**ABCI app** (name for built-in, URL for self-written if it's publicly available):
+
+
+**Merkleeyes version** (use `git rev-parse --verify HEAD`, skip if you don't use it):
+
+
+**Environment**:
+- **OS** (e.g. from /etc/os-release):
+- **Install tools**:
+- **Others**:
+
+
+**What happened**:
+
+
+**What you expected to happen**:
+
+
+**How to reproduce it** (as minimally and precisely as possible):
+
+
+**Anything else do we need to know**:

.gitignore

@@ -12,3 +12,6 @@ remote_dump
 vendor
 .vagrant
 test/p2p/data/
+test/logs
+.glide
+coverage.txt

CHANGELOG.md

@@ -0,0 +1,361 @@
+# Changelog
+
+## 0.10.1 (June 28, 2017)
+
+FEATURES:
+- Use `--trace` to get stack traces for logged errors
+- types: GenesisDoc.ValidatorHash returns the hash of the genesis validator set 
+- types: GenesisDocFromFile parses a GenesiDoc from a JSON file
+
+IMPROVEMENTS:
+- Add a Code of Conduct
+- Variety of improvements as suggested by `megacheck` tool
+- rpc: deduplicate tests between rpc/client and rpc/tests
+- rpc: addresses without a protocol prefix default to `tcp://`. `http://` is also accepted as an alias for `tcp://`
+- cmd: commands are more easily reuseable from other tools
+- DOCKER: automate build/push
+
+BUG FIXES:
+- Fix log statements using keys with spaces (logger does not currently support spaces)
+- rpc: set logger on websocket connection
+- rpc: fix ws connection stability by setting write deadline on pings
+
+## 0.10.0 (June 2, 2017)
+
+Includes major updates to configuration, logging, and json serialization.
+Also includes the Grand Repo-Merge of 2017.
+
+BREAKING CHANGES:
+
+- Config and Flags:
+  - The `config` map is replaced with a [`Config` struct](https://github.com/tendermint/tendermint/blob/master/config/config.go#L11), 
+containing substructs: `BaseConfig`, `P2PConfig`, `MempoolConfig`, `ConsensusConfig`, `RPCConfig`
+  - This affects the following flags:
+    - `--seeds` is now `--p2p.seeds`
+    - `--node_laddr` is now `--p2p.laddr`
+    - `--pex` is now `--p2p.pex`
+    - `--skip_upnp` is now `--p2p.skip_upnp`
+    - `--rpc_laddr` is now `--rpc.laddr`
+    - `--grpc_laddr` is now `--rpc.grpc_laddr`
+  - Any configuration option now within a substract must come under that heading in the `config.toml`, for instance:
+    ```
+    [p2p]
+    laddr="tcp://1.2.3.4:46656"
+    
+    [consensus]
+    timeout_propose=1000
+    ```
+  - Use viper and `DefaultConfig() / TestConfig()` functions to handle defaults, and remove `config/tendermint` and `config/tendermint_test`
+  - Change some function and method signatures to 
+  - Change some [function and method signatures](https://gist.github.com/ebuchman/640d5fc6c2605f73497992fe107ebe0b) accomodate new config
+
+- Logger
+  - Replace static `log15` logger with a simple interface, and provide a new implementation using `go-kit`. 
+See our new [logging library](https://github.com/tendermint/tmlibs/log) and [blog post](https://tendermint.com/blog/abstracting-the-logger-interface-in-go) for more details
+  - Levels `warn` and `notice` are removed (you may need to change them in your `config.toml`!)
+  - Change some [function and method signatures](https://gist.github.com/ebuchman/640d5fc6c2605f73497992fe107ebe0b) to accept a logger
+
+- JSON serialization:
+  - Replace `[TypeByte, Xxx]` with `{"type": "some-type", "data": Xxx}` in RPC and all `.json` files by using `go-wire/data`. For instance, a public key is now:
+    ```
+    "pub_key": {
+      "type": "ed25519",
+      "data": "83DDF8775937A4A12A2704269E2729FCFCD491B933C4B0A7FFE37FE41D7760D0"
+    }
+    ```
+  - Remove type information about RPC responses, so `[TypeByte, {"jsonrpc": "2.0", ... }]` is now just `{"jsonrpc": "2.0", ... }`
+  - Change `[]byte` to `data.Bytes` in all serialized types (for hex encoding)
+  - Lowercase the JSON tags in `ValidatorSet` fields
+  - Introduce `EventDataInner` for serializing events
+
+- Other:
+  - Send InitChain message in handshake if `appBlockHeight == 0`
+  - Do not include the `Accum` field when computing the validator hash. This makes the ValidatorSetHash unique for a given validator set, rather than changing with every block (as the Accum changes)
+  - Unsafe RPC calls are not enabled by default. This includes `/dial_seeds`, and all calls prefixed with `unsafe`. Use the `--rpc.unsafe` flag to enable.
+
+
+FEATURES:
+
+- Per-module log levels. For instance, the new default is `state:info,*:error`, which means the `state` package logs at `info` level, and everything else logs at `error` level
+- Log if a node is validator or not in every consensus round
+- Use ldflags to set git hash as part of the version
+- Ignore `address` and `pub_key` fields in `priv_validator.json` and overwrite them with the values derrived from the `priv_key`
+
+IMPROVEMENTS:
+
+- Merge `tendermint/go-p2p -> tendermint/tendermint/p2p` and `tendermint/go-rpc -> tendermint/tendermint/rpc/lib`
+- Update paths for grand repo merge:
+  - `go-common -> tmlibs/common`
+  - `go-data -> go-wire/data`
+  - All other `go-` libs, except `go-crypto` and `go-wire`, are merged under `tmlibs`
+- No global loggers (loggers are passed into constructors, or preferably set with a SetLogger method)
+- Return HTTP status codes with errors for RPC responses
+- Limit `/blockchain_info` call to return a maximum of 20 blocks
+- Use `.Wrap()` and `.Unwrap()` instead of eg. `PubKeyS` for `go-crypto` types
+- RPC JSON responses use pretty printing (via `json.MarshalIndent`)
+- Color code different instances of the consensus for tests 
+- Isolate viper to `cmd/tendermint/commands` and do not read config from file for tests
+
+
+## 0.9.2 (April 26, 2017)
+
+BUG FIXES:
+
+- Fix bug in `ResetPrivValidator` where we were using the global config and log (causing external consumers, eg. basecoin, to fail).
+
+## 0.9.1 (April 21, 2017)
+
+FEATURES:
+
+- Transaction indexing - txs are indexed by their hash using a simple key-value store; easily extended to more advanced indexers
+- New `/tx?hash=X` endpoint to query for transactions and their DeliverTx result by hash. Optionally returns a proof of the tx's inclusion in the block
+- `tendermint testnet` command initializes files for a testnet
+
+IMPROVEMENTS:
+
+- CLI now uses Cobra framework
+- TMROOT is now TMHOME (TMROOT will stop working in 0.10.0)
+- `/broadcast_tx_XXX` also returns the Hash (can be used to query for the tx)
+- `/broadcast_tx_commit` also returns the height the block was committed in
+- ABCIResponses struct persisted to disk before calling Commit; makes handshake replay much cleaner
+- WAL uses #ENDHEIGHT instead of #HEIGHT (#HEIGHT will stop working in 0.10.0)
+- Peers included via `--seeds`, under `seeds` in the config, or in `/dial_seeds` are now persistent, and will be reconnected to if the connection breaks
+
+BUG FIXES: 
+
+- Fix bug in fast-sync where we stop syncing after a peer is removed, even if they're re-added later
+- Fix handshake replay to handle validator set changes and results of DeliverTx when we crash after app.Commit but before state.Save()
+
+## 0.9.0 (March 6, 2017)
+
+BREAKING CHANGES:
+
+- Update ABCI to v0.4.0, where Query is now `Query(RequestQuery) ResponseQuery`, enabling precise proofs at particular heights:
+
+```
+message RequestQuery{
+	bytes data = 1;
+	string path = 2;
+	uint64 height = 3;
+	bool prove = 4; 
+}
+
+message ResponseQuery{
+	CodeType          code        = 1;
+	int64             index       = 2;
+	bytes             key         = 3;
+	bytes             value       = 4;
+	bytes             proof       = 5;
+	uint64            height      = 6;
+	string            log         = 7;
+}
+```
+
+
+- `BlockMeta` data type unifies its Hash and PartSetHash under a `BlockID`:
+
+```
+type BlockMeta struct {
+	BlockID BlockID `json:"block_id"` // the block hash and partsethash
+	Header  *Header `json:"header"`   // The block's Header
+}
+```
+
+- `ValidatorSet.Proposer` is exposed as a field and persisted with the `State`. Use `GetProposer()` to initialize or update after validator-set changes. 
+
+- `tendermint gen_validator` command output is now pure JSON
+
+FEATURES:
+
+- New RPC endpoint `/commit?height=X` returns header and commit for block at height `X`
+- Client API for each endpoint, including mocks for testing
+
+IMPROVEMENTS:
+
+- `Node` is now a `BaseService`
+- Simplified starting Tendermint in-process from another application
+- Better organized Makefile
+- Scripts for auto-building binaries across platforms
+- Docker image improved, slimmed down (using Alpine), and changed from tendermint/tmbase to tendermint/tendermint
+- New repo files: `CONTRIBUTING.md`, Github `ISSUE_TEMPLATE`, `CHANGELOG.md`
+- Improvements on CircleCI for managing build/test artifacts
+- Handshake replay is doen through the consensus package, possibly using a mockApp
+- Graceful shutdown of RPC listeners
+- Tests for the PEX reactor and DialSeeds
+
+BUG FIXES:
+
+- Check peer.Send for failure before updating PeerState in consensus
+- Fix panic in `/dial_seeds` with invalid addresses
+- Fix proposer selection logic in ValidatorSet by taking the address into account in the `accumComparable`
+- Fix inconcistencies with `ValidatorSet.Proposer` across restarts by persisting it in the `State`
+
+
+## 0.8.0 (January 13, 2017)
+
+BREAKING CHANGES:
+
+- New data type `BlockID` to represent blocks:
+
+```
+type BlockID struct {
+	Hash        []byte        `json:"hash"`
+	PartsHeader PartSetHeader `json:"parts"`
+}
+```
+
+- `Vote` data type now includes validator address and index: 
+
+```
+type Vote struct {
+	ValidatorAddress []byte           `json:"validator_address"`
+	ValidatorIndex   int              `json:"validator_index"`
+	Height           int              `json:"height"`
+	Round            int              `json:"round"`
+	Type             byte             `json:"type"`
+	BlockID          BlockID          `json:"block_id"` // zero if vote is nil.
+	Signature        crypto.Signature `json:"signature"`
+}
+```
+
+- Update TMSP to v0.3.0, where it is now called ABCI and AppendTx is DeliverTx
+- Hex strings in the RPC are now "0x" prefixed
+
+
+FEATURES:
+
+- New message type on the ConsensusReactor, `Maj23Msg`, for peers to alert others they've seen a Maj23, 
+in order to track and handle conflicting votes intelligently to prevent Byzantine faults from causing halts:
+
+```
+type VoteSetMaj23Message struct {
+	Height  int
+	Round   int
+	Type    byte
+	BlockID types.BlockID
+}
+```
+
+- Configurable block part set size
+- Validator set changes
+- Optionally skip TimeoutCommit if we have all the votes
+- Handshake between Tendermint and App on startup to sync latest state and ensure consistent recovery from crashes
+- GRPC server for BroadcastTx endpoint
+
+IMPROVEMENTS:
+
+- Less verbose logging
+- Better test coverage (37% -> 49%)
+- Canonical SignBytes for signable types
+- Write-Ahead Log for Mempool and Consensus via tmlibs/autofile 
+- Better in-process testing for the consensus reactor and byzantine faults
+- Better crash/restart testing for individual nodes at preset failure points, and of networks at arbitrary points
+- Better abstraction over timeout mechanics
+
+BUG FIXES:
+
+- Fix memory leak in mempool peer
+- Fix panic on POLRound=-1
+- Actually set the CommitTime
+- Actually send BeginBlock message
+- Fix a liveness issues caused by Byzantine proposals/votes. Uses the new `Maj23Msg`.
+
+
+## 0.7.4 (December 14, 2016)
+
+FEATURES:
+
+- Enable the Peer Exchange reactor with the `--pex` flag for more resilient gossip network (feature still in development, beware dragons)
+
+IMPROVEMENTS:
+
+- Remove restrictions on RPC endpoint `/dial_seeds` to enable manual network configuration
+
+## 0.7.3 (October 20, 2016)
+
+IMPROVEMENTS:
+
+- Type safe FireEvent
+- More WAL/replay tests
+- Cleanup some docs
+
+BUG FIXES:
+
+- Fix deadlock in mempool for synchronous apps
+- Replay handles non-empty blocks
+- Fix race condition in HeightVoteSet
+
+## 0.7.2 (September 11, 2016)
+
+BUG FIXES:
+
+- Set mustConnect=false so tendermint will retry connecting to the app
+
+## 0.7.1 (September 10, 2016)
+
+FEATURES:
+
+- New TMSP connection for Query/Info
+- New RPC endpoints:
+	- `tmsp_query`
+	- `tmsp_info`
+- Allow application to filter peers through Query (off by default)
+
+IMPROVEMENTS:
+
+- TMSP connection type enforced at compile time
+- All listen/client urls use a "tcp://" or "unix://" prefix
+
+BUG FIXES:
+
+- Save LastSignature/LastSignBytes to `priv_validator.json` for recovery
+- Fix event unsubscribe
+- Fix fastsync/blockchain reactor
+
+## 0.7.0 (August 7, 2016)
+
+BREAKING CHANGES:
+
+- Strict SemVer starting now!
+- Update to ABCI v0.2.0
+- Validation types now called Commit
+- NewBlock event only returns the block header
+
+
+FEATURES:
+
+- TMSP and RPC support TCP and UNIX sockets
+- Addition config options including block size and consensus parameters
+- New WAL mode `cswal_light`; logs only the validator's own votes
+- New RPC endpoints: 
+	- for starting/stopping profilers, and for updating config
+	- `/broadcast_tx_commit`, returns when tx is included in a block, else an error
+	- `/unsafe_flush_mempool`, empties the mempool
+
+
+IMPROVEMENTS:
+
+- Various optimizations
+- Remove bad or invalidated transactions from the mempool cache (allows later duplicates)
+- More elaborate testing using CircleCI including benchmarking throughput on 4 digitalocean droplets
+
+BUG FIXES:
+
+- Various fixes to WAL and replay logic
+- Various race conditions
+
+## PreHistory
+
+Strict versioning only began with the release of v0.7.0, in late summer 2016.
+The project itself began in early summer 2014 and was workable decentralized cryptocurrency software by the end of that year.
+Through the course of 2015, in collaboration with Eris Industries (now Monax Indsutries), 
+many additional features were integrated, including an implementation from scratch of the Ethereum Virtual Machine.
+That implementation now forms the heart of [ErisDB](https://github.com/eris-ltd/eris-db).
+In the later half of 2015, the consensus algorithm was upgraded with a more asynchronous design and a more deterministic and robust implementation.
+
+By late 2015, frustration with the difficulty of forking a large monolithic stack to create alternative cryptocurrency designs led to the 
+invention of the Application Blockchain Interface (ABCI), then called the Tendermint Socket Protocol (TMSP).
+The Ethereum Virtual Machine and various other transaction features were removed, and Tendermint was whittled down to a core consensus engine
+driving an application running in another process. 
+The ABCI interface and implementation were iterated on and improved over the course of 2016,
+until versioned history kicked in with v0.7.0.

CODE_OF_CONDUCT.md

@@ -0,0 +1,56 @@
+# The Tendermint Code of Conduct
+This code of conduct applies to all projects run by the Tendermint/COSMOS team and hence to tendermint.
+
+
+----
+
+
+# Conduct
+## Contact: adrian@tendermint.com
+
+* We are committed to providing a friendly, safe and welcoming environment for all, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, nationality, or other similar characteristic.
+
+* On Slack, please avoid using overtly sexual nicknames or other nicknames that might detract from a friendly, safe and welcoming environment for all.
+
+* Please be kind and courteous. There’s no need to be mean or rude.
+
+* Respect that people have differences of opinion and that every design or implementation choice carries a trade-off and numerous costs. There is seldom a right answer.
+
+* Please keep unstructured critique to a minimum. If you have solid ideas you want to experiment with, make a fork and see how it works.
+
+* We will exclude you from interaction if you insult, demean or harass anyone. That is not welcome behaviour. We interpret the term “harassment” as including the definition in the [Citizen Code of Conduct](http://citizencodeofconduct.org/); if you have any lack of clarity about what might be included in that concept, please read their definition. In particular, we don’t tolerate behavior that excludes people in socially marginalized groups.
+
+* Private harassment is also unacceptable. No matter who you are, if you feel you have been or are being harassed or made uncomfortable by a community member, please contact one of the channel admins or the person mentioned above immediately. Whether you’re a regular contributor or a newcomer, we care about making this community a safe place for you and we’ve got your back.
+
+* Likewise any spamming, trolling, flaming, baiting or other attention-stealing behaviour is not welcome.
+
+
+----
+
+
+# Moderation
+These are the policies for upholding our community’s standards of conduct. If you feel that a thread needs moderation, please contact the above mentioned person.
+
+1. Remarks that violate the Tendermint/COSMOS standards of conduct, including hateful, hurtful, oppressive, or exclusionary remarks, are not allowed. (Cursing is allowed, but never targeting another user, and never in a hateful manner.)
+
+2. Remarks that moderators find inappropriate, whether listed in the code of conduct or not, are also not allowed.
+
+3. Moderators will first respond to such remarks with a warning.
+
+4. If the warning is unheeded, the user will be “kicked,” i.e., kicked out of the communication channel to cool off.
+
+5. If the user comes back and continues to make trouble, they will be banned, i.e., indefinitely excluded.
+
+6. Moderators may choose at their discretion to un-ban the user if it was a first offense and they offer the offended party a genuine apology.
+
+7. If a moderator bans someone and you think it was unjustified, please take it up with that moderator, or with a different moderator, in private. Complaints about bans in-channel are not allowed.
+
+8. Moderators are held to a higher standard than other community members. If a moderator creates an inappropriate situation, they should expect less leeway than others.
+
+In the Tendermint/COSMOS community we strive to go the extra step to look out for each other. Don’t just aim to be technically unimpeachable, try to be your best self. In particular, avoid flirting with offensive or sensitive issues, particularly if they’re off-topic; this all too often leads to unnecessary fights, hurt feelings, and damaged trust; worse, it can drive people away from the community entirely.
+
+And if someone takes issue with something you said or did, resist the urge to be defensive. Just stop doing what it was they complained about and apologize. Even if you feel you were misinterpreted or unfairly accused, chances are good there was something you could’ve communicated better — remember that it’s your responsibility to make your fellow Cosmonauts comfortable. Everyone wants to get along and we are all here first and foremost because we want to talk about cool technology. You will find that people will be eager to assume good intent and forgive as long as you earn their trust.
+
+The enforcement policies listed above apply to all official Tendermint/COSMOS venues.For other projects adopting the Tendermint/COSMOS Code of Conduct, please contact the maintainers of those projects for enforcement. If you wish to use this code of conduct for your own project, consider explicitly mentioning your moderation policy or making a copy with your own moderation policy so as to avoid confusion.
+
+*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling), the [Contributor Covenant v1.3.0](http://contributor-covenant.org/version/1/3/0/) and the [Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html).

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+# Contributing guidelines
+
+**Thanks for considering making contributions to Tendermint!**
+
+Please follow standard github best practices: fork the repo, **branch from the
+tip of develop**, make some commits, test your code changes with `make test`,
+and submit a pull request to develop.
+
+See the [open issues](https://github.com/tendermint/tendermint/issues) for
+things we need help with!
+
+Please make sure to use `gofmt` before every commit - the easiest way to do
+this is have your editor run it for you upon saving a file.
+
+You can read the full guide [on our
+site](https://tendermint.com/docs/guides/contributing).

DOCKER/Dockerfile

@@ -1,54 +1,45 @@
-# Pull base image.
-FROM golang:1.6
+FROM alpine:3.6
 
-ENV USER tmuser
-ENV DATA_ROOT /data/tendermint
+# This is the release of tendermint to pull in.
+ENV TM_VERSION 0.10.0
+ENV TM_SHA256SUM a29852b8d51c00db93c87c3d148fa419a047abd38f32b2507a905805131acc19
+
+# Tendermint will be looking for genesis file in /tendermint (unless you change
+# `genesis_file` in config.toml). You can put your config.toml and private
+# validator file into /tendermint.
+#
+# The /tendermint/data dir is used by tendermint to store state.
+ENV DATA_ROOT /tendermint
+ENV TMHOME $DATA_ROOT
 
 # Set user right away for determinism
-RUN groupadd -r $USER \
-  && useradd -r -s /bin/false -g $USER $USER
-
-# Create home directory for USER
-# Needed for nodejs/nom
-RUN mkdir -p /home/$USER \
-  && chown -R $USER:$USER /home/$USER
+RUN addgroup tmuser && \
+    adduser -S -G tmuser tmuser
 
 # Create directory for persistence and give our user ownership
-RUN mkdir -p $DATA_ROOT \
-  && chown -R $USER:$USER $DATA_ROOT
+RUN mkdir -p $DATA_ROOT && \
+    chown -R tmuser:tmuser $DATA_ROOT
 
-# Set the env variables to non-interactive
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBIAN_PRIORITY critical
-ENV DEBCONF_NOWARNINGS yes
-ENV TERM linux
-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+# jq and curl used for extracting `pub_key` from private validator while
+# deploying tendermint with Kubernetes. It is nice to have bash so the users
+# could execute bash commands.
+RUN apk add --no-cache bash curl jq
 
-# Grab deps (git)
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends \
-    git && \
-  rm -rf /var/lib/apt/lists/*
+RUN apk add --no-cache openssl && \
+    wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \
+    echo "${TM_SHA256SUM}  tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \
+    unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \
+    apk del openssl && \
+    rm -f tendermint_${TM_VERSION}_linux_amd64.zip
 
-# Grab deps (node)
-RUN curl -sL https://deb.nodesource.com/setup_5.x | bash -
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends \
-    nodejs && \
-  rm -rf /var/lib/apt/lists/*
-
-# Copy run.sh
-COPY ./run.sh $DATA_ROOT/run.sh
-RUN chmod +x $DATA_ROOT/run.sh
-
-# Persist data, set user
-WORKDIR $DATA_ROOT
+# Expose the data directory as a volume since there's mutable state in there
 VOLUME $DATA_ROOT
-USER $USER
-ENV TMROOT $DATA_ROOT
 
+# p2p port
 EXPOSE 46656
+# rpc port
 EXPOSE 46657
 
-# Run tendermint
-CMD ["./run.sh"]
+ENTRYPOINT ["tendermint"]
+
+CMD ["node", "--moniker=`hostname`"]

DOCKER/Dockerfile.develop

@@ -0,0 +1,35 @@
+FROM alpine:3.6
+
+ENV DATA_ROOT /tendermint
+ENV TMHOME $DATA_ROOT
+
+RUN addgroup tmuser && \
+    adduser -S -G tmuser tmuser
+
+RUN mkdir -p $DATA_ROOT && \
+    chown -R tmuser:tmuser $DATA_ROOT
+
+RUN apk add --no-cache bash curl jq
+
+ENV GOPATH /go
+ENV PATH "$PATH:/go/bin"
+RUN mkdir -p /go/src/github.com/tendermint/tendermint && \
+    apk add --no-cache go build-base git && \
+    cd /go/src/github.com/tendermint/tendermint && \
+    git clone https://github.com/tendermint/tendermint . && \
+    git checkout develop && \
+    make get_vendor_deps && \
+    make install && \
+    glide cc && \
+    cd - && \
+    rm -rf /go/src/github.com/tendermint/tendermint && \
+    apk del go build-base git
+
+VOLUME $DATA_ROOT
+
+EXPOSE 46656
+EXPOSE 46657
+
+ENTRYPOINT ["tendermint"]
+
+CMD ["node", "--moniker=`hostname`", "--proxy_app=dummy"]

DOCKER/Makefile

@@ -0,0 +1,13 @@
+build:
+	@sh -c "'$(CURDIR)/build.sh'"
+
+push:
+	@sh -c "'$(CURDIR)/push.sh'"
+
+build_develop:
+	docker build -t "tendermint/tendermint:develop" -f Dockerfile.develop .
+
+push_develop:
+	docker push "tendermint/tendermint:develop"
+
+.PHONY: build build_develop push push_develop

DOCKER/README.md

@@ -1,24 +1,58 @@
-# Docker
+# Supported tags and respective `Dockerfile` links
 
-Tendermint uses docker for deployment of testnets via the [mintnet](github.com/tendermint/mintnet) tool. 
+- `0.10.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/e5342f4054ab784b2cd6150e14f01053d7c8deb2/DOCKER/Dockerfile)
+- `0.9.1`, `0.9`, [(Dockerfile)](https://github.com/tendermint/tendermint/blob/809e0e8c5933604ba8b2d096803ada7c5ec4dfd3/DOCKER/Dockerfile)
+- `0.9.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/d474baeeea6c22b289e7402449572f7c89ee21da/DOCKER/Dockerfile)
+- `0.8.0`, `0.8` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/bf64dd21fdb193e54d8addaaaa2ecf7ac371de8c/DOCKER/Dockerfile)
+- `develop` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/master/DOCKER/Dockerfile.develop)
 
-For faster development iterations (ie. to avoid docker builds), 
-the dockerfile just sets up the OS, and tendermint is fetched/installed at runtime.
+`develop` tag points to the [develop](https://github.com/tendermint/tendermint/tree/develop) branch.
 
-For the deterministic docker builds used in testing, see the [tests directory](https://github.com/tendermint/tendermint/tree/master/test)
+# Tendermint
 
-# Build and run a docker image and container
+Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines.
 
-These are notes for the dev team. 
+For more background, see the [introduction](https://tendermint.com/intro).
+
+To get started developing applications, see the [application developers guide](https://tendermint.com/docs/guides/app-development).
+
+# How to use this image
+
+## Start one instance of the Tendermint core with the `dummy` app
+
+A very simple example of a built-in app and Tendermint core in one container.
 
 ```
-# Build base Docker image
-# Make sure ./run.sh exists.
-docker build -t tendermint/tmbase -f Dockerfile .
-
-# Log into dockerhub
-docker login
-
-# Push latest build to dockerhub
-docker push tendermint/tmbase
+docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init
+docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint node --proxy_app=dummy
 ```
+
+## mintnet-kubernetes
+
+If you want to see many containers talking to each other, consider using [mintnet-kubernetes](https://github.com/tendermint/tools/tree/master/mintnet-kubernetes), which is a tool for running Tendermint-based applications on a Kubernetes cluster.
+
+# Supported Docker versions
+
+This image is officially supported on Docker version 1.13.1.
+
+Support for older versions (down to 1.6) is provided on a best-effort basis.
+
+Please see [the Docker installation documentation](https://docs.docker.com/installation/) for details on how to upgrade your Docker daemon.
+
+# License
+
+View [license information](https://raw.githubusercontent.com/tendermint/tendermint/master/LICENSE) for the software contained in this image.
+
+# User Feedback
+
+## Issues
+
+If you have any problems with or questions about this image, please contact us through a [GitHub](https://github.com/tendermint/tendermint/issues) issue. If the issue is related to a CVE, please check for [a `cve-tracker` issue on the `official-images` repository](https://github.com/docker-library/official-images/issues?q=label%3Acve-tracker) first.
+
+You can also reach the image maintainers via [Slack](http://forum.tendermint.com:3000/).
+
+## Contributing
+
+You are invited to contribute new features, fixes, or updates, large or small; we are always thrilled to receive pull requests, and do our best to process them as fast as we can.
+
+Before you start to code, we recommend discussing your plans through a [GitHub](https://github.com/tendermint/tendermint/issues) issue, especially for more ambitious contributions. This gives other contributors a chance to point you in the right direction, give you feedback on your design, and help you find out if someone else is working on the same thing.

DOCKER/build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+# Get the tag from the version, or try to figure it out.
+if [ -z "$TAG" ]; then
+	TAG=$(awk -F\" '/Version =/ { print $2; exit }' < ../version/version.go)
+fi
+if [ -z "$TAG" ]; then
+		echo "Please specify a tag."
+		exit 1
+fi
+
+TAG_NO_PATCH=${TAG%.*}
+
+read -p "==> Build 3 docker images with the following tags (latest, $TAG, $TAG_NO_PATCH)? y/n" -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+		docker build -t "tendermint/tendermint" -t "tendermint/tendermint:$TAG" -t "tendermint/tendermint:$TAG_NO_PATCH" .
+fi

DOCKER/push.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+# Get the tag from the version, or try to figure it out.
+if [ -z "$TAG" ]; then
+	TAG=$(awk -F\" '/Version =/ { print $2; exit }' < ../version/version.go)
+fi
+if [ -z "$TAG" ]; then
+		echo "Please specify a tag."
+		exit 1
+fi
+
+TAG_NO_PATCH=${TAG%.*}
+
+read -p "==> Push 3 docker images with the following tags (latest, $TAG, $TAG_NO_PATCH)? y/n" -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+	docker push "tendermint/tendermint:latest"
+	docker push "tendermint/tendermint:$TAG"
+	docker push "tendermint/tendermint:$TAG_NO_PATCH"
+fi

DOCKER/run.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-mkdir -p $GOPATH/src/$TMREPO
-cd $GOPATH/src/$TMREPO
-git clone https://$TMREPO.git .
-git fetch
-git reset --hard $TMHEAD
-go get -d $TMREPO/cmd/tendermint
-make
-tendermint node --seeds="$TMSEEDS" --moniker="$TMNAME"

Makefile

@@ -1,59 +1,83 @@
-.PHONY: get_deps build all list_deps install
+GOTOOLS = \
+					github.com/mitchellh/gox \
+					github.com/Masterminds/glide \
+					honnef.co/go/tools/cmd/megacheck
 
-all: get_deps install test
+PACKAGES=$(shell go list ./... | grep -v '/vendor/')
+BUILD_TAGS?=tendermint
+TMHOME = $${TMHOME:-$$HOME/.tendermint}
 
-TMROOT = $${TMROOT:-$$HOME/.tendermint}
-define NEWLINE
+all: install test
 
-
-endef
-NOVENDOR = go list github.com/tendermint/tendermint/... | grep -v /vendor/
-
-install: get_deps
-	go install github.com/tendermint/tendermint/cmd/tendermint
+install: get_vendor_deps
+	@go install --ldflags '-extldflags "-static"' \
+		--ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" ./cmd/tendermint
 
 build:
-	go build -o build/tendermint github.com/tendermint/tendermint/cmd/tendermint
+	go build \
+		--ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`"  -o build/tendermint ./cmd/tendermint/
 
 build_race:
-	go build -race -o build/tendermint github.com/tendermint/tendermint/cmd/tendermint
+	go build -race -o build/tendermint ./cmd/tendermint
 
-test: build
-	go test `${NOVENDOR}`
+# dist builds binaries for all platforms and packages them for distribution
+dist:
+	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
 
-test_race: build
-	go test -race `${NOVENDOR}`
+test:
+	@echo "--> Running go test"
+	@go test $(PACKAGES)
+
+test_race:
+	@echo "--> Running go test --race"
+	@go test -v -race $(PACKAGES)
 
 test_integrations:
-	bash ./test/test.sh
+	@bash ./test/test.sh
 
-test100: build
-	for i in {1..100}; do make test; done
+test100:
+	@for i in {1..100}; do make test; done
 
 draw_deps:
-	# requires brew install graphviz
-	go get github.com/hirokidaichi/goviz
-	goviz -i github.com/tendermint/tendermint/cmd/tendermint | dot -Tpng -o huge.png
+	# requires brew install graphviz or apt-get install graphviz
+	go get github.com/RobotsAndPencils/goviz
+	@goviz -i github.com/tendermint/tendermint/cmd/tendermint -d 3 | dot -Tpng -o dependency-graph.png
 
 list_deps:
-	go list -f '{{join .Deps "\n"}}' github.com/tendermint/tendermint/... | \
+	@go list -f '{{join .Deps "\n"}}' ./... | \
 		grep -v /vendor/ | sort | uniq | \
-	  xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
+		xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
 
 get_deps:
-	go get -d `${NOVENDOR}`
-	go list -f '{{join .TestImports "\n"}}' github.com/tendermint/tendermint/... | \
+	@echo "--> Running go get"
+	@go get -v -d $(PACKAGES)
+	@go list -f '{{join .TestImports "\n"}}' ./... | \
 		grep -v /vendor/ | sort | uniq | \
-		xargs go get
+		xargs go get -v -d
 
-get_vendor_deps:
-	go get github.com/Masterminds/glide
-	rm -rf vendor/
-	glide install
+get_vendor_deps: ensure_tools
+	@rm -rf vendor/
+	@echo "--> Running glide install"
+	@glide install
 
-update_deps:
-	go get -d -u github.com/tendermint/tendermint/...
+update_deps: tools
+	@echo "--> Updating dependencies"
+	@go get -d -u ./...
 
 revision:
-	-echo `git rev-parse --verify HEAD` > $(TMROOT)/revision
-	-echo `git rev-parse --verify HEAD` >> $(TMROOT)/revision_history
+	-echo `git rev-parse --verify HEAD` > $(TMHOME)/revision
+	-echo `git rev-parse --verify HEAD` >> $(TMHOME)/revision_history
+
+tools:
+	go get -u -v $(GOTOOLS)
+
+ensure_tools:
+	go get $(GOTOOLS)
+
+### Formatting, linting, and vetting
+
+megacheck:
+	@for pkg in ${PACKAGES}; do megacheck "$$pkg"; done
+
+
+.PHONY: install build build_race dist test test_race test_integrations test100 draw_deps list_deps get_deps get_vendor_deps update_deps revision tools

README.md

@@ -10,13 +10,15 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
 )](https://godoc.org/github.com/tendermint/tendermint)
 [![chat](https://img.shields.io/badge/slack-join%20chat-pink.svg)](http://forum.tendermint.com:3000/)
 [![license](https://img.shields.io/github/license/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/blob/master/LICENSE)
+[![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint)
+
 
 Branch    | Tests | Coverage | Report Card
 ----------|-------|----------|-------------
 develop   | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/develop/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | [![Go Report Card](https://goreportcard.com/badge/github.com/tendermint/tendermint/tree/develop)](https://goreportcard.com/report/github.com/tendermint/tendermint/tree/develop)
 master    | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | [![Go Report Card](https://goreportcard.com/badge/github.com/tendermint/tendermint/tree/master)](https://goreportcard.com/report/github.com/tendermint/tendermint/tree/master)
 
-_NOTE: This is yet pre-alpha non-production-quality software._
+_NOTE: This is alpha software. Please contact us if you intend to run it in production._
 
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language,
 and securely replicates it on many machines.
@@ -25,15 +27,22 @@ For more background, see the [introduction](https://tendermint.com/intro).
 
 To get started developing applications, see the [application developers guide](https://tendermint.com/docs/guides/app-development).
 
+### Code of Conduct
+Please read, understand and adhere to our [code of conduct](CODE_OF_CONDUCT.md).
+
 ## Install
 
+To download pre-built binaries, see our [downloads page](https://tendermint.com/intro/getting-started/download).
+
+To install from source, you should be able to:
+
 `go get -u github.com/tendermint/tendermint/cmd/tendermint`
 
-For more details (or if it fails), see the [install guide](https://tendermint.com/intro/getting-started/install).
+For more details (or if it fails), see the [install guide](https://tendermint.com/docs/guides/install).
 
 ## Contributing
 
-Yay open source! Please see our [contributing guidelines](https://tendermint.com/guides/contributing).
+Yay open source! Please see our [contributing guidelines](https://tendermint.com/docs/guides/contributing).
 
 ## Resources
 
@@ -45,16 +54,19 @@ Yay open source! Please see our [contributing guidelines](https://tendermint.com
 
 ### Sub-projects
 
-* [ABCI](http://github.com/tendermint/abci)
-* [Mintnet](http://github.com/tendermint/mintnet)
-* [Go-Wire](http://github.com/tendermint/go-wire)
-* [Go-P2P](http://github.com/tendermint/go-p2p)
-* [Go-Merkle](http://github.com/tendermint/go-merkle)
+* [ABCI](http://github.com/tendermint/abci), the Application Blockchain Interface
+* [Go-Wire](http://github.com/tendermint/go-wire), a deterministic serialization library
+* [Go-Crypto](http://github.com/tendermint/go-crypto), an elliptic curve cryptography library 
+* [TmLibs](http://github.com/tendermint/tmlibs), an assortment of Go libraries
+* [Merkleeyes](http://github.com/tendermint/merkleeyes), a balanced, binary Merkle tree for ABCI apps
+
+### Tools
+* [Deployment, Benchmarking, and Monitoring](https://github.com/tendermint/tools)
 
 ### Applications
 
-* [Ethermint](http://github.com/tendermint/ethermint)
-* [Basecoin](http://github.com/tendermint/basecoin)
+* [Ethermint](http://github.com/tendermint/ethermint): Ethereum on Tendermint
+* [Basecoin](http://github.com/tendermint/basecoin), a cryptocurrency application framework
 
 ### More 
 

Vagrantfile

@@ -5,7 +5,7 @@ Vagrant.configure("2") do |config|
   config.vm.box = "ubuntu/trusty64"
 
   config.vm.provider "virtualbox" do |v|
-    v.memory = 2048
+    v.memory = 4096
     v.cpus = 2
   end
 
@@ -15,19 +15,24 @@ Vagrant.configure("2") do |config|
 
     wget -qO- https://get.docker.com/ | sh
     usermod -a -G docker vagrant
+    apt-get autoremove -y
 
-    curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
-    tar -xvf go1.7.linux-amd64.tar.gz
+    curl -O https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
+    tar -xvf go1.8.linux-amd64.tar.gz
+    rm -rf /usr/local/go
     mv go /usr/local
-    echo 'export PATH=$PATH:/usr/local/go/bin' >> /home/vagrant/.profile
+    rm -f go1.8.linux-amd64.tar.gz
     mkdir -p /home/vagrant/go/bin
-    chown -R vagrant:vagrant /home/vagrant/go
-    echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.profile
+    echo 'export PATH=$PATH:/usr/local/go/bin:/home/vagrant/go/bin' >> /home/vagrant/.bash_profile
+    echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.bash_profile
+
+    echo 'export LC_ALL=en_US.UTF-8' >> /home/vagrant/.bash_profile
 
     mkdir -p /home/vagrant/go/src/github.com/tendermint
     ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint
 
-    su - vagrant -c 'curl https://glide.sh/get | sh'
-    su - vagrant -c 'cd /vagrant/ && glide install && make test'
+    chown -R vagrant:vagrant /home/vagrant/go
+
+    su - vagrant -c 'cd /home/vagrant/go/src/github.com/tendermint/tendermint && make get_vendor_deps'
   SHELL
 end

benchmarks/codec_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/tendermint/go-crypto"
-	"github.com/tendermint/go-p2p"
+	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/go-wire"
 	proto "github.com/tendermint/tendermint/benchmarks/proto"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
@@ -12,10 +12,10 @@ import (
 
 func BenchmarkEncodeStatusWire(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().(crypto.PubKeyEd25519)
+	pubKey := crypto.GenPrivKeyEd25519().PubKey()
 	status := &ctypes.ResultStatus{
 		NodeInfo: &p2p.NodeInfo{
-			PubKey:     pubKey,
+			PubKey:     pubKey.Unwrap().(crypto.PubKeyEd25519),
 			Moniker:    "SOMENAME",
 			Network:    "SOMENAME",
 			RemoteAddr: "SOMEADDR",
@@ -40,7 +40,7 @@ func BenchmarkEncodeStatusWire(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoWire(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().(crypto.PubKeyEd25519)
+	pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519)
 	nodeInfo := &p2p.NodeInfo{
 		PubKey:     pubKey,
 		Moniker:    "SOMENAME",
@@ -61,7 +61,7 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().(crypto.PubKeyEd25519)
+	pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519)
 	nodeInfo := &p2p.NodeInfo{
 		PubKey:     pubKey,
 		Moniker:    "SOMENAME",
@@ -83,7 +83,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) {
 
 func BenchmarkEncodeNodeInfoProto(b *testing.B) {
 	b.StopTimer()
-	pubKey := crypto.GenPrivKeyEd25519().PubKey().(crypto.PubKeyEd25519)
+	pubKey := crypto.GenPrivKeyEd25519().PubKey().Unwrap().(crypto.PubKeyEd25519)
 	pubKey2 := &proto.PubKey{Ed25519: &proto.PubKeyEd25519{Bytes: pubKey[:]}}
 	nodeInfo := &proto.NodeInfo{
 		PubKey:     pubKey2,

benchmarks/map_test.go

@@ -1,7 +1,7 @@
 package benchmarks
 
 import (
-	. "github.com/tendermint/go-common"
+	. "github.com/tendermint/tmlibs/common"
 	"testing"
 )
 

benchmarks/os_test.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"testing"
 
-	. "github.com/tendermint/go-common"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 func BenchmarkFileWrite(b *testing.B) {

benchmarks/simu/counter.go

@@ -7,11 +7,11 @@ import (
 	"fmt"
 
 	"github.com/gorilla/websocket"
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-rpc/client"
-	"github.com/tendermint/go-rpc/types"
 	"github.com/tendermint/go-wire"
 	_ "github.com/tendermint/tendermint/rpc/core/types" // Register RPCResponse > Result types
+	"github.com/tendermint/tendermint/rpc/lib/client"
+	"github.com/tendermint/tendermint/rpc/lib/types"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 func main() {
@@ -37,11 +37,16 @@ func main() {
 	for i := 0; ; i++ {
 		binary.BigEndian.PutUint64(buf, uint64(i))
 		//txBytes := hex.EncodeToString(buf[:n])
-		request := rpctypes.NewRPCRequest("fakeid", "broadcast_tx", Arr(buf[:8]))
+		request, err := rpctypes.MapToRequest("fakeid",
+			"broadcast_tx",
+			map[string]interface{}{"tx": buf[:8]})
+		if err != nil {
+			Exit(err.Error())
+		}
 		reqBytes := wire.JSONBytes(request)
 		//fmt.Println("!!", string(reqBytes))
 		fmt.Print(".")
-		err := ws.WriteMessage(websocket.TextMessage, reqBytes)
+		err = ws.WriteMessage(websocket.TextMessage, reqBytes)
 		if err != nil {
 			Exit(err.Error())
 		}

blockchain/log.go

@@ -1,7 +0,0 @@
-package blockchain
-
-import (
-	"github.com/tendermint/go-logger"
-)
-
-var log = logger.New("module", "blockchain")

blockchain/pool.go

@@ -5,9 +5,10 @@ import (
 	"sync"
 	"time"
 
-	. "github.com/tendermint/go-common"
-	flow "github.com/tendermint/go-flowrate/flowrate"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
+	flow "github.com/tendermint/tmlibs/flowrate"
+	"github.com/tendermint/tmlibs/log"
 )
 
 const (
@@ -26,8 +27,8 @@ var peerTimeoutSeconds = time.Duration(15) // not const so we can override with
 	in sequence from peers that reported higher heights than ours.
 	Every so often we ask peers what height they're on so we can keep going.
 
-	Requests are continuously made for blocks of heigher heights until
-	the limits. If most of the requests have no available peers, and we
+	Requests are continuously made for blocks of higher heights until
+	the limit is reached. If most of the requests have no available peers, and we
 	are not at peer limits, we can probably switch to consensus reactor
 */
 
@@ -58,12 +59,11 @@ func NewBlockPool(start int, requestsCh chan<- BlockRequest, timeoutsCh chan<- s
 		requestsCh: requestsCh,
 		timeoutsCh: timeoutsCh,
 	}
-	bp.BaseService = *NewBaseService(log, "BlockPool", bp)
+	bp.BaseService = *NewBaseService(nil, "BlockPool", bp)
 	return bp
 }
 
 func (pool *BlockPool) OnStart() error {
-	pool.BaseService.OnStart()
 	go pool.makeRequestersRoutine()
 	pool.startTime = time.Now()
 	return nil
@@ -107,7 +107,7 @@ func (pool *BlockPool) removeTimedoutPeers() {
 			// XXX remove curRate != 0
 			if curRate != 0 && curRate < minRecvRate {
 				pool.sendTimeout(peer.id)
-				log.Warn("SendTimeout", "peer", peer.id, "reason", "curRate too low")
+				pool.Logger.Error("SendTimeout", "peer", peer.id, "reason", "curRate too low")
 				peer.didTimeout = true
 			}
 		}
@@ -129,11 +129,9 @@ func (pool *BlockPool) IsCaughtUp() bool {
 	pool.mtx.Lock()
 	defer pool.mtx.Unlock()
 
-	height := pool.height
-
 	// Need at least 1 peer to be considered caught up.
 	if len(pool.peers) == 0 {
-		log.Debug("Blockpool has no peers")
+		pool.Logger.Debug("Blockpool has no peers")
 		return false
 	}
 
@@ -142,8 +140,11 @@ func (pool *BlockPool) IsCaughtUp() bool {
 		maxPeerHeight = MaxInt(maxPeerHeight, peer.height)
 	}
 
-	isCaughtUp := (height > 0 || time.Now().Sub(pool.startTime) > 5*time.Second) && (maxPeerHeight == 0 || height >= maxPeerHeight)
-	log.Notice(Fmt("IsCaughtUp: %v", isCaughtUp), "height", height, "maxPeerHeight", maxPeerHeight)
+	// some conditions to determine if we're caught up
+	receivedBlockOrTimedOut := (pool.height > 0 || time.Since(pool.startTime) > 5*time.Second)
+	ourChainIsLongestAmongPeers := maxPeerHeight == 0 || pool.height >= maxPeerHeight
+	isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers
+	pool.Logger.Info(Fmt("IsCaughtUp: %v", isCaughtUp), "height", pool.height, "maxPeerHeight", maxPeerHeight)
 	return isCaughtUp
 }
 
@@ -227,6 +228,7 @@ func (pool *BlockPool) SetPeerHeight(peerID string, height int) {
 		peer.height = height
 	} else {
 		peer = newBPPeer(pool, peerID, height)
+		peer.setLogger(pool.Logger.With("peer", peerID))
 		pool.peers[peerID] = peer
 	}
 }
@@ -241,7 +243,9 @@ func (pool *BlockPool) RemovePeer(peerID string) {
 func (pool *BlockPool) removePeer(peerID string) {
 	for _, requester := range pool.requesters {
 		if requester.getPeerID() == peerID {
-			pool.numPending++
+			if requester.getBlock() != nil {
+				pool.numPending++
+			}
 			go requester.redo() // pick another peer and ...
 		}
 	}
@@ -258,7 +262,6 @@ func (pool *BlockPool) pickIncrAvailablePeer(minHeight int) *bpPeer {
 		if peer.didTimeout {
 			pool.removePeer(peer.id)
 			continue
-		} else {
 		}
 		if peer.numPending >= maxPendingRequestsPerPeer {
 			continue
@@ -278,6 +281,7 @@ func (pool *BlockPool) makeNextRequester() {
 
 	nextHeight := pool.height + len(pool.requesters)
 	request := newBPRequester(pool, nextHeight)
+	request.SetLogger(pool.Logger.With("height", nextHeight))
 
 	pool.requesters[nextHeight] = request
 	pool.numPending++
@@ -299,6 +303,7 @@ func (pool *BlockPool) sendTimeout(peerID string) {
 	pool.timeoutsCh <- peerID
 }
 
+// unused by tendermint; left for debugging purposes
 func (pool *BlockPool) debug() string {
 	pool.mtx.Lock() // Lock
 	defer pool.mtx.Unlock()
@@ -322,11 +327,12 @@ type bpPeer struct {
 	id          string
 	recvMonitor *flow.Monitor
 
-	mtx        sync.Mutex
 	height     int
 	numPending int32
 	timeout    *time.Timer
 	didTimeout bool
+
+	logger log.Logger
 }
 
 func newBPPeer(pool *BlockPool, peerID string, height int) *bpPeer {
@@ -335,10 +341,15 @@ func newBPPeer(pool *BlockPool, peerID string, height int) *bpPeer {
 		id:         peerID,
 		height:     height,
 		numPending: 0,
+		logger:     log.NewNopLogger(),
 	}
 	return peer
 }
 
+func (peer *bpPeer) setLogger(l log.Logger) {
+	peer.logger = l
+}
+
 func (peer *bpPeer) resetMonitor() {
 	peer.recvMonitor = flow.New(time.Second, time.Second*40)
 	var initialValue = float64(minRecvRate) * math.E
@@ -376,7 +387,7 @@ func (peer *bpPeer) onTimeout() {
 	defer peer.pool.mtx.Unlock()
 
 	peer.pool.sendTimeout(peer.id)
-	log.Warn("SendTimeout", "peer", peer.id, "reason", "onTimeout")
+	peer.logger.Error("SendTimeout", "reason", "onTimeout")
 	peer.didTimeout = true
 }
 
@@ -409,7 +420,6 @@ func newBPRequester(pool *BlockPool, height int) *bpRequester {
 }
 
 func (bpr *bpRequester) OnStart() error {
-	bpr.BaseService.OnStart()
 	go bpr.requestRoutine()
 	return nil
 }

blockchain/pool_test.go

@@ -5,8 +5,9 @@ import (
 	"testing"
 	"time"
 
-	. "github.com/tendermint/go-common"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 )
 
 func init() {
@@ -34,7 +35,9 @@ func TestBasic(t *testing.T) {
 	timeoutsCh := make(chan string, 100)
 	requestsCh := make(chan BlockRequest, 100)
 	pool := NewBlockPool(start, requestsCh, timeoutsCh)
+	pool.SetLogger(log.TestingLogger())
 	pool.Start()
+	defer pool.Stop()
 
 	// Introduce each peer.
 	go func() {
@@ -64,7 +67,7 @@ func TestBasic(t *testing.T) {
 		case peerID := <-timeoutsCh:
 			t.Errorf("timeout: %v", peerID)
 		case request := <-requestsCh:
-			log.Info("TEST: Pulled new BlockRequest", "request", request)
+			t.Logf("Pulled new BlockRequest %v", request)
 			if request.Height == 300 {
 				return // Done!
 			}
@@ -72,12 +75,10 @@ func TestBasic(t *testing.T) {
 			go func() {
 				block := &types.Block{Header: &types.Header{Height: request.Height}}
 				pool.AddBlock(request.PeerID, block, 123)
-				log.Info("TEST: Added block", "block", request.Height, "peer", request.PeerID)
+				t.Logf("Added block from peer %v (height: %v)", request.PeerID, request.Height)
 			}()
 		}
 	}
-
-	pool.Stop()
 }
 
 func TestTimeout(t *testing.T) {
@@ -86,10 +87,12 @@ func TestTimeout(t *testing.T) {
 	timeoutsCh := make(chan string, 100)
 	requestsCh := make(chan BlockRequest, 100)
 	pool := NewBlockPool(start, requestsCh, timeoutsCh)
+	pool.SetLogger(log.TestingLogger())
 	pool.Start()
+	defer pool.Stop()
 
 	for _, peer := range peers {
-		log.Info("Peer", "id", peer.id)
+		t.Logf("Peer %v", peer.id)
 	}
 
 	// Introduce each peer.
@@ -120,7 +123,7 @@ func TestTimeout(t *testing.T) {
 	for {
 		select {
 		case peerID := <-timeoutsCh:
-			log.Info("Timeout", "peerID", peerID)
+			t.Logf("Peer %v timeouted", peerID)
 			if _, ok := timedOut[peerID]; !ok {
 				counter++
 				if counter == len(peers) {
@@ -128,9 +131,7 @@ func TestTimeout(t *testing.T) {
 				}
 			}
 		case request := <-requestsCh:
-			log.Info("TEST: Pulled new BlockRequest", "request", request)
+			t.Logf("Pulled new BlockRequest %+v", request)
 		}
 	}
-
-	pool.Stop()
 }

blockchain/reactor.go

@@ -3,23 +3,22 @@ package blockchain
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"reflect"
 	"time"
 
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/go-wire"
+	wire "github.com/tendermint/go-wire"
+	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
 )
 
 const (
-	BlockchainChannel      = byte(0x40)
+	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
+	BlockchainChannel = byte(0x40)
+
 	defaultChannelCapacity = 100
-	defaultSleepIntervalMS = 500
 	trySyncIntervalMS      = 100
 	// stop syncing when last block's time is
 	// within this much of the system time.
@@ -42,7 +41,6 @@ type consensusReactor interface {
 type BlockchainReactor struct {
 	p2p.BaseReactor
 
-	config       cfg.Config
 	state        *sm.State
 	proxyAppConn proxy.AppConnConsensus // same as consensus.proxyAppConn
 	store        *BlockStore
@@ -50,17 +48,17 @@ type BlockchainReactor struct {
 	fastSync     bool
 	requestsCh   chan BlockRequest
 	timeoutsCh   chan string
-	lastBlock    *types.Block
 
 	evsw types.EventSwitch
 }
 
-func NewBlockchainReactor(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor {
+// NewBlockchainReactor returns new reactor instance.
+func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor {
 	if state.LastBlockHeight == store.Height()-1 {
-		store.height -= 1 // XXX HACK, make this better
+		store.height-- // XXX HACK, make this better
 	}
 	if state.LastBlockHeight != store.Height() {
-		PanicSanity(Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
+		cmn.PanicSanity(cmn.Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
 	}
 	requestsCh := make(chan BlockRequest, defaultChannelCapacity)
 	timeoutsCh := make(chan string, defaultChannelCapacity)
@@ -70,7 +68,6 @@ func NewBlockchainReactor(config cfg.Config, state *sm.State, proxyAppConn proxy
 		timeoutsCh,
 	)
 	bcR := &BlockchainReactor{
-		config:       config,
 		state:        state,
 		proxyAppConn: proxyAppConn,
 		store:        store,
@@ -79,10 +76,11 @@ func NewBlockchainReactor(config cfg.Config, state *sm.State, proxyAppConn proxy
 		requestsCh:   requestsCh,
 		timeoutsCh:   timeoutsCh,
 	}
-	bcR.BaseReactor = *p2p.NewBaseReactor(log, "BlockchainReactor", bcR)
+	bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
 	return bcR
 }
 
+// OnStart implements BaseService
 func (bcR *BlockchainReactor) OnStart() error {
 	bcR.BaseReactor.OnStart()
 	if bcR.fastSync {
@@ -95,12 +93,13 @@ func (bcR *BlockchainReactor) OnStart() error {
 	return nil
 }
 
+// OnStop implements BaseService
 func (bcR *BlockchainReactor) OnStop() {
 	bcR.BaseReactor.OnStop()
 	bcR.pool.Stop()
 }
 
-// Implements Reactor
+// GetChannels implements Reactor
 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	return []*p2p.ChannelDescriptor{
 		&p2p.ChannelDescriptor{
@@ -111,28 +110,29 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	}
 }
 
-// Implements Reactor
+// AddPeer implements Reactor by sending our state to peer.
 func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
-	// Send peer our state.
-	peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
+	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
+		// doing nothing, will try later in `poolRoutine`
+	}
 }
 
-// Implements Reactor
+// RemovePeer implements Reactor by removing peer from the pool.
 func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
-	// Remove peer from the pool.
 	bcR.pool.RemovePeer(peer.Key)
 }
 
-// Implements Reactor
+// Receive implements Reactor by handling 4 types of messages (look below).
 func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
 	_, msg, err := DecodeMessage(msgBytes)
 	if err != nil {
-		log.Warn("Error decoding message", "error", err)
+		bcR.Logger.Error("Error decoding message", "err", err)
 		return
 	}
 
-	log.Debug("Receive", "src", src, "chID", chID, "msg", msg)
+	bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg)
 
+	// TODO: improve logic to satisfy megacheck
 	switch msg := msg.(type) {
 	case *bcBlockRequestMessage:
 		// Got a request for a block. Respond with block if we have it.
@@ -159,7 +159,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 		// Got a peer status. Unverified.
 		bcR.pool.SetPeerHeight(src.Key, msg.Height)
 	default:
-		log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+		bcR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 	}
 }
 
@@ -193,16 +193,16 @@ FOR_LOOP:
 			if peer != nil {
 				bcR.Switch.StopPeerForError(peer, errors.New("BlockchainReactor Timeout"))
 			}
-		case _ = <-statusUpdateTicker.C:
+		case <-statusUpdateTicker.C:
 			// ask for status updates
 			go bcR.BroadcastStatusRequest()
-		case _ = <-switchToConsensusTicker.C:
+		case <-switchToConsensusTicker.C:
 			height, numPending, _ := bcR.pool.GetStatus()
 			outbound, inbound, _ := bcR.Switch.NumPeers()
-			log.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
+			bcR.Logger.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
 				"outbound", outbound, "inbound", inbound)
 			if bcR.pool.IsCaughtUp() {
-				log.Notice("Time to switch to consensus reactor!", "height", height)
+				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
 				bcR.pool.Stop()
 
 				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
@@ -210,18 +210,18 @@ FOR_LOOP:
 
 				break FOR_LOOP
 			}
-		case _ = <-trySyncTicker.C: // chan time
+		case <-trySyncTicker.C: // chan time
 			// This loop can be slow as long as it's doing syncing work.
 		SYNC_LOOP:
 			for i := 0; i < 10; i++ {
 				// See if there are any blocks to sync.
 				first, second := bcR.pool.PeekTwoBlocks()
-				//log.Info("TrySync peeked", "first", first, "second", second)
+				//bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
 				if first == nil || second == nil {
 					// We need both to sync the first block.
 					break SYNC_LOOP
 				}
-				firstParts := first.MakePartSet(bcR.config.GetInt("block_part_size")) // TODO: put part size in parts header?
+				firstParts := first.MakePartSet(types.DefaultBlockPartSize)
 				firstPartsHeader := firstParts.Header()
 				// Finally, verify the first block using the second's commit
 				// NOTE: we can probably make this more efficient, but note that calling
@@ -230,7 +230,7 @@ FOR_LOOP:
 				err := bcR.state.Validators.VerifyCommit(
 					bcR.state.ChainID, types.BlockID{first.Hash(), firstPartsHeader}, first.Height, second.LastCommit)
 				if err != nil {
-					log.Info("error in validation", "error", err)
+					bcR.Logger.Info("error in validation", "err", err)
 					bcR.pool.RedoRequest(first.Height)
 					break SYNC_LOOP
 				} else {
@@ -242,12 +242,11 @@ FOR_LOOP:
 					// NOTE: we could improve performance if we
 					// didn't make the app commit to disk every block
 					// ... but we would need a way to get the hash without it persisting
-					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, sm.MockMempool{})
+					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{})
 					if err != nil {
 						// TODO This is bad, are we zombie?
-						PanicQ(Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+						cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
 					}
-					bcR.state.Save()
 				}
 			}
 			continue FOR_LOOP
@@ -257,17 +256,13 @@ FOR_LOOP:
 	}
 }
 
-func (bcR *BlockchainReactor) BroadcastStatusResponse() error {
-	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
-	return nil
-}
-
+// BroadcastStatusRequest broadcasts `BlockStore` height.
 func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
 	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusRequestMessage{bcR.store.Height()}})
 	return nil
 }
 
-// implements events.Eventable
+// SetEventSwitch implements events.Eventable
 func (bcR *BlockchainReactor) SetEventSwitch(evsw types.EventSwitch) {
 	bcR.evsw = evsw
 }
@@ -282,6 +277,7 @@ const (
 	msgTypeStatusRequest  = byte(0x21)
 )
 
+// BlockchainMessage is a generic message for this reactor.
 type BlockchainMessage interface{}
 
 var _ = wire.RegisterInterface(
@@ -292,6 +288,7 @@ var _ = wire.RegisterInterface(
 	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
 )
 
+// DecodeMessage decodes BlockchainMessage.
 // TODO: ensure that bz is completely read.
 func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
 	msgType = bz[0]
@@ -299,7 +296,7 @@ func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
 	r := bytes.NewReader(bz)
 	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
 	if err != nil && n != len(bz) {
-		err = errors.New("DecodeMessage() had bytes left over.")
+		err = errors.New("DecodeMessage() had bytes left over")
 	}
 	return
 }
@@ -311,7 +308,7 @@ type bcBlockRequestMessage struct {
 }
 
 func (m *bcBlockRequestMessage) String() string {
-	return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height)
+	return cmn.Fmt("[bcBlockRequestMessage %v]", m.Height)
 }
 
 //-------------------------------------
@@ -322,7 +319,7 @@ type bcBlockResponseMessage struct {
 }
 
 func (m *bcBlockResponseMessage) String() string {
-	return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height)
+	return cmn.Fmt("[bcBlockResponseMessage %v]", m.Block.Height)
 }
 
 //-------------------------------------
@@ -332,7 +329,7 @@ type bcStatusRequestMessage struct {
 }
 
 func (m *bcStatusRequestMessage) String() string {
-	return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height)
+	return cmn.Fmt("[bcStatusRequestMessage %v]", m.Height)
 }
 
 //-------------------------------------
@@ -342,5 +339,5 @@ type bcStatusResponseMessage struct {
 }
 
 func (m *bcStatusResponseMessage) String() string {
-	return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height)
+	return cmn.Fmt("[bcStatusResponseMessage %v]", m.Height)
 }

blockchain/store.go

@@ -7,8 +7,8 @@ import (
 	"io"
 	"sync"
 
-	. "github.com/tendermint/go-common"
-	dbm "github.com/tendermint/go-db"
+	. "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/go-wire"
 	"github.com/tendermint/tendermint/types"
 )
@@ -64,12 +64,12 @@ func (bs *BlockStore) LoadBlock(height int) *types.Block {
 	if r == nil {
 		return nil
 	}
-	meta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
+	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
 	if err != nil {
 		PanicCrisis(Fmt("Error reading block meta: %v", err))
 	}
 	bytez := []byte{}
-	for i := 0; i < meta.PartsHeader.Total; i++ {
+	for i := 0; i < blockMeta.BlockID.PartsHeader.Total; i++ {
 		part := bs.LoadBlockPart(height, i)
 		bytez = append(bytez, part.Bytes...)
 	}
@@ -101,11 +101,11 @@ func (bs *BlockStore) LoadBlockMeta(height int) *types.BlockMeta {
 	if r == nil {
 		return nil
 	}
-	meta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
+	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
 	if err != nil {
 		PanicCrisis(Fmt("Error reading block meta: %v", err))
 	}
-	return meta
+	return blockMeta
 }
 
 // The +2/3 and other Precommit-votes for block at `height`.
@@ -154,8 +154,8 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
 	}
 
 	// Save block meta
-	meta := types.NewBlockMeta(block, blockParts)
-	metaBytes := wire.BinaryBytes(meta)
+	blockMeta := types.NewBlockMeta(block, blockParts)
+	metaBytes := wire.BinaryBytes(blockMeta)
 	bs.db.Set(calcBlockMetaKey(height), metaBytes)
 
 	// Save block parts

circle.yml

@@ -1,38 +1,33 @@
+---
 machine:
   environment:
     MACH_PREFIX: tendermint-test-mach
-    GOPATH: /home/ubuntu/.go_workspace
-    REPO: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
     DOCKER_VERSION: 1.10.0
-    DOCKER_MACHINE_VERSION: 0.6.0
+    DOCKER_MACHINE_VERSION: 0.9.0
+    GOPATH: "$HOME/.go_project"
+    PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
+    PROJECT_PATH: "$PROJECT_PARENT_PATH/$CIRCLE_PROJECT_REPONAME"
   hosts:
-    circlehost: 127.0.0.1
     localhost: 127.0.0.1
 
-checkout:
-  post:
-    - rm -rf $REPO
-    - mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
-    - mv $HOME/$CIRCLE_PROJECT_REPONAME $REPO
-    # - git submodule sync
-    # - git submodule update --init # use submodules
-
 dependencies:
   override:
-    - echo $MACH_PREFIX $GOPATH $REPO $DOCKER_VERSION $DOCKER_MACHINE_VERSION
     - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | sudo bash -s -- $DOCKER_VERSION
-    - sudo curl -sSL -o /usr/bin/docker-machine https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-linux-x86_64; sudo chmod 0755 /usr/bin/docker-machine
     - sudo start docker
+    - sudo curl -sSL -o /usr/bin/docker-machine "https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-`uname -s`-`uname -m`"; sudo chmod 0755 /usr/bin/docker-machine
+    - mkdir -p "$PROJECT_PARENT_PATH"
+    - ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH"
+  post:
     - go version
     - docker version
     - docker-machine version
 
 test:
   override:
-    - "cd $REPO && set -o pipefail && make test_integrations | tee ~/test_integrations.log":
+    - cd "$PROJECT_PATH" && set -o pipefail && make test_integrations 2>&1 | tee test_integrations.log:
         timeout: 1800
-    - "cp ~/test_integrations.log $CIRCLE_ARTIFACTS"
   post:
-    - "cd $REPO && bash <(curl -s https://codecov.io/bash)"
-
-
+    - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
+    - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker_logs.txt"

cmd/tendermint/commands/gen_validator.go

@@ -0,0 +1,27 @@
+package commands
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/types"
+)
+
+var genValidatorCmd = &cobra.Command{
+	Use:   "gen_validator",
+	Short: "Generate new validator keypair",
+	Run:   genValidator,
+}
+
+func init() {
+	RootCmd.AddCommand(genValidatorCmd)
+}
+
+func genValidator(cmd *cobra.Command, args []string) {
+	privValidator := types.GenPrivValidator()
+	privValidatorJSONBytes, _ := json.MarshalIndent(privValidator, "", "\t")
+	fmt.Printf(`%v
+`, string(privValidatorJSONBytes))
+}

cmd/tendermint/commands/init.go

@@ -0,0 +1,47 @@
+package commands
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+var initFilesCmd = &cobra.Command{
+	Use:   "init",
+	Short: "Initialize Tendermint",
+	Run:   initFiles,
+}
+
+func init() {
+	RootCmd.AddCommand(initFilesCmd)
+}
+
+func initFiles(cmd *cobra.Command, args []string) {
+	privValFile := config.PrivValidatorFile()
+	if _, err := os.Stat(privValFile); os.IsNotExist(err) {
+		privValidator := types.GenPrivValidator()
+		privValidator.SetFile(privValFile)
+		privValidator.Save()
+
+		genFile := config.GenesisFile()
+
+		if _, err := os.Stat(genFile); os.IsNotExist(err) {
+			genDoc := types.GenesisDoc{
+				ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)),
+			}
+			genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{
+				PubKey: privValidator.PubKey,
+				Amount: 10,
+			}}
+
+			genDoc.SaveAs(genFile)
+		}
+
+		logger.Info("Initialized tendermint", "genesis", config.GenesisFile(), "priv_validator", config.PrivValidatorFile())
+	} else {
+		logger.Info("Already initialized", "priv_validator", config.PrivValidatorFile())
+	}
+}

cmd/tendermint/commands/probe_upnp.go

@@ -0,0 +1,35 @@
+package commands
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/p2p/upnp"
+)
+
+var probeUpnpCmd = &cobra.Command{
+	Use:   "probe_upnp",
+	Short: "Test UPnP functionality",
+	RunE:  probeUpnp,
+}
+
+func init() {
+	RootCmd.AddCommand(probeUpnpCmd)
+}
+
+func probeUpnp(cmd *cobra.Command, args []string) error {
+	capabilities, err := upnp.Probe(logger)
+	if err != nil {
+		fmt.Println("Probe failed: %v", err)
+	} else {
+		fmt.Println("Probe success!")
+		jsonBytes, err := json.Marshal(capabilities)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(jsonBytes))
+	}
+	return nil
+}

cmd/tendermint/commands/replay.go

@@ -0,0 +1,28 @@
+package commands
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/consensus"
+)
+
+var replayCmd = &cobra.Command{
+	Use:   "replay",
+	Short: "Replay messages from WAL",
+	Run: func(cmd *cobra.Command, args []string) {
+		consensus.RunReplayFile(config.BaseConfig, config.Consensus, false)
+	},
+}
+
+var replayConsoleCmd = &cobra.Command{
+	Use:   "replay_console",
+	Short: "Replay messages from WAL in a console",
+	Run: func(cmd *cobra.Command, args []string) {
+		consensus.RunReplayFile(config.BaseConfig, config.Consensus, true)
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(replayCmd)
+	RootCmd.AddCommand(replayConsoleCmd)
+}

cmd/tendermint/commands/reset_priv_validator.go

@@ -0,0 +1,61 @@
+package commands
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tmlibs/log"
+)
+
+var resetAllCmd = &cobra.Command{
+	Use:   "unsafe_reset_all",
+	Short: "(unsafe) Remove all the data and WAL, reset this node's validator",
+	Run:   resetAll,
+}
+
+var resetPrivValidatorCmd = &cobra.Command{
+	Use:   "unsafe_reset_priv_validator",
+	Short: "(unsafe) Reset this node's validator",
+	Run:   resetPrivValidator,
+}
+
+func init() {
+	RootCmd.AddCommand(resetAllCmd)
+	RootCmd.AddCommand(resetPrivValidatorCmd)
+}
+
+// XXX: this is totally unsafe.
+// it's only suitable for testnets.
+func resetAll(cmd *cobra.Command, args []string) {
+	ResetAll(config.DBDir(), config.PrivValidatorFile(), logger)
+}
+
+// XXX: this is totally unsafe.
+// it's only suitable for testnets.
+func resetPrivValidator(cmd *cobra.Command, args []string) {
+	resetPrivValidatorLocal(config.PrivValidatorFile(), logger)
+}
+
+// Exported so other CLI tools can use  it
+func ResetAll(dbDir, privValFile string, logger log.Logger) {
+	resetPrivValidatorLocal(privValFile, logger)
+	os.RemoveAll(dbDir)
+	logger.Info("Removed all data", "dir", dbDir)
+}
+
+func resetPrivValidatorLocal(privValFile string, logger log.Logger) {
+	// Get PrivValidator
+	var privValidator *types.PrivValidator
+	if _, err := os.Stat(privValFile); err == nil {
+		privValidator = types.LoadPrivValidator(privValFile)
+		privValidator.Reset()
+		logger.Info("Reset PrivValidator", "file", privValFile)
+	} else {
+		privValidator = types.GenPrivValidator()
+		privValidator.SetFile(privValFile)
+		privValidator.Save()
+		logger.Info("Generated PrivValidator", "file", privValFile)
+	}
+}

cmd/tendermint/commands/root.go

@@ -0,0 +1,53 @@
+package commands
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tmlibs/cli"
+	tmflags "github.com/tendermint/tmlibs/cli/flags"
+	"github.com/tendermint/tmlibs/log"
+)
+
+var (
+	config = cfg.DefaultConfig()
+	logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main")
+)
+
+func init() {
+	RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level")
+}
+
+// ParseConfig will setup the tendermint configuration properly
+func ParseConfig() (*cfg.Config, error) {
+	conf := cfg.DefaultConfig()
+	err := viper.Unmarshal(conf)
+	if err != nil {
+		return nil, err
+	}
+	conf.SetRoot(conf.RootDir)
+	cfg.EnsureRoot(conf.RootDir)
+	return conf, err
+}
+
+var RootCmd = &cobra.Command{
+	Use:   "tendermint",
+	Short: "Tendermint Core (BFT Consensus) in Go",
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+		config, err = ParseConfig()
+		if err != nil {
+			return err
+		}
+		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel())
+		if err != nil {
+			return err
+		}
+		if viper.GetBool(cli.TraceFlag) {
+			logger = log.NewTracingLogger(logger)
+		}
+		return nil
+	},
+}

cmd/tendermint/commands/root_test.go

@@ -0,0 +1,99 @@
+package commands
+
+import (
+	"os"
+	"strconv"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tmlibs/cli"
+
+	"testing"
+)
+
+var (
+	defaultRoot = os.ExpandEnv("$HOME/.some/test/dir")
+)
+
+const (
+	rootName = "root"
+)
+
+// isolate provides a clean setup and returns a copy of RootCmd you can
+// modify in the test cases
+func isolate(cmds ...*cobra.Command) cli.Executable {
+	viper.Reset()
+	config = cfg.DefaultConfig()
+	r := &cobra.Command{
+		Use:               rootName,
+		PersistentPreRunE: RootCmd.PersistentPreRunE,
+	}
+	r.AddCommand(cmds...)
+	wr := cli.PrepareBaseCmd(r, "TM", defaultRoot)
+	return wr
+}
+
+func TestRootConfig(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// we pre-create a config file we can refer to in the rest of
+	// the test cases.
+	cvals := map[string]string{
+		"moniker":   "monkey",
+		"fast_sync": "false",
+	}
+	// proper types of the above settings
+	cfast := false
+	conf, err := cli.WriteDemoConfig(cvals)
+	require.Nil(err)
+
+	defaults := cfg.DefaultConfig()
+	dmax := defaults.P2P.MaxNumPeers
+
+	cases := []struct {
+		args     []string
+		env      map[string]string
+		root     string
+		moniker  string
+		fastSync bool
+		maxPeer  int
+	}{
+		{nil, nil, defaultRoot, defaults.Moniker, defaults.FastSync, dmax},
+		// try multiple ways of setting root (two flags, cli vs. env)
+		{[]string{"--home", conf}, nil, conf, cvals["moniker"], cfast, dmax},
+		{nil, map[string]string{"TMROOT": conf}, conf, cvals["moniker"], cfast, dmax},
+		// check setting p2p subflags two different ways
+		{[]string{"--p2p.max_num_peers", "420"}, nil, defaultRoot, defaults.Moniker, defaults.FastSync, 420},
+		{nil, map[string]string{"TM_P2P_MAX_NUM_PEERS": "17"}, defaultRoot, defaults.Moniker, defaults.FastSync, 17},
+		// try to set env that have no flags attached...
+		{[]string{"--home", conf}, map[string]string{"TM_MONIKER": "funny"}, conf, "funny", cfast, dmax},
+	}
+
+	for idx, tc := range cases {
+		i := strconv.Itoa(idx)
+		// test command that does nothing, except trigger unmarshalling in root
+		noop := &cobra.Command{
+			Use: "noop",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return nil
+			},
+		}
+		noop.Flags().Int("p2p.max_num_peers", defaults.P2P.MaxNumPeers, "")
+		cmd := isolate(noop)
+
+		args := append([]string{rootName, noop.Use}, tc.args...)
+		err := cli.RunWithArgs(cmd, args, tc.env)
+		require.Nil(err, i)
+		assert.Equal(tc.root, config.RootDir, i)
+		assert.Equal(tc.root, config.P2P.RootDir, i)
+		assert.Equal(tc.root, config.Consensus.RootDir, i)
+		assert.Equal(tc.root, config.Mempool.RootDir, i)
+		assert.Equal(tc.moniker, config.Moniker, i)
+		assert.Equal(tc.fastSync, config.FastSync, i)
+		assert.Equal(tc.maxPeer, config.P2P.MaxNumPeers, i)
+	}
+
+}

cmd/tendermint/commands/run_node.go

@@ -0,0 +1,86 @@
+package commands
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+var runNodeCmd = &cobra.Command{
+	Use:   "node",
+	Short: "Run the tendermint node",
+	RunE:  runNode,
+}
+
+func init() {
+	AddNodeFlags(runNodeCmd)
+	RootCmd.AddCommand(runNodeCmd)
+}
+
+// AddNodeFlags exposes some common configuration options on the command-line
+// These are exposed for convenience of commands embedding a tendermint node
+func AddNodeFlags(cmd *cobra.Command) {
+	// bind flags
+	cmd.Flags().String("moniker", config.Moniker, "Node Name")
+
+	// node flags
+	cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing")
+
+	// abci flags
+	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.")
+	cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)")
+
+	// rpc flags
+	cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required")
+	cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required")
+	cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods")
+
+	// p2p flags
+	cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)")
+	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes")
+	cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration")
+	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)")
+}
+
+// Users wishing to:
+//	* Use an external signer for their validators
+//	* Supply an in-proc abci app
+// should import github.com/tendermint/tendermint/node and implement
+// their own run_node to call node.NewNode (instead of node.NewNodeDefault)
+// with their custom priv validator and/or custom proxy.ClientCreator
+func runNode(cmd *cobra.Command, args []string) error {
+
+	// Wait until the genesis doc becomes available
+	// This is for Mintnet compatibility.
+	// TODO: If Mintnet gets deprecated or genesis_file is
+	// always available, remove.
+	genDocFile := config.GenesisFile()
+	for !cmn.FileExists(genDocFile) {
+		logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile))
+		time.Sleep(time.Second)
+	}
+
+	genDoc, err := types.GenesisDocFromFile(genDocFile)
+	if err != nil {
+		return err
+	}
+	config.ChainID = genDoc.ChainID
+
+	// Create & start node
+	n := node.NewNodeDefault(config, logger.With("module", "node"))
+	if _, err := n.Start(); err != nil {
+		return fmt.Errorf("Failed to start node: %v", err)
+	} else {
+		logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
+	}
+
+	// Trap signal, run forever.
+	n.RunForever()
+
+	return nil
+}

cmd/tendermint/commands/show_validator.go

@@ -0,0 +1,26 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/go-wire/data"
+	"github.com/tendermint/tendermint/types"
+)
+
+var showValidatorCmd = &cobra.Command{
+	Use:   "show_validator",
+	Short: "Show this node's validator info",
+	Run:   showValidator,
+}
+
+func init() {
+	RootCmd.AddCommand(showValidatorCmd)
+}
+
+func showValidator(cmd *cobra.Command, args []string) {
+	privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger)
+	pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey)
+	fmt.Println(string(pubKeyJSONBytes))
+}

cmd/tendermint/commands/testnet.go

@@ -0,0 +1,93 @@
+package commands
+
+import (
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tendermint/types"
+)
+
+var testnetFilesCmd = &cobra.Command{
+	Use:   "testnet",
+	Short: "Initialize files for a Tendermint testnet",
+	Run:   testnetFiles,
+}
+
+//flags
+var (
+	nValidators int
+	dataDir     string
+)
+
+func init() {
+	testnetFilesCmd.Flags().IntVar(&nValidators, "n", 4,
+		"Number of validators to initialize the testnet with")
+	testnetFilesCmd.Flags().StringVar(&dataDir, "dir", "mytestnet",
+		"Directory to store initialization data for the testnet")
+
+	RootCmd.AddCommand(testnetFilesCmd)
+}
+
+func testnetFiles(cmd *cobra.Command, args []string) {
+
+	genVals := make([]types.GenesisValidator, nValidators)
+
+	// Initialize core dir and priv_validator.json's
+	for i := 0; i < nValidators; i++ {
+		mach := cmn.Fmt("mach%d", i)
+		err := initMachCoreDirectory(dataDir, mach)
+		if err != nil {
+			cmn.Exit(err.Error())
+		}
+		// Read priv_validator.json to populate vals
+		privValFile := path.Join(dataDir, mach, "priv_validator.json")
+		privVal := types.LoadPrivValidator(privValFile)
+		genVals[i] = types.GenesisValidator{
+			PubKey: privVal.PubKey,
+			Amount: 1,
+			Name:   mach,
+		}
+	}
+
+	// Generate genesis doc from generated validators
+	genDoc := &types.GenesisDoc{
+		GenesisTime: time.Now(),
+		ChainID:     "chain-" + cmn.RandStr(6),
+		Validators:  genVals,
+	}
+
+	// Write genesis file.
+	for i := 0; i < nValidators; i++ {
+		mach := cmn.Fmt("mach%d", i)
+		genDoc.SaveAs(path.Join(dataDir, mach, "genesis.json"))
+	}
+
+	fmt.Println(cmn.Fmt("Successfully initialized %v node directories", nValidators))
+}
+
+// Initialize per-machine core directory
+func initMachCoreDirectory(base, mach string) error {
+	dir := path.Join(base, mach)
+	err := cmn.EnsureDir(dir, 0777)
+	if err != nil {
+		return err
+	}
+
+	// Create priv_validator.json file if not present
+	ensurePrivValidator(path.Join(dir, "priv_validator.json"))
+	return nil
+
+}
+
+func ensurePrivValidator(file string) {
+	if cmn.FileExists(file) {
+		return
+	}
+	privValidator := types.GenPrivValidator()
+	privValidator.SetFile(file)
+	privValidator.Save()
+}

cmd/tendermint/commands/version.go

@@ -0,0 +1,21 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/version"
+)
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Show version info",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(version.Version)
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(versionCmd)
+}

cmd/tendermint/flags.go

@@ -1,66 +0,0 @@
-package main
-
-import (
-	flag "github.com/spf13/pflag"
-	"os"
-
-	cfg "github.com/tendermint/go-config"
-)
-
-func parseFlags(config cfg.Config, args []string) {
-	var (
-		printHelp     bool
-		moniker       string
-		nodeLaddr     string
-		seeds         string
-		fastSync      bool
-		skipUPNP      bool
-		rpcLaddr      string
-		grpcLaddr     string
-		logLevel      string
-		proxyApp      string
-		abciTransport string
-
-		pex bool
-	)
-
-	// Declare flags
-	var flags = flag.NewFlagSet("main", flag.ExitOnError)
-	flags.BoolVar(&printHelp, "help", false, "Print this help message.")
-
-	// configuration options
-	flags.StringVar(&moniker, "moniker", config.GetString("moniker"), "Node Name")
-	flags.StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), "Node listen address. (0.0.0.0:0 means any interface, any port)")
-	flags.StringVar(&seeds, "seeds", config.GetString("seeds"), "Comma delimited host:port seed nodes")
-	flags.BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing")
-	flags.BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration")
-	flags.StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required")
-	flags.StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required")
-	flags.StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level")
-	flags.StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"),
-		"Proxy app address, or 'nilapp' or 'dummy' for local testing.")
-	flags.StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)")
-
-	// feature flags
-	flags.BoolVar(&pex, "pex", config.GetBool("pex_reactor"), "Enable Peer-Exchange (dev feature)")
-
-	flags.Parse(args)
-	if printHelp {
-		flags.PrintDefaults()
-		os.Exit(0)
-	}
-
-	// Merge parsed flag values onto app.
-	config.Set("moniker", moniker)
-	config.Set("node_laddr", nodeLaddr)
-	config.Set("seeds", seeds)
-	config.Set("fast_sync", fastSync)
-	config.Set("skip_upnp", skipUPNP)
-	config.Set("rpc_laddr", rpcLaddr)
-	config.Set("grpc_laddr", grpcLaddr)
-	config.Set("log_level", logLevel)
-	config.Set("proxy_app", proxyApp)
-	config.Set("abci", abciTransport)
-
-	config.Set("pex_reactor", pex)
-}

cmd/tendermint/gen_validator.go

@@ -1,23 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/tendermint/go-wire"
-	"github.com/tendermint/tendermint/types"
-)
-
-func gen_validator() {
-
-	privValidator := types.GenPrivValidator()
-	privValidatorJSONBytes := wire.JSONBytesPretty(privValidator)
-	fmt.Printf(`Generated a new validator!
-Paste the following JSON into your %v file
-
-%v
-
-`,
-		config.GetString("priv_validator_file"),
-		string(privValidatorJSONBytes),
-	)
-}

cmd/tendermint/init.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/tendermint/types"
-)
-
-func init_files() {
-	privValidator := types.GenPrivValidator()
-	privValidator.SetFile(config.GetString("priv_validator_file"))
-	privValidator.Save()
-
-	genDoc := types.GenesisDoc{
-		ChainID: Fmt("test-chain-%v", RandStr(6)),
-	}
-	genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{
-		PubKey: privValidator.PubKey,
-		Amount: 10,
-	}}
-
-	genDoc.SaveAs(config.GetString("genesis_file"))
-
-	log.Notice("Initialized tendermint", "genesis", config.GetString("genesis_file"), "priv_validator", config.GetString("priv_validator_file"))
-}

cmd/tendermint/log.go

@@ -1,7 +0,0 @@
-package main
-
-import (
-	"github.com/tendermint/go-logger"
-)
-
-var log = logger.New("module", "main")

cmd/tendermint/main.go

@@ -1,66 +1,13 @@
 package main
 
 import (
-	"fmt"
 	"os"
 
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-logger"
-	tmcfg "github.com/tendermint/tendermint/config/tendermint"
-	"github.com/tendermint/tendermint/node"
-	"github.com/tendermint/tendermint/version"
+	"github.com/tendermint/tendermint/cmd/tendermint/commands"
+	"github.com/tendermint/tmlibs/cli"
 )
 
-var config cfg.Config
-
 func main() {
-
-	args := os.Args[1:]
-	if len(args) == 0 {
-		fmt.Println(`Tendermint
-
-Commands:
-    node            Run the tendermint node
-    show_validator  Show this node's validator info
-    gen_validator   Generate new validator keypair
-    probe_upnp      Test UPnP functionality
-    version         Show version info
-`)
-		return
-	}
-
-	// Get configuration
-	config = tmcfg.GetConfig("")
-	parseFlags(config, args[1:]) // Command line overrides
-
-	// set the log level
-	logger.SetLogLevel(config.GetString("log_level"))
-
-	switch args[0] {
-	case "node":
-		node.RunNode(config)
-	case "replay":
-		if len(args) > 2 && args[1] == "console" {
-			node.RunReplayConsole(config, args[2])
-		} else {
-			node.RunReplay(config, args[1])
-		}
-	case "init":
-		init_files()
-	case "show_validator":
-		show_validator()
-	case "gen_validator":
-		gen_validator()
-	case "probe_upnp":
-		probe_upnp()
-	case "unsafe_reset_all":
-		reset_all()
-	case "unsafe_reset_priv_validator":
-		reset_priv_validator()
-	case "version":
-		fmt.Println(version.Version)
-	default:
-		Exit(Fmt("Unknown command %v\n", args[0]))
-	}
+	cmd := cli.PrepareBaseCmd(commands.RootCmd, "TM", os.ExpandEnv("$HOME/.tendermint"))
+	cmd.Execute()
 }

cmd/tendermint/probe_upnp.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/tendermint/go-p2p/upnp"
-)
-
-func probe_upnp() {
-
-	capabilities, err := upnp.Probe()
-	if err != nil {
-		fmt.Println("Probe failed: %v", err)
-	} else {
-		fmt.Println("Probe success!")
-		jsonBytes, err := json.Marshal(capabilities)
-		if err != nil {
-			panic(err)
-		}
-		fmt.Println(string(jsonBytes))
-	}
-
-}

cmd/tendermint/reset_priv_validator.go

@@ -1,33 +0,0 @@
-package main
-
-import (
-	"os"
-
-	"github.com/tendermint/tendermint/types"
-)
-
-// NOTE: this is totally unsafe.
-// it's only suitable for testnets.
-func reset_all() {
-	reset_priv_validator()
-	os.RemoveAll(config.GetString("db_dir"))
-	os.RemoveAll(config.GetString("cs_wal_dir"))
-}
-
-// NOTE: this is totally unsafe.
-// it's only suitable for testnets.
-func reset_priv_validator() {
-	// Get PrivValidator
-	var privValidator *types.PrivValidator
-	privValidatorFile := config.GetString("priv_validator_file")
-	if _, err := os.Stat(privValidatorFile); err == nil {
-		privValidator = types.LoadPrivValidator(privValidatorFile)
-		privValidator.Reset()
-		log.Notice("Reset PrivValidator", "file", privValidatorFile)
-	} else {
-		privValidator = types.GenPrivValidator()
-		privValidator.SetFile(privValidatorFile)
-		privValidator.Save()
-		log.Notice("Generated PrivValidator", "file", privValidatorFile)
-	}
-}

cmd/tendermint/show_validator.go

@@ -1,14 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/tendermint/go-wire"
-	"github.com/tendermint/tendermint/types"
-)
-
-func show_validator() {
-	privValidatorFile := config.GetString("priv_validator_file")
-	privValidator := types.LoadOrGenPrivValidator(privValidatorFile)
-	fmt.Println(string(wire.JSONBytesPretty(privValidator.PubKey)))
-}

config/config.go

@@ -0,0 +1,345 @@
+package config
+
+import (
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/tendermint/tendermint/types"
+)
+
+type Config struct {
+	// Top level options use an anonymous struct
+	BaseConfig `mapstructure:",squash"`
+
+	// Options for services
+	RPC       *RPCConfig       `mapstructure:"rpc"`
+	P2P       *P2PConfig       `mapstructure:"p2p"`
+	Mempool   *MempoolConfig   `mapstructure:"mempool"`
+	Consensus *ConsensusConfig `mapstructure:"consensus"`
+}
+
+func DefaultConfig() *Config {
+	return &Config{
+		BaseConfig: DefaultBaseConfig(),
+		RPC:        DefaultRPCConfig(),
+		P2P:        DefaultP2PConfig(),
+		Mempool:    DefaultMempoolConfig(),
+		Consensus:  DefaultConsensusConfig(),
+	}
+}
+
+func TestConfig() *Config {
+	return &Config{
+		BaseConfig: TestBaseConfig(),
+		RPC:        TestRPCConfig(),
+		P2P:        TestP2PConfig(),
+		Mempool:    DefaultMempoolConfig(),
+		Consensus:  TestConsensusConfig(),
+	}
+}
+
+// Set the RootDir for all Config structs
+func (cfg *Config) SetRoot(root string) *Config {
+	cfg.BaseConfig.RootDir = root
+	cfg.RPC.RootDir = root
+	cfg.P2P.RootDir = root
+	cfg.Mempool.RootDir = root
+	cfg.Consensus.RootDir = root
+	return cfg
+}
+
+//-----------------------------------------------------------------------------
+// BaseConfig
+
+// BaseConfig struct for a Tendermint node
+type BaseConfig struct {
+	// The root directory for all data.
+	// This should be set in viper so it can unmarshal into this struct
+	RootDir string `mapstructure:"home"`
+
+	// The ID of the chain to join (should be signed with every transaction and vote)
+	ChainID string `mapstructure:"chain_id"`
+
+	// A JSON file containing the initial validator set and other meta data
+	Genesis string `mapstructure:"genesis_file"`
+
+	// A JSON file containing the private key to use as a validator in the consensus protocol
+	PrivValidator string `mapstructure:"priv_validator_file"`
+
+	// A custom human readable name for this node
+	Moniker string `mapstructure:"moniker"`
+
+	// TCP or UNIX socket address of the ABCI application,
+	// or the name of an ABCI application compiled in with the Tendermint binary
+	ProxyApp string `mapstructure:"proxy_app"`
+
+	// Mechanism to connect to the ABCI application: socket | grpc
+	ABCI string `mapstructure:"abci"`
+
+	// Output level for logging
+	LogLevel string `mapstructure:"log_level"`
+
+	// TCP or UNIX socket address for the profiling server to listen on
+	ProfListenAddress string `mapstructure:"prof_laddr"`
+
+	// If this node is many blocks behind the tip of the chain, FastSync
+	// allows them to catchup quickly by downloading blocks in parallel
+	// and verifying their commits
+	FastSync bool `mapstructure:"fast_sync"`
+
+	// If true, query the ABCI app on connecting to a new peer
+	// so the app can decide if we should keep the connection or not
+	FilterPeers bool `mapstructure:"filter_peers"` // false
+
+	// What indexer to use for transactions
+	TxIndex string `mapstructure:"tx_index"`
+
+	// Database backend: leveldb | memdb
+	DBBackend string `mapstructure:"db_backend"`
+
+	// Database directory
+	DBPath string `mapstructure:"db_dir"`
+}
+
+func DefaultBaseConfig() BaseConfig {
+	return BaseConfig{
+		Genesis:           "genesis.json",
+		PrivValidator:     "priv_validator.json",
+		Moniker:           "anonymous",
+		ProxyApp:          "tcp://127.0.0.1:46658",
+		ABCI:              "socket",
+		LogLevel:          DefaultPackageLogLevels(),
+		ProfListenAddress: "",
+		FastSync:          true,
+		FilterPeers:       false,
+		TxIndex:           "kv",
+		DBBackend:         "leveldb",
+		DBPath:            "data",
+	}
+}
+
+func TestBaseConfig() BaseConfig {
+	conf := DefaultBaseConfig()
+	conf.ChainID = "tendermint_test"
+	conf.ProxyApp = "dummy"
+	conf.FastSync = false
+	conf.DBBackend = "memdb"
+	return conf
+}
+
+func (b BaseConfig) GenesisFile() string {
+	return rootify(b.Genesis, b.RootDir)
+}
+
+func (b BaseConfig) PrivValidatorFile() string {
+	return rootify(b.PrivValidator, b.RootDir)
+}
+
+func (b BaseConfig) DBDir() string {
+	return rootify(b.DBPath, b.RootDir)
+}
+
+func DefaultLogLevel() string {
+	return "error"
+}
+
+func DefaultPackageLogLevels() string {
+	return fmt.Sprintf("state:info,*:%s", DefaultLogLevel())
+}
+
+//-----------------------------------------------------------------------------
+// RPCConfig
+
+type RPCConfig struct {
+	RootDir string `mapstructure:"home"`
+
+	// TCP or UNIX socket address for the RPC server to listen on
+	ListenAddress string `mapstructure:"laddr"`
+
+	// TCP or UNIX socket address for the gRPC server to listen on
+	// NOTE: This server only supports /broadcast_tx_commit
+	GRPCListenAddress string `mapstructure:"grpc_laddr"`
+
+	// Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
+	Unsafe bool `mapstructure:"unsafe"`
+}
+
+func DefaultRPCConfig() *RPCConfig {
+	return &RPCConfig{
+		ListenAddress:     "tcp://0.0.0.0:46657",
+		GRPCListenAddress: "",
+		Unsafe:            false,
+	}
+}
+
+func TestRPCConfig() *RPCConfig {
+	conf := DefaultRPCConfig()
+	conf.ListenAddress = "tcp://0.0.0.0:36657"
+	conf.GRPCListenAddress = "tcp://0.0.0.0:36658"
+	conf.Unsafe = true
+	return conf
+}
+
+//-----------------------------------------------------------------------------
+// P2PConfig
+
+type P2PConfig struct {
+	RootDir        string `mapstructure:"home"`
+	ListenAddress  string `mapstructure:"laddr"`
+	Seeds          string `mapstructure:"seeds"`
+	SkipUPNP       bool   `mapstructure:"skip_upnp"`
+	AddrBook       string `mapstructure:"addr_book_file"`
+	AddrBookStrict bool   `mapstructure:"addr_book_strict"`
+	PexReactor     bool   `mapstructure:"pex"`
+	MaxNumPeers    int    `mapstructure:"max_num_peers"`
+}
+
+func DefaultP2PConfig() *P2PConfig {
+	return &P2PConfig{
+		ListenAddress:  "tcp://0.0.0.0:46656",
+		AddrBook:       "addrbook.json",
+		AddrBookStrict: true,
+		MaxNumPeers:    50,
+	}
+}
+
+func TestP2PConfig() *P2PConfig {
+	conf := DefaultP2PConfig()
+	conf.ListenAddress = "tcp://0.0.0.0:36656"
+	conf.SkipUPNP = true
+	return conf
+}
+
+func (p *P2PConfig) AddrBookFile() string {
+	return rootify(p.AddrBook, p.RootDir)
+}
+
+//-----------------------------------------------------------------------------
+// MempoolConfig
+
+type MempoolConfig struct {
+	RootDir      string `mapstructure:"home"`
+	Recheck      bool   `mapstructure:"recheck"`
+	RecheckEmpty bool   `mapstructure:"recheck_empty"`
+	Broadcast    bool   `mapstructure:"broadcast"`
+	WalPath      string `mapstructure:"wal_dir"`
+}
+
+func DefaultMempoolConfig() *MempoolConfig {
+	return &MempoolConfig{
+		Recheck:      true,
+		RecheckEmpty: true,
+		Broadcast:    true,
+		WalPath:      "data/mempool.wal",
+	}
+}
+
+func (m *MempoolConfig) WalDir() string {
+	return rootify(m.WalPath, m.RootDir)
+}
+
+//-----------------------------------------------------------------------------
+// ConsensusConfig
+
+// ConsensusConfig holds timeouts and details about the WAL, the block structure,
+// and timeouts in the consensus protocol.
+type ConsensusConfig struct {
+	RootDir  string `mapstructure:"home"`
+	WalPath  string `mapstructure:"wal_file"`
+	WalLight bool   `mapstructure:"wal_light"`
+	walFile  string // overrides WalPath if set
+
+	// All timeouts are in ms
+	TimeoutPropose        int `mapstructure:"timeout_propose"`
+	TimeoutProposeDelta   int `mapstructure:"timeout_propose_delta"`
+	TimeoutPrevote        int `mapstructure:"timeout_prevote"`
+	TimeoutPrevoteDelta   int `mapstructure:"timeout_prevote_delta"`
+	TimeoutPrecommit      int `mapstructure:"timeout_precommit"`
+	TimeoutPrecommitDelta int `mapstructure:"timeout_precommit_delta"`
+	TimeoutCommit         int `mapstructure:"timeout_commit"`
+
+	// Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)
+	SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"`
+
+	// BlockSize
+	MaxBlockSizeTxs   int `mapstructure:"max_block_size_txs"`
+	MaxBlockSizeBytes int `mapstructure:"max_block_size_bytes"`
+
+	// TODO: This probably shouldn't be exposed but it makes it
+	// easy to write tests for the wal/replay
+	BlockPartSize int `mapstructure:"block_part_size"`
+}
+
+// Wait this long for a proposal
+func (cfg *ConsensusConfig) Propose(round int) time.Duration {
+	return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond
+}
+
+// After receiving any +2/3 prevote, wait this long for stragglers
+func (cfg *ConsensusConfig) Prevote(round int) time.Duration {
+	return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond
+}
+
+// After receiving any +2/3 precommits, wait this long for stragglers
+func (cfg *ConsensusConfig) Precommit(round int) time.Duration {
+	return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond
+}
+
+// After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight
+func (cfg *ConsensusConfig) Commit(t time.Time) time.Time {
+	return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond)
+}
+
+func DefaultConsensusConfig() *ConsensusConfig {
+	return &ConsensusConfig{
+		WalPath:               "data/cs.wal/wal",
+		WalLight:              false,
+		TimeoutPropose:        3000,
+		TimeoutProposeDelta:   500,
+		TimeoutPrevote:        1000,
+		TimeoutPrevoteDelta:   500,
+		TimeoutPrecommit:      1000,
+		TimeoutPrecommitDelta: 500,
+		TimeoutCommit:         1000,
+		SkipTimeoutCommit:     false,
+		MaxBlockSizeTxs:       10000,
+		MaxBlockSizeBytes:     1,                          // TODO
+		BlockPartSize:         types.DefaultBlockPartSize, // TODO: we shouldnt be importing types
+	}
+}
+
+func TestConsensusConfig() *ConsensusConfig {
+	config := DefaultConsensusConfig()
+	config.TimeoutPropose = 2000
+	config.TimeoutProposeDelta = 1
+	config.TimeoutPrevote = 10
+	config.TimeoutPrevoteDelta = 1
+	config.TimeoutPrecommit = 10
+	config.TimeoutPrecommitDelta = 1
+	config.TimeoutCommit = 10
+	config.SkipTimeoutCommit = true
+	return config
+}
+
+func (c *ConsensusConfig) WalFile() string {
+	if c.walFile != "" {
+		return c.walFile
+	}
+	return rootify(c.WalPath, c.RootDir)
+}
+
+func (c *ConsensusConfig) SetWalFile(walFile string) {
+	c.walFile = walFile
+}
+
+//-----------------------------------------------------------------------------
+// Utils
+
+// helper function to make config creation independent of root dir
+func rootify(path, root string) string {
+	if filepath.IsAbs(path) {
+		return path
+	}
+	return filepath.Join(root, path)
+}

config/config_test.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	// set up some defaults
+	cfg := DefaultConfig()
+	assert.NotNil(cfg.P2P)
+	assert.NotNil(cfg.Mempool)
+	assert.NotNil(cfg.Consensus)
+
+	// check the root dir stuff...
+	cfg.SetRoot("/foo")
+	cfg.Genesis = "bar"
+	cfg.DBPath = "/opt/data"
+	cfg.Mempool.WalPath = "wal/mem/"
+
+	assert.Equal("/foo/bar", cfg.GenesisFile())
+	assert.Equal("/opt/data", cfg.DBDir())
+	assert.Equal("/foo/wal/mem", cfg.Mempool.WalDir())
+
+}

config/tendermint/config.go

@@ -1,115 +0,0 @@
-package tendermint
-
-import (
-	"os"
-	"path"
-	"strings"
-
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-)
-
-func getTMRoot(rootDir string) string {
-	if rootDir == "" {
-		rootDir = os.Getenv("TMROOT")
-	}
-	if rootDir == "" {
-		rootDir = os.Getenv("HOME") + "/.tendermint"
-	}
-	return rootDir
-}
-
-func initTMRoot(rootDir string) {
-	rootDir = getTMRoot(rootDir)
-	EnsureDir(rootDir, 0700)
-	EnsureDir(rootDir+"/data", 0700)
-
-	configFilePath := path.Join(rootDir, "config.toml")
-
-	// Write default config file if missing.
-	if !FileExists(configFilePath) {
-		// Ask user for moniker
-		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
-		MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
-	}
-}
-
-func GetConfig(rootDir string) cfg.Config {
-	rootDir = getTMRoot(rootDir)
-	initTMRoot(rootDir)
-
-	configFilePath := path.Join(rootDir, "config.toml")
-	mapConfig, err := cfg.ReadMapConfigFromFile(configFilePath)
-	if err != nil {
-		Exit(Fmt("Could not read config: %v", err))
-	}
-
-	// Set defaults or panic
-	if mapConfig.IsSet("chain_id") {
-		Exit("Cannot set 'chain_id' via config.toml")
-	}
-	if mapConfig.IsSet("revision_file") {
-		Exit("Cannot set 'revision_file' via config.toml. It must match what's in the Makefile")
-	}
-	mapConfig.SetRequired("chain_id") // blows up if you try to use it before setting.
-	mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json")
-	mapConfig.SetDefault("proxy_app", "tcp://127.0.0.1:46658")
-	mapConfig.SetDefault("abci", "socket")
-	mapConfig.SetDefault("moniker", "anonymous")
-	mapConfig.SetDefault("node_laddr", "tcp://0.0.0.0:46656")
-	mapConfig.SetDefault("seeds", "")
-	// mapConfig.SetDefault("seeds", "goldenalchemist.chaintest.net:46656")
-	mapConfig.SetDefault("fast_sync", true)
-	mapConfig.SetDefault("skip_upnp", false)
-	mapConfig.SetDefault("addrbook_file", rootDir+"/addrbook.json")
-	mapConfig.SetDefault("addrbook_strict", true) // disable to allow connections locally
-	mapConfig.SetDefault("pex_reactor", false)    // enable for peer exchange
-	mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json")
-	mapConfig.SetDefault("db_backend", "leveldb")
-	mapConfig.SetDefault("db_dir", rootDir+"/data")
-	mapConfig.SetDefault("log_level", "info")
-	mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:46657")
-	mapConfig.SetDefault("grpc_laddr", "")
-	mapConfig.SetDefault("prof_laddr", "")
-	mapConfig.SetDefault("revision_file", rootDir+"/revision")
-	mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal")
-	mapConfig.SetDefault("cs_wal_light", false)
-	mapConfig.SetDefault("filter_peers", false)
-
-	mapConfig.SetDefault("block_size", 10000)      // max number of txs
-	mapConfig.SetDefault("block_part_size", 65536) // part size 64K
-	mapConfig.SetDefault("disable_data_hash", false)
-	mapConfig.SetDefault("timeout_propose", 3000)
-	mapConfig.SetDefault("timeout_propose_delta", 500)
-	mapConfig.SetDefault("timeout_prevote", 1000)
-	mapConfig.SetDefault("timeout_prevote_delta", 500)
-	mapConfig.SetDefault("timeout_precommit", 1000)
-	mapConfig.SetDefault("timeout_precommit_delta", 500)
-	mapConfig.SetDefault("timeout_commit", 1000)
-	// make progress asap (no `timeout_commit`) on full precommit votes
-	mapConfig.SetDefault("skip_timeout_commit", false)
-	mapConfig.SetDefault("mempool_recheck", true)
-	mapConfig.SetDefault("mempool_recheck_empty", true)
-	mapConfig.SetDefault("mempool_broadcast", true)
-	mapConfig.SetDefault("mempool_wal_dir", rootDir+"/data/mempool.wal")
-
-	return mapConfig
-}
-
-var defaultConfigTmpl = `# This is a TOML config file.
-# For more information, see https://github.com/toml-lang/toml
-
-proxy_app = "tcp://127.0.0.1:46658"
-moniker = "__MONIKER__"
-node_laddr = "tcp://0.0.0.0:46656"
-seeds = ""
-fast_sync = true
-db_backend = "leveldb"
-log_level = "notice"
-rpc_laddr = "tcp://0.0.0.0:46657"
-`
-
-func defaultConfig(moniker string) (defaultConfig string) {
-	defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
-	return
-}

config/tendermint/logrotate.config

@@ -1,22 +0,0 @@
-// If you wanted to use logrotate, I suppose this might be the config you want.
-// Instead, I'll just write our own, that way we don't need sudo to install.
-
-$HOME/.tendermint/logs/tendermint.log {
-  missingok
-  notifempty
-  rotate 12
-  daily
-  size 10M
-  compress
-  delaycompress
-}
-
-$HOME/.barak/logs/barak.log {
-  missingok
-  notifempty
-  rotate 12
-  weekly
-  size 10M
-  compress
-  delaycompress
-}

config/tendermint_test/config.go

@@ -1,161 +0,0 @@
-// Import this in all *_test.go files to initialize ~/.tendermint_test.
-
-package tendermint_test
-
-import (
-	"os"
-	"path"
-	"strings"
-
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-logger"
-)
-
-func init() {
-	// Creates ~/.tendermint_test
-	EnsureDir(os.Getenv("HOME")+"/.tendermint_test", 0700)
-}
-
-func initTMRoot(rootDir string) {
-	// Remove ~/.tendermint_test_bak
-	if FileExists(rootDir + "_bak") {
-		err := os.RemoveAll(rootDir + "_bak")
-		if err != nil {
-			PanicSanity(err.Error())
-		}
-	}
-	// Move ~/.tendermint_test to ~/.tendermint_test_bak
-	if FileExists(rootDir) {
-		err := os.Rename(rootDir, rootDir+"_bak")
-		if err != nil {
-			PanicSanity(err.Error())
-		}
-	}
-	// Create new dir
-	EnsureDir(rootDir, 0700)
-	EnsureDir(rootDir+"/data", 0700)
-
-	configFilePath := path.Join(rootDir, "config.toml")
-	genesisFilePath := path.Join(rootDir, "genesis.json")
-	privFilePath := path.Join(rootDir, "priv_validator.json")
-
-	// Write default config file if missing.
-	if !FileExists(configFilePath) {
-		// Ask user for moniker
-		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
-		MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
-	}
-	if !FileExists(genesisFilePath) {
-		MustWriteFile(genesisFilePath, []byte(defaultGenesis), 0644)
-	}
-	// we always overwrite the priv val
-	MustWriteFile(privFilePath, []byte(defaultPrivValidator), 0644)
-}
-
-func ResetConfig(localPath string) cfg.Config {
-	rootDir := os.Getenv("HOME") + "/.tendermint_test/" + localPath
-	initTMRoot(rootDir)
-
-	configFilePath := path.Join(rootDir, "config.toml")
-	mapConfig, err := cfg.ReadMapConfigFromFile(configFilePath)
-	if err != nil {
-		Exit(Fmt("Could not read config: %v", err))
-	}
-
-	// Set defaults or panic
-	if mapConfig.IsSet("chain_id") {
-		Exit("Cannot set 'chain_id' via config.toml")
-	}
-	mapConfig.SetDefault("chain_id", "tendermint_test")
-	mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json")
-	mapConfig.SetDefault("proxy_app", "dummy")
-	mapConfig.SetDefault("abci", "socket")
-	mapConfig.SetDefault("moniker", "anonymous")
-	mapConfig.SetDefault("node_laddr", "tcp://0.0.0.0:36656")
-	mapConfig.SetDefault("fast_sync", false)
-	mapConfig.SetDefault("skip_upnp", true)
-	mapConfig.SetDefault("addrbook_file", rootDir+"/addrbook.json")
-	mapConfig.SetDefault("addrbook_strict", true) // disable to allow connections locally
-	mapConfig.SetDefault("pex_reactor", false)    // enable for peer exchange
-	mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json")
-	mapConfig.SetDefault("db_backend", "memdb")
-	mapConfig.SetDefault("db_dir", rootDir+"/data")
-	mapConfig.SetDefault("log_level", "info")
-	mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:36657")
-	mapConfig.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658")
-	mapConfig.SetDefault("prof_laddr", "")
-	mapConfig.SetDefault("revision_file", rootDir+"/revision")
-	mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal")
-	mapConfig.SetDefault("cs_wal_light", false)
-	mapConfig.SetDefault("filter_peers", false)
-
-	mapConfig.SetDefault("block_size", 10000)
-	mapConfig.SetDefault("block_part_size", 65536) // part size 64K
-	mapConfig.SetDefault("disable_data_hash", false)
-	mapConfig.SetDefault("timeout_propose", 2000)
-	mapConfig.SetDefault("timeout_propose_delta", 1)
-	mapConfig.SetDefault("timeout_prevote", 10)
-	mapConfig.SetDefault("timeout_prevote_delta", 1)
-	mapConfig.SetDefault("timeout_precommit", 10)
-	mapConfig.SetDefault("timeout_precommit_delta", 1)
-	mapConfig.SetDefault("timeout_commit", 10)
-	mapConfig.SetDefault("skip_timeout_commit", true)
-	mapConfig.SetDefault("mempool_recheck", true)
-	mapConfig.SetDefault("mempool_recheck_empty", true)
-	mapConfig.SetDefault("mempool_broadcast", true)
-	mapConfig.SetDefault("mempool_wal_dir", "")
-
-	logger.SetLogLevel(mapConfig.GetString("log_level"))
-
-	return mapConfig
-}
-
-var defaultConfigTmpl = `# This is a TOML config file.
-# For more information, see https://github.com/toml-lang/toml
-
-proxy_app = "dummy"
-moniker = "__MONIKER__"
-node_laddr = "tcp://0.0.0.0:36656"
-seeds = ""
-fast_sync = false
-db_backend = "memdb"
-log_level = "info"
-rpc_laddr = "tcp://0.0.0.0:36657"
-`
-
-func defaultConfig(moniker string) (defaultConfig string) {
-	defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
-	return
-}
-
-var defaultGenesis = `{
-  "genesis_time": "0001-01-01T00:00:00.000Z",
-  "chain_id": "tendermint_test",
-  "validators": [
-    {
-      "pub_key": [
-        1,
-        "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
-      ],
-      "amount": 10,
-      "name": ""
-    }
-  ],
-  "app_hash": ""
-}`
-
-var defaultPrivValidator = `{
-  "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456",
-  "pub_key": [
-    1,
-    "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
-  ],
-  "priv_key": [
-    1,
-    "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
-  ],
-  "last_height": 0,
-  "last_round": 0,
-  "last_step": 0
-}`

config/toml.go

@@ -0,0 +1,142 @@
+package config
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+/****** these are for production settings ***********/
+
+func EnsureRoot(rootDir string) {
+	cmn.EnsureDir(rootDir, 0700)
+	cmn.EnsureDir(rootDir+"/data", 0700)
+
+	configFilePath := path.Join(rootDir, "config.toml")
+
+	// Write default config file if missing.
+	if !cmn.FileExists(configFilePath) {
+		// Ask user for moniker
+		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
+		cmn.MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
+	}
+}
+
+var defaultConfigTmpl = `# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+proxy_app = "tcp://127.0.0.1:46658"
+moniker = "__MONIKER__"
+fast_sync = true
+db_backend = "leveldb"
+log_level = "state:info,*:error"
+
+[rpc]
+laddr = "tcp://0.0.0.0:46657"
+
+[p2p]
+laddr = "tcp://0.0.0.0:46656"
+seeds = ""
+`
+
+func defaultConfig(moniker string) string {
+	return strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
+}
+
+/****** these are for test settings ***********/
+
+func ResetTestRoot(testName string) *Config {
+	rootDir := os.ExpandEnv("$HOME/.tendermint_test")
+	rootDir = filepath.Join(rootDir, testName)
+	// Remove ~/.tendermint_test_bak
+	if cmn.FileExists(rootDir + "_bak") {
+		err := os.RemoveAll(rootDir + "_bak")
+		if err != nil {
+			cmn.PanicSanity(err.Error())
+		}
+	}
+	// Move ~/.tendermint_test to ~/.tendermint_test_bak
+	if cmn.FileExists(rootDir) {
+		err := os.Rename(rootDir, rootDir+"_bak")
+		if err != nil {
+			cmn.PanicSanity(err.Error())
+		}
+	}
+	// Create new dir
+	cmn.EnsureDir(rootDir, 0700)
+	cmn.EnsureDir(rootDir+"/data", 0700)
+
+	configFilePath := path.Join(rootDir, "config.toml")
+	genesisFilePath := path.Join(rootDir, "genesis.json")
+	privFilePath := path.Join(rootDir, "priv_validator.json")
+
+	// Write default config file if missing.
+	if !cmn.FileExists(configFilePath) {
+		// Ask user for moniker
+		cmn.MustWriteFile(configFilePath, []byte(testConfig("anonymous")), 0644)
+	}
+	if !cmn.FileExists(genesisFilePath) {
+		cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644)
+	}
+	// we always overwrite the priv val
+	cmn.MustWriteFile(privFilePath, []byte(testPrivValidator), 0644)
+
+	config := TestConfig().SetRoot(rootDir)
+	return config
+}
+
+var testConfigTmpl = `# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+proxy_app = "dummy"
+moniker = "__MONIKER__"
+fast_sync = false
+db_backend = "memdb"
+log_level = "info"
+
+[rpc]
+laddr = "tcp://0.0.0.0:36657"
+
+[p2p]
+laddr = "tcp://0.0.0.0:36656"
+seeds = ""
+`
+
+func testConfig(moniker string) (testConfig string) {
+	testConfig = strings.Replace(testConfigTmpl, "__MONIKER__", moniker, -1)
+	return
+}
+
+var testGenesis = `{
+  "genesis_time": "0001-01-01T00:00:00.000Z",
+  "chain_id": "tendermint_test",
+  "validators": [
+    {
+      "pub_key": {
+        "type": "ed25519",
+        "data":"3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
+      },
+      "amount": 10,
+      "name": ""
+    }
+  ],
+  "app_hash": ""
+}`
+
+var testPrivValidator = `{
+  "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456",
+  "pub_key": {
+    "type": "ed25519",
+    "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
+  },
+  "priv_key": {
+    "type": "ed25519",
+    "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
+  },
+  "last_height": 0,
+  "last_round": 0,
+  "last_step": 0
+}`

config/toml_test.go

@@ -0,0 +1,57 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func ensureFiles(t *testing.T, rootDir string, files ...string) {
+	for _, f := range files {
+		p := rootify(rootDir, f)
+		_, err := os.Stat(p)
+		assert.Nil(t, err, p)
+	}
+}
+
+func TestEnsureRoot(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// setup temp dir for test
+	tmpDir, err := ioutil.TempDir("", "config-test")
+	require.Nil(err)
+	defer os.RemoveAll(tmpDir)
+
+	// create root dir
+	EnsureRoot(tmpDir)
+
+	// make sure config is set properly
+	data, err := ioutil.ReadFile(filepath.Join(tmpDir, "config.toml"))
+	require.Nil(err)
+	assert.Equal([]byte(defaultConfig("anonymous")), data)
+
+	ensureFiles(t, tmpDir, "data")
+}
+
+func TestEnsureTestRoot(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	testName := "ensureTestRoot"
+
+	// create root dir
+	cfg := ResetTestRoot(testName)
+	rootDir := cfg.RootDir
+
+	// make sure config is set properly
+	data, err := ioutil.ReadFile(filepath.Join(rootDir, "config.toml"))
+	require.Nil(err)
+	assert.Equal([]byte(testConfig("anonymous")), data)
+
+	// TODO: make sure the cfg returned and testconfig are the same!
+
+	ensureFiles(t, rootDir, "data", "genesis.json", "priv_validator.json")
+}

consensus/byzantine_test.go

@@ -5,17 +5,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/config/tendermint_test"
-
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-events"
-	"github.com/tendermint/go-p2p"
+	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/events"
 )
 
 func init() {
-	config = tendermint_test.ResetConfig("consensus_byzantine_test")
+	config = ResetConfig("consensus_byzantine_test")
 }
 
 //----------------------------------------------
@@ -29,14 +26,17 @@ func init() {
 // Heal partition and ensure A sees the commit
 func TestByzantine(t *testing.T) {
 	N := 4
+	logger := consensusLogger()
 	css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter)
 
 	// give the byzantine validator a normal ticker
 	css[0].SetTimeoutTicker(NewTimeoutTicker())
 
 	switches := make([]*p2p.Switch, N)
+	p2pLogger := logger.With("module", "p2p")
 	for i := 0; i < N; i++ {
-		switches[i] = p2p.NewSwitch(cfg.NewMapConfig(nil))
+		switches[i] = p2p.NewSwitch(config.P2P)
+		switches[i].SetLogger(p2pLogger.With("validator", i))
 	}
 
 	reactors := make([]p2p.Reactor, N)
@@ -50,19 +50,21 @@ func TestByzantine(t *testing.T) {
 		}
 	}()
 	eventChans := make([]chan interface{}, N)
+	eventLogger := logger.With("module", "events")
 	for i := 0; i < N; i++ {
 		if i == 0 {
 			css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator.(*types.PrivValidator))
 			// make byzantine
 			css[i].decideProposal = func(j int) func(int, int) {
 				return func(height, round int) {
-					byzantineDecideProposalFunc(height, round, css[j], switches[j])
+					byzantineDecideProposalFunc(t, height, round, css[j], switches[j])
 				}
 			}(i)
 			css[i].doPrevote = func(height, round int) {}
 		}
 
 		eventSwitch := events.NewEventSwitch()
+		eventSwitch.SetLogger(eventLogger.With("validator", i))
 		_, err := eventSwitch.Start()
 		if err != nil {
 			t.Fatalf("Failed to start switch: %v", err)
@@ -70,17 +72,19 @@ func TestByzantine(t *testing.T) {
 		eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1)
 
 		conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states
+		conR.SetLogger(logger.With("validator", i))
 		conR.SetEventSwitch(eventSwitch)
 
 		var conRI p2p.Reactor
 		conRI = conR
+
 		if i == 0 {
 			conRI = NewByzantineReactor(conR)
 		}
 		reactors[i] = conRI
 	}
 
-	p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch {
+	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
 		// ignore new switch s, we already made ours
 		switches[i].AddReactor("CONSENSUS", reactors[i])
 		return switches[i]
@@ -113,12 +117,9 @@ func TestByzantine(t *testing.T) {
 	p2p.Connect2Switches(switches, ind1, ind2)
 
 	// wait for someone in the big partition to make a block
+	<-eventChans[ind2]
 
-	select {
-	case <-eventChans[ind2]:
-	}
-
-	log.Notice("A block has been committed. Healing partition")
+	t.Log("A block has been committed. Healing partition")
 
 	// connect the partitions
 	p2p.Connect2Switches(switches, ind0, ind1)
@@ -156,7 +157,7 @@ func TestByzantine(t *testing.T) {
 //-------------------------------
 // byzantine consensus functions
 
-func byzantineDecideProposalFunc(height, round int, cs *ConsensusState, sw *p2p.Switch) {
+func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusState, sw *p2p.Switch) {
 	// byzantine user should create two proposals and try to split the vote.
 	// Avoid sending on internalMsgQueue and running consensus state.
 
@@ -177,7 +178,7 @@ func byzantineDecideProposalFunc(height, round int, cs *ConsensusState, sw *p2p.
 
 	// broadcast conflicting proposals/block parts to peers
 	peers := sw.Peers().List()
-	log.Notice("Byzantine: broadcasting conflicting proposals", "peers", len(peers))
+	t.Logf("Byzantine: broadcasting conflicting proposals to %d peers", len(peers))
 	for i, peer := range peers {
 		if i < len(peers)/2 {
 			go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1)
@@ -242,7 +243,7 @@ func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) {
 	// Send our state to peer.
 	// If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus().
 	if !br.reactor.fastSync {
-		br.reactor.sendNewRoundStepMessage(peer)
+		br.reactor.sendNewRoundStepMessages(peer)
 	}
 }
 func (br *ByzantineReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {

consensus/common.go

@@ -27,3 +27,9 @@ func subscribeToEventRespond(evsw types.EventSwitch, receiver, eventID string) c
 	})
 	return ch
 }
+
+func discardFromChan(ch chan interface{}, n int) {
+	for i := 0; i < n; i++ {
+		<-ch
+	}
+}

consensus/common_test.go

@@ -11,23 +11,26 @@ import (
 	"testing"
 	"time"
 
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	dbm "github.com/tendermint/go-db"
-	"github.com/tendermint/go-p2p"
-	bc "github.com/tendermint/tendermint/blockchain"
-	"github.com/tendermint/tendermint/config/tendermint_test"
-	mempl "github.com/tendermint/tendermint/mempool"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
 	abcicli "github.com/tendermint/abci/client"
 	abci "github.com/tendermint/abci/types"
+	bc "github.com/tendermint/tendermint/blockchain"
+	cfg "github.com/tendermint/tendermint/config"
+	mempl "github.com/tendermint/tendermint/mempool"
+	"github.com/tendermint/tendermint/p2p"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
 
 	"github.com/tendermint/abci/example/counter"
 	"github.com/tendermint/abci/example/dummy"
+
+	"github.com/go-kit/kit/log/term"
 )
 
-var config cfg.Config // NOTE: must be reset for each _test.go file
+// genesis, chain_id, priv_val
+var config *cfg.Config // NOTE: must be reset for each _test.go file
 var ensureTimeout = time.Duration(2)
 
 func ensureDir(dir string, mode os.FileMode) {
@@ -36,6 +39,10 @@ func ensureDir(dir string, mode os.FileMode) {
 	}
 }
 
+func ResetConfig(name string) *cfg.Config {
+	return cfg.ResetTestRoot(name)
+}
+
 //-------------------------------------------------------------------------------
 // validator stub (a dummy consensus peer we control)
 
@@ -64,7 +71,7 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS
 		Type:             voteType,
 		BlockID:          types.BlockID{hash, header},
 	}
-	err := vs.PrivValidator.SignVote(config.GetString("chain_id"), vote)
+	err := vs.PrivValidator.SignVote(config.ChainID, vote)
 	return vote, err
 }
 
@@ -115,7 +122,7 @@ func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) (
 	// Make proposal
 	polRound, polBlockID := cs1.Votes.POLInfo()
 	proposal = types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID)
-	if err := vs.SignProposal(config.GetString("chain_id"), proposal); err != nil {
+	if err := vs.SignProposal(config.ChainID, proposal); err != nil {
 		panic(err)
 	}
 	return
@@ -205,7 +212,7 @@ func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} {
 	go func() {
 		for {
 			v := <-voteCh0
-			vote := v.(types.EventDataVote)
+			vote := v.(types.TMEventData).Unwrap().(types.EventDataVote)
 			// we only fire for our own votes
 			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
 				voteCh <- v
@@ -215,17 +222,6 @@ func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} {
 	return voteCh
 }
 
-func readVotes(ch chan interface{}, reads int) chan struct{} {
-	wg := make(chan struct{})
-	go func() {
-		for i := 0; i < reads; i++ {
-			<-ch // read the precommit event
-		}
-		close(wg)
-	}()
-	return wg
-}
-
 //-------------------------------------------------------------------------------
 // consensus states
 
@@ -233,7 +229,7 @@ func newConsensusState(state *sm.State, pv *types.PrivValidator, app abci.Applic
 	return newConsensusStateWithConfig(config, state, pv, app)
 }
 
-func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState {
+func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState {
 	// Get BlockStore
 	blockDB := dbm.NewMemDB()
 	blockStore := bc.NewBlockStore(blockDB)
@@ -244,39 +240,36 @@ func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *typ
 	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
 
 	// Make Mempool
-	mempool := mempl.NewMempool(thisConfig, proxyAppConnMem)
+	mempool := mempl.NewMempool(thisConfig.Mempool, proxyAppConnMem)
+	mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
 
 	// Make ConsensusReactor
-	cs := NewConsensusState(thisConfig, state, proxyAppConnCon, blockStore, mempool)
+	cs := NewConsensusState(thisConfig.Consensus, state, proxyAppConnCon, blockStore, mempool)
+	cs.SetLogger(log.TestingLogger())
 	cs.SetPrivValidator(pv)
 
 	evsw := types.NewEventSwitch()
+	evsw.SetLogger(log.TestingLogger().With("module", "events"))
 	cs.SetEventSwitch(evsw)
 	evsw.Start()
 	return cs
 }
 
-func loadPrivValidator(conf cfg.Config) *types.PrivValidator {
-	privValidatorFile := conf.GetString("priv_validator_file")
+func loadPrivValidator(config *cfg.Config) *types.PrivValidator {
+	privValidatorFile := config.PrivValidatorFile()
 	ensureDir(path.Dir(privValidatorFile), 0700)
-	privValidator := types.LoadOrGenPrivValidator(privValidatorFile)
+	privValidator := types.LoadOrGenPrivValidator(privValidatorFile, log.TestingLogger())
 	privValidator.Reset()
 	return privValidator
 }
 
-func fixedConsensusState() *ConsensusState {
-	stateDB := dbm.NewMemDB()
-	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
-	privValidator := loadPrivValidator(config)
-	cs := newConsensusState(state, privValidator, counter.NewCounterApplication(true))
-	return cs
-}
-
 func fixedConsensusStateDummy() *ConsensusState {
 	stateDB := dbm.NewMemDB()
-	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
+	state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
+	state.SetLogger(log.TestingLogger().With("module", "state"))
 	privValidator := loadPrivValidator(config)
 	cs := newConsensusState(state, privValidator, dummy.NewDummyApplication())
+	cs.SetLogger(log.TestingLogger())
 	return cs
 }
 
@@ -287,6 +280,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 	vss := make([]*validatorStub, nValidators)
 
 	cs := newConsensusState(state, privVals[0], counter.NewCounterApplication(true))
+	cs.SetLogger(log.TestingLogger())
 
 	for i := 0; i < nValidators; i++ {
 		vss[i] = NewValidatorStub(privVals[i], i)
@@ -312,16 +306,32 @@ func ensureNoNewStep(stepCh chan interface{}) {
 //-------------------------------------------------------------------------------
 // consensus nets
 
+// consensusLogger is a TestingLogger which uses a different
+// color for each validator ("validator" key must exist).
+func consensusLogger() log.Logger {
+	return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor {
+		for i := 0; i < len(keyvals)-1; i += 2 {
+			if keyvals[i] == "validator" {
+				return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))}
+			}
+		}
+		return term.FgBgColor{}
+	})
+}
+
 func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application) []*ConsensusState {
 	genDoc, privVals := randGenesisDoc(nValidators, false, 10)
 	css := make([]*ConsensusState, nValidators)
+	logger := consensusLogger()
 	for i := 0; i < nValidators; i++ {
 		db := dbm.NewMemDB() // each state needs its own db
 		state := sm.MakeGenesisState(db, genDoc)
+		state.SetLogger(logger.With("module", "state", "validator", i))
 		state.Save()
-		thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
-		ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal
+		thisConfig := ResetConfig(Fmt("%s_%d", testName, i))
+		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
 		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc())
+		css[i].SetLogger(logger.With("validator", i))
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
 	return css
@@ -334,9 +344,10 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 	for i := 0; i < nPeers; i++ {
 		db := dbm.NewMemDB() // each state needs its own db
 		state := sm.MakeGenesisState(db, genDoc)
+		state.SetLogger(log.TestingLogger().With("module", "state"))
 		state.Save()
-		thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
-		ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal
+		thisConfig := ResetConfig(Fmt("%s_%d", testName, i))
+		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
 		var privVal *types.PrivValidator
 		if i < nValidators {
 			privVal = privVals[i]
@@ -347,6 +358,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 		}
 
 		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc())
+		css[i].SetLogger(log.TestingLogger())
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
 	return css
@@ -379,7 +391,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G
 	sort.Sort(types.PrivValidatorsByAddress(privValidators))
 	return &types.GenesisDoc{
 		GenesisTime: time.Now(),
-		ChainID:     config.GetString("chain_id"),
+		ChainID:     config.ChainID,
 		Validators:  validators,
 	}, privValidators
 }
@@ -388,6 +400,7 @@ func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.St
 	genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower)
 	db := dbm.NewMemDB()
 	s0 := sm.MakeGenesisState(db, genDoc)
+	s0.SetLogger(log.TestingLogger().With("module", "state"))
 	s0.Save()
 	return s0, privValidators
 }
@@ -438,6 +451,9 @@ func (m *mockTicker) Chan() <-chan timeoutInfo {
 	return m.c
 }
 
+func (mockTicker) SetLogger(log.Logger) {
+}
+
 //------------------------------------
 
 func newCounter() abci.Application {

consensus/height_vote_set.go

@@ -4,8 +4,8 @@ import (
 	"strings"
 	"sync"
 
-	. "github.com/tendermint/go-common"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 type RoundVoteSet struct {
@@ -91,7 +91,7 @@ func (hvs *HeightVoteSet) addRound(round int) {
 	if _, ok := hvs.roundVoteSets[round]; ok {
 		PanicSanity("addRound() for an existing round")
 	}
-	log.Debug("addRound(round)", "round", round)
+	// log.Debug("addRound(round)", "round", round)
 	prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, types.VoteTypePrevote, hvs.valSet)
 	precommits := types.NewVoteSet(hvs.chainID, hvs.height, round, types.VoteTypePrecommit, hvs.valSet)
 	hvs.roundVoteSets[round] = RoundVoteSet{
@@ -118,7 +118,7 @@ func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool,
 			// Peer has sent a vote that does not match our round,
 			// for more than one round.  Bad peer!
 			// TODO punish peer.
-			log.Warn("Deal with peer giving votes from unwanted rounds")
+			// log.Warn("Deal with peer giving votes from unwanted rounds")
 			return
 		}
 	}

consensus/height_vote_set_test.go

@@ -3,19 +3,18 @@ package consensus
 import (
 	"testing"
 
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/tendermint/config/tendermint_test"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 func init() {
-	config = tendermint_test.ResetConfig("consensus_height_vote_set_test")
+	config = ResetConfig("consensus_height_vote_set_test")
 }
 
 func TestPeerCatchupRounds(t *testing.T) {
 	valSet, privVals := types.RandValidatorSet(10, 1)
 
-	hvs := NewHeightVoteSet(config.GetString("chain_id"), 1, valSet)
+	hvs := NewHeightVoteSet(config.ChainID, 1, valSet)
 
 	vote999_0 := makeVoteHR(t, 1, 999, privVals, 0)
 	added, err := hvs.AddVote(vote999_0, "peer1")
@@ -31,6 +30,9 @@ func TestPeerCatchupRounds(t *testing.T) {
 
 	vote1001_0 := makeVoteHR(t, 1, 1001, privVals, 0)
 	added, err = hvs.AddVote(vote1001_0, "peer1")
+	if err != nil {
+		t.Error("AddVote error", err)
+	}
 	if added {
 		t.Error("Expected to *not* add vote from peer, too many catchup rounds.")
 	}
@@ -52,7 +54,7 @@ func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidator
 		Type:             types.VoteTypePrecommit,
 		BlockID:          types.BlockID{[]byte("fakehash"), types.PartSetHeader{}},
 	}
-	chainID := config.GetString("chain_id")
+	chainID := config.ChainID
 	err := privVal.SignVote(chainID, vote)
 	if err != nil {
 		panic(Fmt("Error signing vote: %v", err))

consensus/log.go

@@ -1,18 +0,0 @@
-package consensus
-
-import (
-	"github.com/tendermint/go-logger"
-)
-
-var log = logger.New("module", "consensus")
-
-/*
-func init() {
-	log.SetHandler(
-		logger.LvlFilterHandler(
-			logger.LvlDebug,
-			logger.BypassHandler(),
-		),
-	)
-}
-*/

consensus/mempool_test.go

@@ -5,15 +5,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/config/tendermint_test"
-	"github.com/tendermint/tendermint/types"
 	abci "github.com/tendermint/abci/types"
+	"github.com/tendermint/tendermint/types"
 
-	. "github.com/tendermint/go-common"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 func init() {
-	config = tendermint_test.ResetConfig("consensus_mempool_test")
+	config = ResetConfig("consensus_mempool_test")
 }
 
 func TestTxConcurrentWithCommit(t *testing.T) {
@@ -44,7 +43,7 @@ func TestTxConcurrentWithCommit(t *testing.T) {
 	for nTxs := 0; nTxs < NTxs; {
 		select {
 		case b := <-newBlockCh:
-			nTxs += b.(types.EventDataNewBlock).Block.Header.NumTxs
+			nTxs += b.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block.Header.NumTxs
 		case <-ticker.C:
 			panic("Timed out waiting to commit blocks with transactions")
 		}
@@ -81,15 +80,12 @@ func TestRmBadTx(t *testing.T) {
 		// check for the tx
 		for {
 			time.Sleep(time.Second)
-			select {
-			case <-ch:
-			default:
-				txs := cs.mempool.Reap(1)
-				if len(txs) == 0 {
-					ch <- struct{}{}
-				}
-
+			txs := cs.mempool.Reap(1)
+			if len(txs) == 0 {
+				ch <- struct{}{}
+				return
 			}
+
 		}
 	}()
 
@@ -114,6 +110,8 @@ func TestRmBadTx(t *testing.T) {
 
 // CounterApplication that maintains a mempool state and resets it upon commit
 type CounterApplication struct {
+	abci.BaseApplication
+
 	txCount        int
 	mempoolTxCount int
 }
@@ -126,10 +124,6 @@ func (app *CounterApplication) Info() abci.ResponseInfo {
 	return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)}
 }
 
-func (app *CounterApplication) SetOption(key string, value string) (log string) {
-	return ""
-}
-
 func (app *CounterApplication) DeliverTx(tx []byte) abci.Result {
 	return runTx(tx, &app.txCount)
 }
@@ -160,7 +154,3 @@ func (app *CounterApplication) Commit() abci.Result {
 		return abci.NewResultOK(hash, "")
 	}
 }
-
-func (app *CounterApplication) Query(query []byte) abci.Result {
-	return abci.NewResultOK(nil, Fmt("Query is not supported"))
-}

consensus/reactor.go

@@ -8,11 +8,11 @@ import (
 	"sync"
 	"time"
 
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/go-wire"
+	wire "github.com/tendermint/go-wire"
+	"github.com/tendermint/tendermint/p2p"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 const (
@@ -41,12 +41,12 @@ func NewConsensusReactor(consensusState *ConsensusState, fastSync bool) *Consens
 		conS:     consensusState,
 		fastSync: fastSync,
 	}
-	conR.BaseReactor = *p2p.NewBaseReactor(log, "ConsensusReactor", conR)
+	conR.BaseReactor = *p2p.NewBaseReactor("ConsensusReactor", conR)
 	return conR
 }
 
 func (conR *ConsensusReactor) OnStart() error {
-	log.Notice("ConsensusReactor ", "fastSync", conR.fastSync)
+	conR.Logger.Info("ConsensusReactor ", "fastSync", conR.fastSync)
 	conR.BaseReactor.OnStart()
 
 	// callbacks for broadcasting new steps and votes to peers
@@ -70,7 +70,7 @@ func (conR *ConsensusReactor) OnStop() {
 // Switch from the fast_sync to the consensus:
 // reset the state, turn off fast_sync, start the consensus-state-machine
 func (conR *ConsensusReactor) SwitchToConsensus(state *sm.State) {
-	log.Notice("SwitchToConsensus")
+	conR.Logger.Info("SwitchToConsensus")
 	conR.conS.reconstructLastCommit(state)
 	// NOTE: The line below causes broadcastNewRoundStepRoutine() to
 	// broadcast a NewRoundStepMessage.
@@ -127,7 +127,7 @@ func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) {
 	// Send our state to peer.
 	// If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus().
 	if !conR.fastSync {
-		conR.sendNewRoundStepMessage(peer)
+		conR.sendNewRoundStepMessages(peer)
 	}
 }
 
@@ -148,17 +148,17 @@ func (conR *ConsensusReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
 // NOTE: blocks on consensus state for proposals, block parts, and votes
 func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
 	if !conR.IsRunning() {
-		log.Debug("Receive", "src", src, "chId", chID, "bytes", msgBytes)
+		conR.Logger.Debug("Receive", "src", src, "chId", chID, "bytes", msgBytes)
 		return
 	}
 
 	_, msg, err := DecodeMessage(msgBytes)
 	if err != nil {
-		log.Warn("Error decoding message", "src", src, "chId", chID, "msg", msg, "error", err, "bytes", msgBytes)
+		conR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
 		// TODO punish peer?
 		return
 	}
-	log.Debug("Receive", "src", src, "chId", chID, "msg", msg)
+	conR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg)
 
 	// Get peer states
 	ps := src.Data.Get(types.PeerStateKey).(*PeerState)
@@ -191,7 +191,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 			case types.VoteTypePrecommit:
 				ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID)
 			default:
-				log.Warn("Bad VoteSetBitsMessage field Type")
+				conR.Logger.Error("Bad VoteSetBitsMessage field Type")
 				return
 			}
 			src.TrySend(VoteSetBitsChannel, struct{ ConsensusMessage }{&VoteSetBitsMessage{
@@ -201,14 +201,13 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 				BlockID: msg.BlockID,
 				Votes:   ourVotes,
 			}})
-
 		default:
-			log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+			conR.Logger.Error(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 		}
 
 	case DataChannel:
 		if conR.fastSync {
-			log.Warn("Ignoring message received during fastSync", "msg", msg)
+			conR.Logger.Info("Ignoring message received during fastSync", "msg", msg)
 			return
 		}
 		switch msg := msg.(type) {
@@ -221,12 +220,12 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 			ps.SetHasProposalBlockPart(msg.Height, msg.Round, msg.Part.Index)
 			conR.conS.peerMsgQueue <- msgInfo{msg, src.Key}
 		default:
-			log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+			conR.Logger.Error(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 		}
 
 	case VoteChannel:
 		if conR.fastSync {
-			log.Warn("Ignoring message received during fastSync", "msg", msg)
+			conR.Logger.Info("Ignoring message received during fastSync", "msg", msg)
 			return
 		}
 		switch msg := msg.(type) {
@@ -243,12 +242,12 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 
 		default:
 			// don't punish (leave room for soft upgrades)
-			log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+			conR.Logger.Error(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 		}
 
 	case VoteSetBitsChannel:
 		if conR.fastSync {
-			log.Warn("Ignoring message received during fastSync", "msg", msg)
+			conR.Logger.Info("Ignoring message received during fastSync", "msg", msg)
 			return
 		}
 		switch msg := msg.(type) {
@@ -266,7 +265,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 				case types.VoteTypePrecommit:
 					ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID)
 				default:
-					log.Warn("Bad VoteSetBitsMessage field Type")
+					conR.Logger.Error("Bad VoteSetBitsMessage field Type")
 					return
 				}
 				ps.ApplyVoteSetBitsMessage(msg, ourVotes)
@@ -275,15 +274,15 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 			}
 		default:
 			// don't punish (leave room for soft upgrades)
-			log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+			conR.Logger.Error(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 		}
 
 	default:
-		log.Warn(Fmt("Unknown chId %X", chID))
+		conR.Logger.Error(Fmt("Unknown chId %X", chID))
 	}
 
 	if err != nil {
-		log.Warn("Error in Receive()", "error", err)
+		conR.Logger.Error("Error in Receive()", "err", err)
 	}
 }
 
@@ -300,12 +299,12 @@ func (conR *ConsensusReactor) SetEventSwitch(evsw types.EventSwitch) {
 func (conR *ConsensusReactor) registerEventCallbacks() {
 
 	types.AddListenerForEvent(conR.evsw, "conR", types.EventStringNewRoundStep(), func(data types.TMEventData) {
-		rs := data.(types.EventDataRoundState).RoundState.(*RoundState)
+		rs := data.Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 		conR.broadcastNewRoundStep(rs)
 	})
 
 	types.AddListenerForEvent(conR.evsw, "conR", types.EventStringVote(), func(data types.TMEventData) {
-		edv := data.(types.EventDataVote)
+		edv := data.Unwrap().(types.EventDataVote)
 		conR.broadcastHasVoteMessage(edv.Vote)
 	})
 }
@@ -352,7 +351,7 @@ func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg *
 		Height: rs.Height,
 		Round:  rs.Round,
 		Step:   rs.Step,
-		SecondsSinceStartTime: int(time.Now().Sub(rs.StartTime).Seconds()),
+		SecondsSinceStartTime: int(time.Since(rs.StartTime).Seconds()),
 		LastCommitRound:       rs.LastCommit.Round(),
 	}
 	if rs.Step == RoundStepCommit {
@@ -365,7 +364,7 @@ func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg *
 	return
 }
 
-func (conR *ConsensusReactor) sendNewRoundStepMessage(peer *p2p.Peer) {
+func (conR *ConsensusReactor) sendNewRoundStepMessages(peer *p2p.Peer) {
 	rs := conR.conS.GetRoundState()
 	nrsMsg, csMsg := makeRoundStepMessages(rs)
 	if nrsMsg != nil {
@@ -377,13 +376,13 @@ func (conR *ConsensusReactor) sendNewRoundStepMessage(peer *p2p.Peer) {
 }
 
 func (conR *ConsensusReactor) gossipDataRoutine(peer *p2p.Peer, ps *PeerState) {
-	log := log.New("peer", peer)
+	logger := conR.Logger.With("peer", peer)
 
 OUTER_LOOP:
 	for {
 		// Manage disconnects from self or peer.
 		if !peer.IsRunning() || !conR.IsRunning() {
-			log.Notice(Fmt("Stopping gossipDataRoutine for %v.", peer))
+			logger.Info("Stopping gossipDataRoutine for peer")
 			return
 		}
 		rs := conR.conS.GetRoundState()
@@ -391,7 +390,7 @@ OUTER_LOOP:
 
 		// Send proposal Block parts?
 		if rs.ProposalBlockParts.HasHeader(prs.ProposalBlockPartsHeader) {
-			//log.Info("ProposalBlockParts matched", "blockParts", prs.ProposalBlockParts)
+			//logger.Info("ProposalBlockParts matched", "blockParts", prs.ProposalBlockParts)
 			if index, ok := rs.ProposalBlockParts.BitArray().Sub(prs.ProposalBlockParts.Copy()).PickRandom(); ok {
 				part := rs.ProposalBlockParts.GetPart(index)
 				msg := &BlockPartMessage{
@@ -399,33 +398,34 @@ OUTER_LOOP:
 					Round:  rs.Round,  // This tells peer that this part applies to us.
 					Part:   part,
 				}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
-				ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				}
 				continue OUTER_LOOP
 			}
 		}
 
 		// If the peer is on a previous height, help catch up.
 		if (0 < prs.Height) && (prs.Height < rs.Height) {
-			//log.Info("Data catchup", "height", rs.Height, "peerHeight", prs.Height, "peerProposalBlockParts", prs.ProposalBlockParts)
+			//logger.Info("Data catchup", "height", rs.Height, "peerHeight", prs.Height, "peerProposalBlockParts", prs.ProposalBlockParts)
 			if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok {
 				// Ensure that the peer's PartSetHeader is correct
 				blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height)
 				if blockMeta == nil {
-					log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height, "blockstore height", conR.conS.blockStore.Height(), "pv", conR.conS.privValidator)
+					logger.Error("Failed to load block meta", "peer height", prs.Height, "ourHeight", rs.Height, "blockstoreHeight", conR.conS.blockStore.Height(), "pv", conR.conS.privValidator)
 					time.Sleep(peerGossipSleepDuration)
 					continue OUTER_LOOP
-				} else if !blockMeta.PartsHeader.Equals(prs.ProposalBlockPartsHeader) {
-					log.Info("Peer ProposalBlockPartsHeader mismatch, sleeping",
-						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
+				} else if !blockMeta.BlockID.PartsHeader.Equals(prs.ProposalBlockPartsHeader) {
+					logger.Info("Peer ProposalBlockPartsHeader mismatch, sleeping",
+						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.BlockID.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
 					time.Sleep(peerGossipSleepDuration)
 					continue OUTER_LOOP
 				}
 				// Load the part
 				part := conR.conS.blockStore.LoadBlockPart(prs.Height, index)
 				if part == nil {
-					log.Warn("Could not load part", "index", index,
-						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
+					logger.Error("Could not load part", "index", index,
+						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.BlockID.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
 					time.Sleep(peerGossipSleepDuration)
 					continue OUTER_LOOP
 				}
@@ -435,11 +435,12 @@ OUTER_LOOP:
 					Round:  prs.Round,  // Not our height, so it doesn't matter.
 					Part:   part,
 				}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
-				ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				}
 				continue OUTER_LOOP
 			} else {
-				//log.Info("No parts to send in catch-up, sleeping")
+				//logger.Info("No parts to send in catch-up, sleeping")
 				time.Sleep(peerGossipSleepDuration)
 				continue OUTER_LOOP
 			}
@@ -447,7 +448,7 @@ OUTER_LOOP:
 
 		// If height and round don't match, sleep.
 		if (rs.Height != prs.Height) || (rs.Round != prs.Round) {
-			//log.Info("Peer Height|Round mismatch, sleeping", "peerHeight", prs.Height, "peerRound", prs.Round, "peer", peer)
+			//logger.Info("Peer Height|Round mismatch, sleeping", "peerHeight", prs.Height, "peerRound", prs.Round, "peer", peer)
 			time.Sleep(peerGossipSleepDuration)
 			continue OUTER_LOOP
 		}
@@ -462,8 +463,9 @@ OUTER_LOOP:
 			// Proposal: share the proposal metadata with peer.
 			{
 				msg := &ProposalMessage{Proposal: rs.Proposal}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
-				ps.SetHasProposal(rs.Proposal)
+				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					ps.SetHasProposal(rs.Proposal)
+				}
 			}
 			// ProposalPOL: lets peer know which POL votes we have so far.
 			// Peer must receive ProposalMessage first.
@@ -487,7 +489,7 @@ OUTER_LOOP:
 }
 
 func (conR *ConsensusReactor) gossipVotesRoutine(peer *p2p.Peer, ps *PeerState) {
-	log := log.New("peer", peer)
+	logger := conR.Logger.With("peer", peer)
 
 	// Simple hack to throttle logs upon sleep.
 	var sleeping = 0
@@ -496,7 +498,7 @@ OUTER_LOOP:
 	for {
 		// Manage disconnects from self or peer.
 		if !peer.IsRunning() || !conR.IsRunning() {
-			log.Notice(Fmt("Stopping gossipVotesRoutine for %v.", peer))
+			logger.Info("Stopping gossipVotesRoutine for peer")
 			return
 		}
 		rs := conR.conS.GetRoundState()
@@ -509,7 +511,7 @@ OUTER_LOOP:
 			sleeping = 0
 		}
 
-		//log.Debug("gossipVotesRoutine", "rsHeight", rs.Height, "rsRound", rs.Round,
+		//logger.Debug("gossipVotesRoutine", "rsHeight", rs.Height, "rsRound", rs.Round,
 		//	"prsHeight", prs.Height, "prsRound", prs.Round, "prsStep", prs.Step)
 
 		// If height matches, then send LastCommit, Prevotes, Precommits.
@@ -517,21 +519,21 @@ OUTER_LOOP:
 			// If there are lastCommits to send...
 			if prs.Step == RoundStepNewHeight {
 				if ps.PickSendVote(rs.LastCommit) {
-					log.Debug("Picked rs.LastCommit to send")
+					logger.Debug("Picked rs.LastCommit to send")
 					continue OUTER_LOOP
 				}
 			}
 			// If there are prevotes to send...
 			if prs.Step <= RoundStepPrevote && prs.Round != -1 && prs.Round <= rs.Round {
 				if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) {
-					log.Debug("Picked rs.Prevotes(prs.Round) to send")
+					logger.Debug("Picked rs.Prevotes(prs.Round) to send")
 					continue OUTER_LOOP
 				}
 			}
 			// If there are precommits to send...
 			if prs.Step <= RoundStepPrecommit && prs.Round != -1 && prs.Round <= rs.Round {
 				if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) {
-					log.Debug("Picked rs.Precommits(prs.Round) to send")
+					logger.Debug("Picked rs.Precommits(prs.Round) to send")
 					continue OUTER_LOOP
 				}
 			}
@@ -539,7 +541,7 @@ OUTER_LOOP:
 			if prs.ProposalPOLRound != -1 {
 				if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil {
 					if ps.PickSendVote(polPrevotes) {
-						log.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send")
+						logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send")
 						continue OUTER_LOOP
 					}
 				}
@@ -550,7 +552,7 @@ OUTER_LOOP:
 		// If peer is lagging by height 1, send LastCommit.
 		if prs.Height != 0 && rs.Height == prs.Height+1 {
 			if ps.PickSendVote(rs.LastCommit) {
-				log.Debug("Picked rs.LastCommit to send")
+				logger.Debug("Picked rs.LastCommit to send")
 				continue OUTER_LOOP
 			}
 		}
@@ -561,9 +563,9 @@ OUTER_LOOP:
 			// Load the block commit for prs.Height,
 			// which contains precommit signatures for prs.Height.
 			commit := conR.conS.blockStore.LoadBlockCommit(prs.Height)
-			log.Info("Loaded BlockCommit for catch-up", "height", prs.Height, "commit", commit)
+			logger.Info("Loaded BlockCommit for catch-up", "height", prs.Height, "commit", commit)
 			if ps.PickSendVote(commit) {
-				log.Debug("Picked Catchup commit to send")
+				logger.Debug("Picked Catchup commit to send")
 				continue OUTER_LOOP
 			}
 		}
@@ -571,7 +573,7 @@ OUTER_LOOP:
 		if sleeping == 0 {
 			// We sent nothing. Sleep...
 			sleeping = 1
-			log.Debug("No votes to send, sleeping", "peer", peer,
+			logger.Debug("No votes to send, sleeping",
 				"localPV", rs.Votes.Prevotes(rs.Round).BitArray(), "peerPV", prs.Prevotes,
 				"localPC", rs.Votes.Precommits(rs.Round).BitArray(), "peerPC", prs.Precommits)
 		} else if sleeping == 2 {
@@ -587,13 +589,13 @@ OUTER_LOOP:
 // NOTE: `queryMaj23Routine` has a simple crude design since it only comes
 // into play for liveness when there's a signature DDoS attack happening.
 func (conR *ConsensusReactor) queryMaj23Routine(peer *p2p.Peer, ps *PeerState) {
-	log := log.New("peer", peer)
+	logger := conR.Logger.With("peer", peer)
 
 OUTER_LOOP:
 	for {
 		// Manage disconnects from self or peer.
 		if !peer.IsRunning() || !conR.IsRunning() {
-			log.Notice(Fmt("Stopping queryMaj23Routine for %v.", peer))
+			logger.Info("Stopping queryMaj23Routine for peer")
 			return
 		}
 
@@ -806,13 +808,12 @@ func (ps *PeerState) SetHasProposalBlockPart(height int, round int, index int) {
 	ps.ProposalBlockParts.SetIndex(index, true)
 }
 
-// Convenience function to send vote to peer.
+// PickVoteToSend sends vote to peer.
 // Returns true if vote was sent.
 func (ps *PeerState) PickSendVote(votes types.VoteSetReader) (ok bool) {
 	if vote, ok := ps.PickVoteToSend(votes); ok {
 		msg := &VoteMessage{vote}
-		ps.Peer.Send(VoteChannel, struct{ ConsensusMessage }{msg})
-		return true
+		return ps.Peer.Send(VoteChannel, struct{ ConsensusMessage }{msg})
 	}
 	return false
 }
@@ -951,8 +952,8 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) {
 }
 
 func (ps *PeerState) setHasVote(height int, round int, type_ byte, index int) {
-	log := log.New("peer", ps.Peer, "peerRound", ps.Round, "height", height, "round", round)
-	log.Debug("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index)
+	logger := ps.Peer.Logger.With("peerRound", ps.Round, "height", height, "round", round)
+	logger.Debug("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index)
 
 	// NOTE: some may be nil BitArrays -> no side effects.
 	ps.getVoteBitArray(height, round, type_).SetIndex(index, true)

consensus/reactor_test.go

@@ -6,16 +6,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/config/tendermint_test"
-
-	"github.com/tendermint/go-events"
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/tendermint/types"
 	"github.com/tendermint/abci/example/dummy"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tmlibs/events"
 )
 
 func init() {
-	config = tendermint_test.ResetConfig("consensus_reactor_test")
+	config = ResetConfig("consensus_reactor_test")
 }
 
 //----------------------------------------------
@@ -24,10 +22,13 @@ func init() {
 func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEventRespond bool) ([]*ConsensusReactor, []chan interface{}) {
 	reactors := make([]*ConsensusReactor, N)
 	eventChans := make([]chan interface{}, N)
+	logger := consensusLogger()
 	for i := 0; i < N; i++ {
 		reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states
+		reactors[i].SetLogger(logger.With("validator", i))
 
 		eventSwitch := events.NewEventSwitch()
+		eventSwitch.SetLogger(logger.With("module", "events", "validator", i))
 		_, err := eventSwitch.Start()
 		if err != nil {
 			t.Fatalf("Failed to start switch: %v", err)
@@ -41,7 +42,7 @@ func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEven
 		}
 	}
 	// make connected switches and start all reactors
-	p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch {
+	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
 		s.AddReactor("CONSENSUS", reactors[i])
 		return s
 	}, p2p.Connect2Switches)
@@ -98,7 +99,7 @@ func TestVotingPowerChange(t *testing.T) {
 	}, css)
 
 	//---------------------------------------------------------------------------
-	log.Info("---------------------------- Testing changing the voting power of one validator a few times")
+	t.Log("---------------------------- Testing changing the voting power of one validator a few times")
 
 	val1PubKey := css[0].privValidator.(*types.PrivValidator).PubKey
 	updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25)
@@ -159,7 +160,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	}, css)
 
 	//---------------------------------------------------------------------------
-	log.Info("---------------------------- Testing adding one validator")
+	t.Log("---------------------------- Testing adding one validator")
 
 	newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey
 	newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower))
@@ -185,7 +186,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 
 	//---------------------------------------------------------------------------
-	log.Info("---------------------------- Testing changing the voting power of one validator")
+	t.Log("---------------------------- Testing changing the voting power of one validator")
 
 	updateValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey
 	updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25)
@@ -201,7 +202,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	}
 
 	//---------------------------------------------------------------------------
-	log.Info("---------------------------- Testing adding two validators at once")
+	t.Log("---------------------------- Testing adding two validators at once")
 
 	newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey
 	newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower))
@@ -217,7 +218,7 @@ func TestValidatorSetChanges(t *testing.T) {
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 
 	//---------------------------------------------------------------------------
-	log.Info("---------------------------- Testing removing two validators at once")
+	t.Log("---------------------------- Testing removing two validators at once")
 
 	removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0)
 	removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0)
@@ -236,7 +237,7 @@ func TestReactorWithTimeoutCommit(t *testing.T) {
 	css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter)
 	// override default SkipTimeoutCommit == true for tests
 	for i := 0; i < N; i++ {
-		css[i].timeoutParams.SkipTimeoutCommit = false
+		css[i].config.SkipTimeoutCommit = false
 	}
 
 	reactors, eventChans := startConsensusNet(t, css, N-1, false)
@@ -252,8 +253,8 @@ func TestReactorWithTimeoutCommit(t *testing.T) {
 func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
 	timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) {
 		newBlockI := <-eventChans[j]
-		newBlock := newBlockI.(types.EventDataNewBlock).Block
-		log.Warn("Got block", "height", newBlock.Height, "validator", j)
+		newBlock := newBlockI.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block
+		t.Logf("[WARN] Got block height=%v validator=%v", newBlock.Height, j)
 		err := validateBlock(newBlock, activeVals)
 		if err != nil {
 			t.Fatal(err)
@@ -264,7 +265,6 @@ func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}
 
 		eventChans[j] <- struct{}{}
 		wg.Done()
-		log.Warn("Done wait group", "height", newBlock.Height, "validator", j)
 	}, css)
 }
 

consensus/replay.go

@@ -1,24 +1,35 @@
 package consensus
 
 import (
-	"bufio"
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"reflect"
 	"strconv"
 	"strings"
 	"time"
 
-	auto "github.com/tendermint/go-autofile"
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-wire"
+	abci "github.com/tendermint/abci/types"
+	wire "github.com/tendermint/go-wire"
+	auto "github.com/tendermint/tmlibs/autofile"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 
+	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
+// Functionality to replay blocks and messages on recovery from a crash.
+// There are two general failure scenarios: failure during consensus, and failure while applying the block.
+// The former is handled by the WAL, the latter by the proxyApp Handshake on restart,
+// which ultimately hands off the work to the WAL.
+
+//-----------------------------------------
+// recover from failure during consensus
+// by replaying messages from the WAL
+
 // Unmarshal and apply a single message to the consensus state
 // as if it were received in receiveRoutine
 // Lines that start with "#" are ignored.
@@ -39,7 +50,7 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte
 	// for logging
 	switch m := msg.Msg.(type) {
 	case types.EventDataRoundState:
-		log.Notice("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
+		cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
 		// these are playback checks
 		ticker := time.After(time.Second * 2)
 		if newStepCh != nil {
@@ -61,19 +72,19 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte
 		switch msg := m.Msg.(type) {
 		case *ProposalMessage:
 			p := msg.Proposal
-			log.Notice("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
+			cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
 				p.BlockPartsHeader, "pol", p.POLRound, "peer", peerKey)
 		case *BlockPartMessage:
-			log.Notice("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerKey)
+			cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerKey)
 		case *VoteMessage:
 			v := msg.Vote
-			log.Notice("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
+			cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
 				"blockID", v.BlockID, "peer", peerKey)
 		}
 
 		cs.handleMsg(m, cs.RoundState)
 	case timeoutInfo:
-		log.Notice("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
+		cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
 		cs.handleTimeout(m, cs.RoundState)
 	default:
 		return fmt.Errorf("Replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg))
@@ -89,29 +100,53 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 	cs.replayMode = true
 	defer func() { cs.replayMode = false }()
 
-	// Ensure that height+1 doesn't exist
-	gr, found, err := cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight+1))
-	if found {
-		return errors.New(Fmt("WAL should not contain height %d.", csHeight+1))
-	}
+	// Ensure that ENDHEIGHT for this height doesn't exist
+	// NOTE: This is just a sanity check. As far as we know things work fine without it,
+	// and Handshake could reuse ConsensusState if it weren't for this check (since we can crash after writing ENDHEIGHT).
+	gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight))
 	if gr != nil {
 		gr.Close()
 	}
+	if found {
+		return errors.New(cmn.Fmt("WAL should not contain #ENDHEIGHT %d.", csHeight))
+	}
 
-	// Search for height marker
-	gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight))
+	// Search for last height marker
+	gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1))
 	if err == io.EOF {
-		log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight)
-		return nil
+		cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", csHeight-1)
+		// if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead
+		// TODO (0.10.0): remove this
+		gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight))
+		if err == io.EOF {
+			cs.Logger.Error("Replay: wal.group.Search returned EOF", "#HEIGHT", csHeight)
+			return nil
+		} else if err != nil {
+			return err
+		}
 	} else if err != nil {
 		return err
+	} else {
+		defer gr.Close()
 	}
 	if !found {
-		return errors.New(Fmt("WAL does not contain height %d.", csHeight))
-	}
-	defer gr.Close()
+		// if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead
+		// TODO (0.10.0): remove this
+		gr, _, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight))
+		if err == io.EOF {
+			cs.Logger.Error("Replay: wal.group.Search returned EOF", "#HEIGHT", csHeight)
+			return nil
+		} else if err != nil {
+			return err
+		} else {
+			defer gr.Close()
+		}
 
-	log.Notice("Catchup by replaying consensus messages", "height", csHeight)
+		// TODO (0.10.0): uncomment
+		// return errors.New(cmn.Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1))
+	}
+
+	cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight)
 
 	for {
 		line, err := gr.ReadLine()
@@ -129,232 +164,14 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 			return err
 		}
 	}
-	log.Notice("Replay: Done")
+	cs.Logger.Info("Replay: Done")
 	return nil
 }
 
-//--------------------------------------------------------
-// replay messages interactively or all at once
-
-// Interactive playback
-func (cs ConsensusState) ReplayConsole(file string) error {
-	return cs.replay(file, true)
-}
-
-// Full playback, with tests
-func (cs ConsensusState) ReplayMessages(file string) error {
-	return cs.replay(file, false)
-}
-
-// replay all msgs or start the console
-func (cs *ConsensusState) replay(file string, console bool) error {
-	if cs.IsRunning() {
-		return errors.New("cs is already running, cannot replay")
-	}
-	if cs.wal != nil {
-		return errors.New("cs wal is open, cannot replay")
-	}
-
-	cs.startForReplay()
-
-	// ensure all new step events are regenerated as expected
-	newStepCh := subscribeToEvent(cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
-
-	// just open the file for reading, no need to use wal
-	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
-	if err != nil {
-		return err
-	}
-
-	pb := newPlayback(file, fp, cs, cs.state.Copy())
-	defer pb.fp.Close()
-
-	var nextN int // apply N msgs in a row
-	for pb.scanner.Scan() {
-		if nextN == 0 && console {
-			nextN = pb.replayConsoleLoop()
-		}
-
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
-			return err
-		}
-
-		if nextN > 0 {
-			nextN -= 1
-		}
-		pb.count += 1
-	}
-	return nil
-}
-
-//------------------------------------------------
-// playback manager
-
-type playback struct {
-	cs *ConsensusState
-
-	fp      *os.File
-	scanner *bufio.Scanner
-	count   int // how many lines/msgs into the file are we
-
-	// replays can be reset to beginning
-	fileName     string    // so we can close/reopen the file
-	genesisState *sm.State // so the replay session knows where to restart from
-}
-
-func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm.State) *playback {
-	return &playback{
-		cs:           cs,
-		fp:           fp,
-		fileName:     fileName,
-		genesisState: genState,
-		scanner:      bufio.NewScanner(fp),
-	}
-}
-
-// go back count steps by resetting the state and running (pb.count - count) steps
-func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
-
-	pb.cs.Stop()
-	pb.cs.Wait()
-
-	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool)
-	newCS.SetEventSwitch(pb.cs.evsw)
-	newCS.startForReplay()
-
-	pb.fp.Close()
-	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0666)
-	if err != nil {
-		return err
-	}
-	pb.fp = fp
-	pb.scanner = bufio.NewScanner(fp)
-	count = pb.count - count
-	log.Notice(Fmt("Reseting from %d to %d", pb.count, count))
-	pb.count = 0
-	pb.cs = newCS
-	for i := 0; pb.scanner.Scan() && i < count; i++ {
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
-			return err
-		}
-		pb.count += 1
-	}
-	return nil
-}
-
-func (cs *ConsensusState) startForReplay() {
-	// don't want to start full cs
-	cs.BaseService.OnStart()
-
-	log.Warn("Replay commands are disabled until someone updates them and writes tests")
-	/* TODO:!
-	// since we replay tocks we just ignore ticks
-		go func() {
-			for {
-				select {
-				case <-cs.tickChan:
-				case <-cs.Quit:
-					return
-				}
-			}
-		}()*/
-}
-
-// console function for parsing input and running commands
-func (pb *playback) replayConsoleLoop() int {
-	for {
-		fmt.Printf("> ")
-		bufReader := bufio.NewReader(os.Stdin)
-		line, more, err := bufReader.ReadLine()
-		if more {
-			Exit("input is too long")
-		} else if err != nil {
-			Exit(err.Error())
-		}
-
-		tokens := strings.Split(string(line), " ")
-		if len(tokens) == 0 {
-			continue
-		}
-
-		switch tokens[0] {
-		case "next":
-			// "next" -> replay next message
-			// "next N" -> replay next N messages
-
-			if len(tokens) == 1 {
-				return 0
-			} else {
-				i, err := strconv.Atoi(tokens[1])
-				if err != nil {
-					fmt.Println("next takes an integer argument")
-				} else {
-					return i
-				}
-			}
-
-		case "back":
-			// "back" -> go back one message
-			// "back N" -> go back N messages
-
-			// NOTE: "back" is not supported in the state machine design,
-			// so we restart and replay up to
-
-			// ensure all new step events are regenerated as expected
-			newStepCh := subscribeToEvent(pb.cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
-			if len(tokens) == 1 {
-				pb.replayReset(1, newStepCh)
-			} else {
-				i, err := strconv.Atoi(tokens[1])
-				if err != nil {
-					fmt.Println("back takes an integer argument")
-				} else if i > pb.count {
-					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
-				} else {
-					pb.replayReset(i, newStepCh)
-				}
-			}
-
-		case "rs":
-			// "rs" -> print entire round state
-			// "rs short" -> print height/round/step
-			// "rs <field>" -> print another field of the round state
-
-			rs := pb.cs.RoundState
-			if len(tokens) == 1 {
-				fmt.Println(rs)
-			} else {
-				switch tokens[1] {
-				case "short":
-					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
-				case "validators":
-					fmt.Println(rs.Validators)
-				case "proposal":
-					fmt.Println(rs.Proposal)
-				case "proposal_block":
-					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
-				case "locked_round":
-					fmt.Println(rs.LockedRound)
-				case "locked_block":
-					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
-				case "votes":
-					fmt.Println(rs.Votes.StringIndented("    "))
-
-				default:
-					fmt.Println("Unknown option", tokens[1])
-				}
-			}
-		case "n":
-			fmt.Println(pb.count)
-		}
-	}
-	return 0
-}
-
 //--------------------------------------------------------------------------------
 
 // Parses marker lines of the form:
-// #HEIGHT: 12345
+// #ENDHEIGHT: 12345
 func makeHeightSearchFunc(height int) auto.SearchFunc {
 	return func(line string) (int, error) {
 		line = strings.TrimRight(line, "\n")
@@ -375,3 +192,229 @@ func makeHeightSearchFunc(height int) auto.SearchFunc {
 		}
 	}
 }
+
+//----------------------------------------------
+// Recover from failure during block processing
+// by handshaking with the app to figure out where
+// we were last and using the WAL to recover there
+
+type Handshaker struct {
+	state  *sm.State
+	store  types.BlockStore
+	logger log.Logger
+
+	nBlocks int // number of blocks applied to the state
+}
+
+func NewHandshaker(state *sm.State, store types.BlockStore) *Handshaker {
+	return &Handshaker{state, store, log.NewNopLogger(), 0}
+}
+
+func (h *Handshaker) SetLogger(l log.Logger) {
+	h.logger = l
+}
+
+func (h *Handshaker) NBlocks() int {
+	return h.nBlocks
+}
+
+// TODO: retry the handshake/replay if it fails ?
+func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
+	// handshake is done via info request on the query conn
+	res, err := proxyApp.Query().InfoSync()
+	if err != nil {
+		return errors.New(cmn.Fmt("Error calling Info: %v", err))
+	}
+
+	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
+	appHash := res.LastBlockAppHash
+
+	h.logger.Info("ABCI Handshake", "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
+
+	// TODO: check version
+
+	// replay blocks up to the latest in the blockstore
+	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
+	if err != nil {
+		return errors.New(cmn.Fmt("Error on replay: %v", err))
+	}
+
+	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced", "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
+
+	// TODO: (on restart) replay mempool
+
+	return nil
+}
+
+// Replay all blocks since appBlockHeight and ensure the result matches the current state.
+// Returns the final AppHash or an error
+func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
+
+	storeBlockHeight := h.store.Height()
+	stateBlockHeight := h.state.LastBlockHeight
+	h.logger.Info("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)
+
+	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain
+	if appBlockHeight == 0 {
+		validators := types.TM2PB.Validators(h.state.Validators)
+		proxyApp.Consensus().InitChainSync(validators)
+	}
+
+	// First handle edge cases and constraints on the storeBlockHeight
+	if storeBlockHeight == 0 {
+		return appHash, h.checkAppHash(appHash)
+
+	} else if storeBlockHeight < appBlockHeight {
+		// the app should never be ahead of the store (but this is under app's control)
+		return appHash, sm.ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
+
+	} else if storeBlockHeight < stateBlockHeight {
+		// the state should never be ahead of the store (this is under tendermint's control)
+		cmn.PanicSanity(cmn.Fmt("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
+
+	} else if storeBlockHeight > stateBlockHeight+1 {
+		// store should be at most one ahead of the state (this is under tendermint's control)
+		cmn.PanicSanity(cmn.Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
+	}
+
+	// Now either store is equal to state, or one ahead.
+	// For each, consider all cases of where the app could be, given app <= store
+	if storeBlockHeight == stateBlockHeight {
+		// Tendermint ran Commit and saved the state.
+		// Either the app is asking for replay, or we're all synced up.
+		if appBlockHeight < storeBlockHeight {
+			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
+			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// We're good!
+			return appHash, h.checkAppHash(appHash)
+		}
+
+	} else if storeBlockHeight == stateBlockHeight+1 {
+		// We saved the block in the store but haven't updated the state,
+		// so we'll need to replay a block using the WAL.
+		if appBlockHeight < stateBlockHeight {
+			// the app is further behind than it should be, so replay blocks
+			// but leave the last block to go through the WAL
+			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)
+
+		} else if appBlockHeight == stateBlockHeight {
+			// We haven't run Commit (both the state and app are one block behind),
+			// so replayBlock with the real app.
+			// NOTE: We could instead use the cs.WAL on cs.Start,
+			// but we'd have to allow the WAL to replay a block that wrote it's ENDHEIGHT
+			h.logger.Info("Replay last block using real app")
+			return h.replayBlock(storeBlockHeight, proxyApp.Consensus())
+
+		} else if appBlockHeight == storeBlockHeight {
+			// We ran Commit, but didn't save the state, so replayBlock with mock app
+			abciResponses := h.state.LoadABCIResponses()
+			mockApp := newMockProxyApp(appHash, abciResponses)
+			h.logger.Info("Replay last block using mock app")
+			return h.replayBlock(storeBlockHeight, mockApp)
+		}
+
+	}
+
+	cmn.PanicSanity("Should never happen")
+	return nil, nil
+}
+
+func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) {
+	// App is further behind than it should be, so we need to replay blocks.
+	// We replay all blocks from appBlockHeight+1.
+	// Note that we don't have an old version of the state,
+	// so we by-pass state validation/mutation using sm.ExecCommitBlock.
+	// If mutateState == true, the final block is replayed with h.replayBlock()
+
+	var appHash []byte
+	var err error
+	finalBlock := storeBlockHeight
+	if mutateState {
+		finalBlock -= 1
+	}
+	for i := appBlockHeight + 1; i <= finalBlock; i++ {
+		h.logger.Info("Applying block", "height", i)
+		block := h.store.LoadBlock(i)
+		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger)
+		if err != nil {
+			return nil, err
+		}
+
+		h.nBlocks += 1
+	}
+
+	if mutateState {
+		// sync the final block
+		return h.replayBlock(storeBlockHeight, proxyApp.Consensus())
+	}
+
+	return appHash, h.checkAppHash(appHash)
+}
+
+// ApplyBlock on the proxyApp with the last block.
+func (h *Handshaker) replayBlock(height int, proxyApp proxy.AppConnConsensus) ([]byte, error) {
+	mempool := types.MockMempool{}
+
+	var eventCache types.Fireable // nil
+	block := h.store.LoadBlock(height)
+	meta := h.store.LoadBlockMeta(height)
+
+	if err := h.state.ApplyBlock(eventCache, proxyApp, block, meta.BlockID.PartsHeader, mempool); err != nil {
+		return nil, err
+	}
+
+	h.nBlocks += 1
+
+	return h.state.AppHash, nil
+}
+
+func (h *Handshaker) checkAppHash(appHash []byte) error {
+	if !bytes.Equal(h.state.AppHash, appHash) {
+		panic(errors.New(cmn.Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)).Error())
+		return nil
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------------
+// mockProxyApp uses ABCIResponses to give the right results
+// Useful because we don't want to call Commit() twice for the same block on the real app.
+
+func newMockProxyApp(appHash []byte, abciResponses *sm.ABCIResponses) proxy.AppConnConsensus {
+	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{
+		appHash:       appHash,
+		abciResponses: abciResponses,
+	})
+	cli, _ := clientCreator.NewABCIClient()
+	cli.Start()
+	return proxy.NewAppConnConsensus(cli)
+}
+
+type mockProxyApp struct {
+	abci.BaseApplication
+
+	appHash       []byte
+	txCount       int
+	abciResponses *sm.ABCIResponses
+}
+
+func (mock *mockProxyApp) DeliverTx(tx []byte) abci.Result {
+	r := mock.abciResponses.DeliverTx[mock.txCount]
+	mock.txCount += 1
+	return abci.Result{
+		r.Code,
+		r.Data,
+		r.Log,
+	}
+}
+
+func (mock *mockProxyApp) EndBlock(height uint64) abci.ResponseEndBlock {
+	mock.txCount = 0
+	return mock.abciResponses.EndBlock
+}
+
+func (mock *mockProxyApp) Commit() abci.Result {
+	return abci.NewResultOK(mock.appHash, "")
+}

consensus/replay_file.go

@@ -0,0 +1,264 @@
+package consensus
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	bc "github.com/tendermint/tendermint/blockchain"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
+)
+
+//--------------------------------------------------------
+// replay messages interactively or all at once
+
+func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) {
+	consensusState := newConsensusStateForReplay(config, csConfig)
+
+	if err := consensusState.ReplayFile(csConfig.WalFile(), console); err != nil {
+		cmn.Exit(cmn.Fmt("Error during consensus replay: %v", err))
+	}
+}
+
+// Replay msgs in file or start the console
+func (cs *ConsensusState) ReplayFile(file string, console bool) error {
+
+	if cs.IsRunning() {
+		return errors.New("cs is already running, cannot replay")
+	}
+	if cs.wal != nil {
+		return errors.New("cs wal is open, cannot replay")
+	}
+
+	cs.startForReplay()
+
+	// ensure all new step events are regenerated as expected
+	newStepCh := subscribeToEvent(cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+
+	// just open the file for reading, no need to use wal
+	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
+	if err != nil {
+		return err
+	}
+
+	pb := newPlayback(file, fp, cs, cs.state.Copy())
+	defer pb.fp.Close()
+
+	var nextN int // apply N msgs in a row
+	for pb.scanner.Scan() {
+		if nextN == 0 && console {
+			nextN = pb.replayConsoleLoop()
+		}
+
+		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+			return err
+		}
+
+		if nextN > 0 {
+			nextN -= 1
+		}
+		pb.count += 1
+	}
+	return nil
+}
+
+//------------------------------------------------
+// playback manager
+
+type playback struct {
+	cs *ConsensusState
+
+	fp      *os.File
+	scanner *bufio.Scanner
+	count   int // how many lines/msgs into the file are we
+
+	// replays can be reset to beginning
+	fileName     string    // so we can close/reopen the file
+	genesisState *sm.State // so the replay session knows where to restart from
+}
+
+func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm.State) *playback {
+	return &playback{
+		cs:           cs,
+		fp:           fp,
+		fileName:     fileName,
+		genesisState: genState,
+		scanner:      bufio.NewScanner(fp),
+	}
+}
+
+// go back count steps by resetting the state and running (pb.count - count) steps
+func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
+
+	pb.cs.Stop()
+	pb.cs.Wait()
+
+	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool)
+	newCS.SetEventSwitch(pb.cs.evsw)
+	newCS.startForReplay()
+
+	pb.fp.Close()
+	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0666)
+	if err != nil {
+		return err
+	}
+	pb.fp = fp
+	pb.scanner = bufio.NewScanner(fp)
+	count = pb.count - count
+	fmt.Printf("Reseting from %d to %d\n", pb.count, count)
+	pb.count = 0
+	pb.cs = newCS
+	for i := 0; pb.scanner.Scan() && i < count; i++ {
+		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+			return err
+		}
+		pb.count += 1
+	}
+	return nil
+}
+
+func (cs *ConsensusState) startForReplay() {
+	cs.Logger.Error("Replay commands are disabled until someone updates them and writes tests")
+	/* TODO:!
+	// since we replay tocks we just ignore ticks
+		go func() {
+			for {
+				select {
+				case <-cs.tickChan:
+				case <-cs.Quit:
+					return
+				}
+			}
+		}()*/
+}
+
+// console function for parsing input and running commands
+func (pb *playback) replayConsoleLoop() int {
+	for {
+		fmt.Printf("> ")
+		bufReader := bufio.NewReader(os.Stdin)
+		line, more, err := bufReader.ReadLine()
+		if more {
+			cmn.Exit("input is too long")
+		} else if err != nil {
+			cmn.Exit(err.Error())
+		}
+
+		tokens := strings.Split(string(line), " ")
+		if len(tokens) == 0 {
+			continue
+		}
+
+		switch tokens[0] {
+		case "next":
+			// "next" -> replay next message
+			// "next N" -> replay next N messages
+
+			if len(tokens) == 1 {
+				return 0
+			} else {
+				i, err := strconv.Atoi(tokens[1])
+				if err != nil {
+					fmt.Println("next takes an integer argument")
+				} else {
+					return i
+				}
+			}
+
+		case "back":
+			// "back" -> go back one message
+			// "back N" -> go back N messages
+
+			// NOTE: "back" is not supported in the state machine design,
+			// so we restart and replay up to
+
+			// ensure all new step events are regenerated as expected
+			newStepCh := subscribeToEvent(pb.cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+			if len(tokens) == 1 {
+				pb.replayReset(1, newStepCh)
+			} else {
+				i, err := strconv.Atoi(tokens[1])
+				if err != nil {
+					fmt.Println("back takes an integer argument")
+				} else if i > pb.count {
+					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
+				} else {
+					pb.replayReset(i, newStepCh)
+				}
+			}
+
+		case "rs":
+			// "rs" -> print entire round state
+			// "rs short" -> print height/round/step
+			// "rs <field>" -> print another field of the round state
+
+			rs := pb.cs.RoundState
+			if len(tokens) == 1 {
+				fmt.Println(rs)
+			} else {
+				switch tokens[1] {
+				case "short":
+					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
+				case "validators":
+					fmt.Println(rs.Validators)
+				case "proposal":
+					fmt.Println(rs.Proposal)
+				case "proposal_block":
+					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
+				case "locked_round":
+					fmt.Println(rs.LockedRound)
+				case "locked_block":
+					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
+				case "votes":
+					fmt.Println(rs.Votes.StringIndented("    "))
+
+				default:
+					fmt.Println("Unknown option", tokens[1])
+				}
+			}
+		case "n":
+			fmt.Println(pb.count)
+		}
+	}
+	return 0
+}
+
+//--------------------------------------------------------------------------------
+
+// convenience for replay mode
+func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *ConsensusState {
+	// Get BlockStore
+	blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir())
+	blockStore := bc.NewBlockStore(blockStoreDB)
+
+	// Get State
+	stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir())
+	state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
+
+	// Create proxyAppConn connection (consensus, mempool, query)
+	clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
+	proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore))
+	_, err := proxyApp.Start()
+	if err != nil {
+		cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err))
+	}
+
+	// Make event switch
+	eventSwitch := types.NewEventSwitch()
+	if _, err := eventSwitch.Start(); err != nil {
+		cmn.Exit(cmn.Fmt("Failed to start event switch: %v", err))
+	}
+
+	consensusState := NewConsensusState(csConfig, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{})
+
+	consensusState.SetEventSwitch(eventSwitch)
+	return consensusState
+}

consensus/replay_test.go

@@ -1,7 +1,10 @@
 package consensus
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -9,25 +12,46 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/tendermint/config/tendermint_test"
+	"github.com/tendermint/abci/example/dummy"
+	crypto "github.com/tendermint/go-crypto"
+	wire "github.com/tendermint/go-wire"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
 
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-wire"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tmlibs/log"
 )
 
 func init() {
-	config = tendermint_test.ResetConfig("consensus_replay_test")
+	config = ResetConfig("consensus_replay_test")
 }
 
-// TODO: these tests ensure we can always recover from any state of the wal,
-// assuming it comes with a correct related state for the priv_validator.json.
-// It would be better to verify explicitly which states we can recover from without the wal
+// These tests ensure we can always recover from failure at any part of the consensus process.
+// There are two general failure scenarios: failure during consensus, and failure while applying the block.
+// Only the latter interacts with the app and store,
+// but the former has to deal with restrictions on re-use of priv_validator keys.
+// The `WAL Tests` are for failures during the consensus;
+// the `Handshake Tests` are for failures in applying the block.
+// With the help of the WAL, we can recover from it all!
+
+// NOTE: Files in this dir are generated by running the `build.sh` therein.
+// It's a simple way to generate wals for a single block, or multiple blocks, with random transactions,
+// and different part sizes. The output is not deterministic, and the stepChanges may need to be adjusted
+// after running it (eg. sometimes small_block2 will have 5 block parts, sometimes 6).
+// It should only have to be re-run if there is some breaking change to the consensus data structures (eg. blocks, votes)
+// or to the behaviour of the app (eg. computes app hash differently)
+var data_dir = path.Join(cmn.GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
+
+//------------------------------------------------------------------------------------------
+// WAL Tests
+
+// TODO: It would be better to verify explicitly which states we can recover from without the wal
 // and which ones we need the wal for - then we'd also be able to only flush the
 // wal writer when we need to, instead of with every message.
 
-var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
-
 // the priv validator changes step at these lines for a block with 1 val and 1 part
 var baseStepChanges = []int{3, 6, 8}
 
@@ -35,7 +59,7 @@ var baseStepChanges = []int{3, 6, 8}
 var testCases = []*testCase{
 	newTestCase("empty_block", baseStepChanges),   // empty block (has 1 block part)
 	newTestCase("small_block1", baseStepChanges),  // small block with txs in 1 block part
-	newTestCase("small_block2", []int{3, 10, 12}), // small block with txs across 5 smaller block parts
+	newTestCase("small_block2", []int{3, 11, 13}), // small block with txs across 6 smaller block parts
 }
 
 type testCase struct {
@@ -50,7 +74,7 @@ type testCase struct {
 
 func newTestCase(name string, stepChanges []int) *testCase {
 	if len(stepChanges) != 3 {
-		panic(Fmt("a full wal has 3 step changes! Got array %v", stepChanges))
+		panic(cmn.Fmt("a full wal has 3 step changes! Got array %v", stepChanges))
 	}
 	return &testCase{
 		name:    name,
@@ -85,18 +109,19 @@ func readWAL(p string) string {
 
 func writeWAL(walMsgs string) string {
 	tempDir := os.TempDir()
-	walDir := tempDir + "/wal" + RandStr(12)
+	walDir := path.Join(tempDir, "/wal"+cmn.RandStr(12))
+	walFile := path.Join(walDir, "wal")
 	// Create WAL directory
-	err := EnsureDir(walDir, 0700)
+	err := cmn.EnsureDir(walDir, 0700)
 	if err != nil {
 		panic(err)
 	}
 	// Write the needed WAL to file
-	err = WriteFile(walDir+"/wal", []byte(walMsgs), 0600)
+	err = cmn.WriteFile(walFile, []byte(walMsgs), 0600)
 	if err != nil {
 		panic(err)
 	}
-	return walDir
+	return walFile
 }
 
 func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
@@ -104,15 +129,21 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
 	select {
 	case <-newBlockCh:
 	case <-after:
-		panic(Fmt("Timed out waiting for new block for case '%s' line %d", thisCase.name, i))
+		panic(cmn.Fmt("Timed out waiting for new block for case '%s' line %d", thisCase.name, i))
 	}
 }
 
-func runReplayTest(t *testing.T, cs *ConsensusState, walDir string, newBlockCh chan interface{},
+func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{},
 	thisCase *testCase, i int) {
 
-	cs.config.Set("cs_wal_dir", walDir)
-	cs.Start()
+	cs.config.SetWalFile(walFile)
+	started, err := cs.Start()
+	if err != nil {
+		t.Fatalf("Cannot start consensus: %v", err)
+	}
+	if !started {
+		t.Error("Consensus did not start")
+	}
 	// Wait to make a new block.
 	// This is just a signal that we haven't halted; its not something contained in the WAL itself.
 	// Assuming the consensus state is running, replay of any WAL, including the empty one,
@@ -135,9 +166,9 @@ func toPV(pv PrivValidator) *types.PrivValidator {
 	return pv.(*types.PrivValidator)
 }
 
-func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) {
-	fmt.Println("-------------------------------------")
-	log.Notice(Fmt("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter))
+func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) {
+	t.Log("-------------------------------------")
+	t.Logf("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter)
 
 	lineStep := nLines
 	if crashAfter {
@@ -148,7 +179,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu
 	lastMsg := split[nLines]
 
 	// we write those lines up to (not including) one with the signature
-	walDir := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
+	walFile := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
 
 	cs := fixedConsensusStateDummy()
 
@@ -156,11 +187,11 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu
 	toPV(cs.privValidator).LastHeight = 1 // first block
 	toPV(cs.privValidator).LastStep = thisCase.stepMap[lineStep]
 
-	log.Warn("setupReplayTest", "LastStep", toPV(cs.privValidator).LastStep)
+	t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).LastStep)
 
 	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
 
-	return cs, newBlockCh, lastMsg, walDir
+	return cs, newBlockCh, lastMsg, walFile
 }
 
 func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
@@ -177,12 +208,12 @@ func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
 // Test the log at every iteration, and set the privVal last step
 // as if the log was written after signing, before the crash
 
-func TestReplayCrashAfterWrite(t *testing.T) {
+func TestWALCrashAfterWrite(t *testing.T) {
 	for _, thisCase := range testCases {
 		split := strings.Split(thisCase.log, "\n")
 		for i := 0; i < len(split)-1; i++ {
-			cs, newBlockCh, _, walDir := setupReplayTest(thisCase, i+1, true)
-			runReplayTest(t, cs, walDir, newBlockCh, thisCase, i+1)
+			cs, newBlockCh, _, walFile := setupReplayTest(t, thisCase, i+1, true)
+			runReplayTest(t, cs, walFile, newBlockCh, thisCase, i+1)
 		}
 	}
 }
@@ -191,27 +222,27 @@ func TestReplayCrashAfterWrite(t *testing.T) {
 // Test the log as if we crashed after signing but before writing.
 // This relies on privValidator.LastSignature being set
 
-func TestReplayCrashBeforeWritePropose(t *testing.T) {
+func TestWALCrashBeforeWritePropose(t *testing.T) {
 	for _, thisCase := range testCases {
 		lineNum := thisCase.proposeLine
 		// setup replay test where last message is a proposal
-		cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false)
+		cs, newBlockCh, proposalMsg, walFile := setupReplayTest(t, thisCase, lineNum, false)
 		msg := readTimedWALMessage(t, proposalMsg)
 		proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage)
 		// Set LastSig
 		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal)
 		toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature
-		runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum)
+		runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
 	}
 }
 
-func TestReplayCrashBeforeWritePrevote(t *testing.T) {
+func TestWALCrashBeforeWritePrevote(t *testing.T) {
 	for _, thisCase := range testCases {
 		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal())
 	}
 }
 
-func TestReplayCrashBeforeWritePrecommit(t *testing.T) {
+func TestWALCrashBeforeWritePrecommit(t *testing.T) {
 	for _, thisCase := range testCases {
 		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka())
 	}
@@ -219,7 +250,7 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) {
 
 func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) {
 	// setup replay test where last message is a vote
-	cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false)
+	cs, newBlockCh, voteMsg, walFile := setupReplayTest(t, thisCase, lineNum, false)
 	types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) {
 		msg := readTimedWALMessage(t, voteMsg)
 		vote := msg.Msg.(msgInfo).Msg.(*VoteMessage)
@@ -227,5 +258,344 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in
 		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote)
 		toPV(cs.privValidator).LastSignature = vote.Vote.Signature
 	})
-	runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum)
+	runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
+}
+
+//------------------------------------------------------------------------------------------
+// Handshake Tests
+
+var (
+	NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal
+	mempool    = types.MockMempool{}
+
+	testPartSize int
+)
+
+//---------------------------------------
+// Test handshake/replay
+
+// 0 - all synced up
+// 1 - saved block but app and state are behind
+// 2 - save block and committed but state is behind
+var modes = []uint{0, 1, 2}
+
+// Sync from scratch
+func TestHandshakeReplayAll(t *testing.T) {
+	for _, m := range modes {
+		testHandshakeReplay(t, 0, m)
+	}
+}
+
+// Sync many, not from scratch
+func TestHandshakeReplaySome(t *testing.T) {
+	for _, m := range modes {
+		testHandshakeReplay(t, 1, m)
+	}
+}
+
+// Sync from lagging by one
+func TestHandshakeReplayOne(t *testing.T) {
+	for _, m := range modes {
+		testHandshakeReplay(t, NUM_BLOCKS-1, m)
+	}
+}
+
+// Sync from caught up
+func TestHandshakeReplayNone(t *testing.T) {
+	for _, m := range modes {
+		testHandshakeReplay(t, NUM_BLOCKS, m)
+	}
+}
+
+// Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
+func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
+	config := ResetConfig("proxy_test_")
+
+	// copy the many_blocks file
+	walBody, err := cmn.ReadFile(path.Join(data_dir, "many_blocks.cswal"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	walFile := writeWAL(string(walBody))
+	config.Consensus.SetWalFile(walFile)
+
+	privVal := types.LoadPrivValidator(config.PrivValidatorFile())
+	testPartSize = config.Consensus.BlockPartSize
+
+	wal, err := NewWAL(walFile, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wal.SetLogger(log.TestingLogger())
+	if _, err := wal.Start(); err != nil {
+		t.Fatal(err)
+	}
+	chain, commits, err := makeBlockchainFromWAL(wal)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	state, store := stateAndStore(config, privVal.PubKey)
+	store.chain = chain
+	store.commits = commits
+
+	// run the chain through state.ApplyBlock to build up the tendermint state
+	latestAppHash := buildTMStateFromChain(config, state, chain, mode)
+
+	// make a new client creator
+	dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "2"))
+	clientCreator2 := proxy.NewLocalClientCreator(dummyApp)
+	if nBlocks > 0 {
+		// run nBlocks against a new client to build up the app state.
+		// use a throwaway tendermint state
+		proxyApp := proxy.NewAppConns(clientCreator2, nil)
+		state, _ := stateAndStore(config, privVal.PubKey)
+		buildAppStateFromChain(proxyApp, state, chain, nBlocks, mode)
+	}
+
+	// now start the app using the handshake - it should sync
+	handshaker := NewHandshaker(state, store)
+	proxyApp := proxy.NewAppConns(clientCreator2, handshaker)
+	if _, err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+
+	// get the latest app hash from the app
+	res, err := proxyApp.Query().InfoSync()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the app hash should be synced up
+	if !bytes.Equal(latestAppHash, res.LastBlockAppHash) {
+		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
+	}
+
+	expectedBlocksToSync := NUM_BLOCKS - nBlocks
+	if nBlocks == NUM_BLOCKS && mode > 0 {
+		expectedBlocksToSync += 1
+	} else if nBlocks > 0 && mode == 1 {
+		expectedBlocksToSync += 1
+	}
+
+	if handshaker.NBlocks() != expectedBlocksToSync {
+		t.Fatalf("Expected handshake to sync %d blocks, got %d", expectedBlocksToSync, handshaker.NBlocks())
+	}
+}
+
+func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) {
+	err := st.ApplyBlock(nil, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func buildAppStateFromChain(proxyApp proxy.AppConns,
+	state *sm.State, chain []*types.Block, nBlocks int, mode uint) {
+	// start a new app without handshake, play nBlocks blocks
+	if _, err := proxyApp.Start(); err != nil {
+		panic(err)
+	}
+
+	validators := types.TM2PB.Validators(state.Validators)
+	proxyApp.Consensus().InitChainSync(validators)
+
+	defer proxyApp.Stop()
+	switch mode {
+	case 0:
+		for i := 0; i < nBlocks; i++ {
+			block := chain[i]
+			applyBlock(state, block, proxyApp)
+		}
+	case 1, 2:
+		for i := 0; i < nBlocks-1; i++ {
+			block := chain[i]
+			applyBlock(state, block, proxyApp)
+		}
+
+		if mode == 2 {
+			// update the dummy height and apphash
+			// as if we ran commit but not
+			applyBlock(state, chain[nBlocks-1], proxyApp)
+		}
+	}
+
+}
+
+func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.Block, mode uint) []byte {
+	// run the whole chain against this client to build up the tendermint state
+	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "1")))
+	proxyApp := proxy.NewAppConns(clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
+	if _, err := proxyApp.Start(); err != nil {
+		panic(err)
+	}
+	defer proxyApp.Stop()
+
+	validators := types.TM2PB.Validators(state.Validators)
+	proxyApp.Consensus().InitChainSync(validators)
+
+	var latestAppHash []byte
+
+	switch mode {
+	case 0:
+		// sync right up
+		for _, block := range chain {
+			applyBlock(state, block, proxyApp)
+		}
+
+		latestAppHash = state.AppHash
+	case 1, 2:
+		// sync up to the penultimate as if we stored the block.
+		// whether we commit or not depends on the appHash
+		for _, block := range chain[:len(chain)-1] {
+			applyBlock(state, block, proxyApp)
+		}
+
+		// apply the final block to a state copy so we can
+		// get the right next appHash but keep the state back
+		stateCopy := state.Copy()
+		applyBlock(stateCopy, chain[len(chain)-1], proxyApp)
+		latestAppHash = stateCopy.AppHash
+	}
+
+	return latestAppHash
+}
+
+//--------------------------
+// utils for making blocks
+
+func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
+	// Search for height marker
+	gr, found, err := wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(0))
+	if err != nil {
+		return nil, nil, err
+	}
+	if !found {
+		return nil, nil, errors.New(cmn.Fmt("WAL does not contain height %d.", 1))
+	}
+	defer gr.Close()
+
+	// log.Notice("Build a blockchain by reading from the WAL")
+
+	var blockParts *types.PartSet
+	var blocks []*types.Block
+	var commits []*types.Commit
+	for {
+		line, err := gr.ReadLine()
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return nil, nil, err
+			}
+		}
+
+		piece, err := readPieceFromWAL([]byte(line))
+		if err != nil {
+			return nil, nil, err
+		}
+		if piece == nil {
+			continue
+		}
+
+		switch p := piece.(type) {
+		case *types.PartSetHeader:
+			// if its not the first one, we have a full block
+			if blockParts != nil {
+				var n int
+				block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
+				blocks = append(blocks, block)
+			}
+			blockParts = types.NewPartSetFromHeader(*p)
+		case *types.Part:
+			_, err := blockParts.AddPart(p, false)
+			if err != nil {
+				return nil, nil, err
+			}
+		case *types.Vote:
+			if p.Type == types.VoteTypePrecommit {
+				commit := &types.Commit{
+					BlockID:    p.BlockID,
+					Precommits: []*types.Vote{p},
+				}
+				commits = append(commits, commit)
+			}
+		}
+	}
+	// grab the last block too
+	var n int
+	block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
+	blocks = append(blocks, block)
+	return blocks, commits, nil
+}
+
+func readPieceFromWAL(msgBytes []byte) (interface{}, error) {
+	// Skip over empty and meta lines
+	if len(msgBytes) == 0 || msgBytes[0] == '#' {
+		return nil, nil
+	}
+	var err error
+	var msg TimedWALMessage
+	wire.ReadJSON(&msg, msgBytes, &err)
+	if err != nil {
+		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
+		return nil, fmt.Errorf("Error reading json data: %v", err)
+	}
+
+	// for logging
+	switch m := msg.Msg.(type) {
+	case msgInfo:
+		switch msg := m.Msg.(type) {
+		case *ProposalMessage:
+			return &msg.Proposal.BlockPartsHeader, nil
+		case *BlockPartMessage:
+			return msg.Part, nil
+		case *VoteMessage:
+			return msg.Vote, nil
+		}
+	}
+	return nil, nil
+}
+
+// fresh state and mock store
+func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) {
+	stateDB := dbm.NewMemDB()
+	state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile())
+	state.SetLogger(log.TestingLogger().With("module", "state"))
+
+	store := NewMockBlockStore(config)
+	return state, store
+}
+
+//----------------------------------
+// mock block store
+
+type mockBlockStore struct {
+	config  *cfg.Config
+	chain   []*types.Block
+	commits []*types.Commit
+}
+
+// TODO: NewBlockStore(db.NewMemDB) ...
+func NewMockBlockStore(config *cfg.Config) *mockBlockStore {
+	return &mockBlockStore{config, nil, nil}
+}
+
+func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
+func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
+func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
+	block := bs.chain[height-1]
+	return &types.BlockMeta{
+		BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.Consensus.BlockPartSize).Header()},
+		Header:  block.Header,
+	}
+}
+func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
+func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+}
+func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
+	return bs.commits[height-1]
+}
+func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
+	return bs.commits[height-1]
 }

consensus/state.go

@@ -4,72 +4,23 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
+	"path"
 	"reflect"
 	"sync"
 	"time"
 
-	"github.com/ebuchman/fail-test"
-
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-wire"
-	bc "github.com/tendermint/tendermint/blockchain"
-	mempl "github.com/tendermint/tendermint/mempool"
+	fail "github.com/ebuchman/fail-test"
+	wire "github.com/tendermint/go-wire"
+	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 )
 
 //-----------------------------------------------------------------------------
-// Timeout Parameters
-
-// TimeoutParams holds timeouts and deltas for each round step.
-// All timeouts and deltas in milliseconds.
-type TimeoutParams struct {
-	Propose0          int
-	ProposeDelta      int
-	Prevote0          int
-	PrevoteDelta      int
-	Precommit0        int
-	PrecommitDelta    int
-	Commit0           int
-	SkipTimeoutCommit bool
-}
-
-// Wait this long for a proposal
-func (tp *TimeoutParams) Propose(round int) time.Duration {
-	return time.Duration(tp.Propose0+tp.ProposeDelta*round) * time.Millisecond
-}
-
-// After receiving any +2/3 prevote, wait this long for stragglers
-func (tp *TimeoutParams) Prevote(round int) time.Duration {
-	return time.Duration(tp.Prevote0+tp.PrevoteDelta*round) * time.Millisecond
-}
-
-// After receiving any +2/3 precommits, wait this long for stragglers
-func (tp *TimeoutParams) Precommit(round int) time.Duration {
-	return time.Duration(tp.Precommit0+tp.PrecommitDelta*round) * time.Millisecond
-}
-
-// After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight
-func (tp *TimeoutParams) Commit(t time.Time) time.Time {
-	return t.Add(time.Duration(tp.Commit0) * time.Millisecond)
-}
-
-// InitTimeoutParamsFromConfig initializes parameters from config
-func InitTimeoutParamsFromConfig(config cfg.Config) *TimeoutParams {
-	return &TimeoutParams{
-		Propose0:          config.GetInt("timeout_propose"),
-		ProposeDelta:      config.GetInt("timeout_propose_delta"),
-		Prevote0:          config.GetInt("timeout_prevote"),
-		PrevoteDelta:      config.GetInt("timeout_prevote_delta"),
-		Precommit0:        config.GetInt("timeout_precommit"),
-		PrecommitDelta:    config.GetInt("timeout_precommit_delta"),
-		Commit0:           config.GetInt("timeout_commit"),
-		SkipTimeoutCommit: config.GetBool("skip_timeout_commit"),
-	}
-}
+// Config
 
 //-----------------------------------------------------------------------------
 // Errors
@@ -124,6 +75,8 @@ func (rs RoundStepType) String() string {
 //-----------------------------------------------------------------------------
 
 // Immutable when returned from ConsensusState.GetRoundState()
+// TODO: Actually, only the top pointer is copied,
+// so access to field pointers is still racey
 type RoundState struct {
 	Height             int // Height we are working on
 	Round              int
@@ -222,40 +175,50 @@ type PrivValidator interface {
 
 // Tracks consensus state across block heights and rounds.
 type ConsensusState struct {
-	BaseService
-
-	config       cfg.Config
-	proxyAppConn proxy.AppConnConsensus
-	blockStore   *bc.BlockStore
-	mempool      *mempl.Mempool
+	cmn.BaseService
 
+	// config details
+	config        *cfg.ConsensusConfig
 	privValidator PrivValidator // for signing votes
 
+	// services for creating and executing blocks
+	proxyAppConn proxy.AppConnConsensus
+	blockStore   types.BlockStore
+	mempool      types.Mempool
+
+	// internal state
 	mtx sync.Mutex
 	RoundState
 	state *sm.State // State until height-1.
 
-	peerMsgQueue     chan msgInfo   // serializes msgs affecting state (proposals, block parts, votes)
-	internalMsgQueue chan msgInfo   // like peerMsgQueue but for our own proposals, parts, votes
-	timeoutTicker    TimeoutTicker  // ticker for timeouts
-	timeoutParams    *TimeoutParams // parameters and functions for timeout intervals
+	// state changes may be triggered by msgs from peers,
+	// msgs from ourself, or by timeouts
+	peerMsgQueue     chan msgInfo
+	internalMsgQueue chan msgInfo
+	timeoutTicker    TimeoutTicker
 
+	// we use PubSub to trigger msg broadcasts in the reactor,
+	// and to notify external subscribers, eg. through a websocket
 	evsw types.EventSwitch
 
+	// a Write-Ahead Log ensures we can recover from any kind of crash
+	// and helps us avoid signing conflicting votes
 	wal        *WAL
 	replayMode bool // so we don't log signing errors during replay
 
-	nSteps int // used for testing to limit the number of transitions the state makes
+	// for tests where we want to limit the number of transitions the state makes
+	nSteps int
 
-	// allow certain function to be overwritten for testing
+	// some functions can be overwritten for testing
 	decideProposal func(height, round int)
 	doPrevote      func(height, round int)
 	setProposal    func(proposal *types.Proposal) error
 
+	// closed when we finish shutting down
 	done chan struct{}
 }
 
-func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState {
+func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
 		proxyAppConn:     proxyAppConn,
@@ -264,7 +227,6 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap
 		peerMsgQueue:     make(chan msgInfo, msgQueueSize),
 		internalMsgQueue: make(chan msgInfo, msgQueueSize),
 		timeoutTicker:    NewTimeoutTicker(),
-		timeoutParams:    InitTimeoutParamsFromConfig(config),
 		done:             make(chan struct{}),
 	}
 	// set function defaults (may be overwritten before calling Start)
@@ -276,21 +238,27 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap
 	// Don't call scheduleRound0 yet.
 	// We do that upon Start().
 	cs.reconstructLastCommit(state)
-	cs.BaseService = *NewBaseService(log, "ConsensusState", cs)
+	cs.BaseService = *cmn.NewBaseService(nil, "ConsensusState", cs)
 	return cs
 }
 
 //----------------------------------------
 // Public interface
 
-// implements events.Eventable
+// SetLogger implements Service.
+func (cs *ConsensusState) SetLogger(l log.Logger) {
+	cs.BaseService.Logger = l
+	cs.timeoutTicker.SetLogger(l)
+}
+
+// SetEventSwitch implements events.Eventable
 func (cs *ConsensusState) SetEventSwitch(evsw types.EventSwitch) {
 	cs.evsw = evsw
 }
 
 func (cs *ConsensusState) String() string {
 	// better not to access shared variables
-	return Fmt("ConsensusState") //(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step)
+	return cmn.Fmt("ConsensusState") //(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step)
 }
 
 func (cs *ConsensusState) GetState() *sm.State {
@@ -340,36 +308,12 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit {
 }
 
 func (cs *ConsensusState) OnStart() error {
-	cs.BaseService.OnStart()
 
-	walDir := cs.config.GetString("cs_wal_dir")
-	err := EnsureDir(walDir, 0700)
-	if err != nil {
-		log.Error("Error ensuring ConsensusState wal dir", "error", err.Error())
+	walFile := cs.config.WalFile()
+	if err := cs.OpenWAL(walFile); err != nil {
+		cs.Logger.Error("Error loading ConsensusState wal", "err", err.Error())
 		return err
 	}
-	err = cs.OpenWAL(walDir)
-	if err != nil {
-		log.Error("Error loading ConsensusState wal", "error", err.Error())
-		return err
-	}
-
-	// If the latest block was applied in the abci handshake,
-	// we may not have written the current height to the wal,
-	// so check here and write it if not found.
-	// TODO: remove this and run the handhsake/replay
-	// through the consensus state with a mock app
-	gr, found, err := cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(cs.Height))
-	if (err == io.EOF || !found) && cs.Step == RoundStepNewHeight {
-		log.Warn("Height not found in wal. Writing new height", "height", cs.Height)
-		rs := cs.RoundStateEvent()
-		cs.wal.Save(rs)
-	} else if err != nil {
-		return err
-	}
-	if gr != nil {
-		gr.Close()
-	}
 
 	// we need the timeoutRoutine for replay so
 	//  we don't block on the tick chan.
@@ -381,8 +325,9 @@ func (cs *ConsensusState) OnStart() error {
 	// we may have lost some votes if the process crashed
 	// reload from consensus log to catchup
 	if err := cs.catchupReplay(cs.Height); err != nil {
-		log.Error("Error on catchup replay", "error", err.Error())
-		// let's go for it anyways, maybe we're fine
+		cs.Logger.Error("Error on catchup replay. Proceeding to start ConsensusState anyway", "err", err.Error())
+		// NOTE: if we ever do return an error here,
+		// make sure to stop the timeoutTicker
 	}
 
 	// now start the receiveRoutine
@@ -420,13 +365,23 @@ func (cs *ConsensusState) Wait() {
 }
 
 // Open file to log all consensus messages and timeouts for deterministic accountability
-func (cs *ConsensusState) OpenWAL(walDir string) (err error) {
+func (cs *ConsensusState) OpenWAL(walFile string) (err error) {
+	err = cmn.EnsureDir(path.Dir(walFile), 0700)
+	if err != nil {
+		cs.Logger.Error("Error ensuring ConsensusState wal dir", "err", err.Error())
+		return err
+	}
+
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	wal, err := NewWAL(walDir, cs.config.GetBool("cs_wal_light"))
+	wal, err := NewWAL(walFile, cs.config.WalLight)
 	if err != nil {
 		return err
 	}
+	wal.SetLogger(cs.Logger.With("wal", walFile))
+	if _, err := wal.Start(); err != nil {
+		return err
+	}
 	cs.wal = wal
 	return nil
 }
@@ -498,7 +453,7 @@ func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) {
 
 // enterNewRound(height, 0) at cs.StartTime.
 func (cs *ConsensusState) scheduleRound0(rs *RoundState) {
-	//log.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
+	//cs.Logger.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
 	sleepDuration := rs.StartTime.Sub(time.Now())
 	cs.scheduleTimeout(sleepDuration, rs.Height, 0, RoundStepNewHeight)
 }
@@ -517,7 +472,7 @@ func (cs *ConsensusState) sendInternalMessage(mi msgInfo) {
 		// be processed out of order.
 		// TODO: use CList here for strict determinism and
 		// attempt push to internalMsgQueue in receiveRoutine
-		log.Warn("Internal msg queue is full. Using a go-routine")
+		cs.Logger.Info("Internal msg queue is full. Using a go-routine")
 		go func() { cs.internalMsgQueue <- mi }()
 	}
 }
@@ -529,18 +484,18 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
 		return
 	}
 	seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight)
-	lastPrecommits := types.NewVoteSet(cs.config.GetString("chain_id"), state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators)
+	lastPrecommits := types.NewVoteSet(cs.state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators)
 	for _, precommit := range seenCommit.Precommits {
 		if precommit == nil {
 			continue
 		}
 		added, err := lastPrecommits.AddVote(precommit)
 		if !added || err != nil {
-			PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err))
+			cmn.PanicCrisis(cmn.Fmt("Failed to reconstruct LastCommit: %v", err))
 		}
 	}
 	if !lastPrecommits.HasTwoThirdsMajority() {
-		PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
+		cmn.PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
 	}
 	cs.LastCommit = lastPrecommits
 }
@@ -549,13 +504,13 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
 // The round becomes 0 and cs.Step becomes RoundStepNewHeight.
 func (cs *ConsensusState) updateToState(state *sm.State) {
 	if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
-		PanicSanity(Fmt("updateToState() expected state height of %v but found %v",
+		cmn.PanicSanity(cmn.Fmt("updateToState() expected state height of %v but found %v",
 			cs.Height, state.LastBlockHeight))
 	}
 	if cs.state != nil && cs.state.LastBlockHeight+1 != cs.Height {
 		// This might happen when someone else is mutating cs.state.
 		// Someone forgot to pass in state.Copy() somewhere?!
-		PanicSanity(Fmt("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
+		cmn.PanicSanity(cmn.Fmt("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
 			cs.state.LastBlockHeight+1, cs.Height))
 	}
 
@@ -563,21 +518,23 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 	// This happens when SwitchToConsensus() is called in the reactor.
 	// We don't want to reset e.g. the Votes.
 	if cs.state != nil && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
-		log.Notice("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
+		cs.Logger.Info("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
 		return
 	}
 
 	// Reset fields based on state.
 	validators := state.Validators
-	height := state.LastBlockHeight + 1 // Next desired block height
 	lastPrecommits := (*types.VoteSet)(nil)
 	if cs.CommitRound > -1 && cs.Votes != nil {
 		if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
-			PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
+			cmn.PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
 		}
 		lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
 	}
 
+	// Next desired block height
+	height := state.LastBlockHeight + 1
+
 	// RoundState fields
 	cs.updateHeight(height)
 	cs.updateRoundStep(0, RoundStepNewHeight)
@@ -587,9 +544,9 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 		// to be gathered for the first block.
 		// And alternative solution that relies on clocks:
 		//  cs.StartTime = state.LastBlockTime.Add(timeoutCommit)
-		cs.StartTime = cs.timeoutParams.Commit(time.Now())
+		cs.StartTime = cs.config.Commit(time.Now())
 	} else {
-		cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime)
+		cs.StartTime = cs.config.Commit(cs.CommitTime)
 	}
 	cs.Validators = validators
 	cs.Proposal = nil
@@ -598,7 +555,7 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 	cs.LockedRound = 0
 	cs.LockedBlock = nil
 	cs.LockedBlockParts = nil
-	cs.Votes = NewHeightVoteSet(cs.config.GetString("chain_id"), height, validators)
+	cs.Votes = NewHeightVoteSet(state.ChainID, height, validators)
 	cs.CommitRound = -1
 	cs.LastCommit = lastPrecommits
 	cs.LastValidators = state.LastValidators
@@ -622,11 +579,6 @@ func (cs *ConsensusState) newStep() {
 //-----------------------------------------
 // the main go routines
 
-// a nice idea but probably more trouble than its worth
-func (cs *ConsensusState) stopTimer() {
-	cs.timeoutTicker.Stop()
-}
-
 // receiveRoutine handles messages which may cause state transitions.
 // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
 // It keeps the RoundState and is the only thing that updates it.
@@ -635,7 +587,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) {
 	for {
 		if maxSteps > 0 {
 			if cs.nSteps >= maxSteps {
-				log.Warn("reached max steps. exiting receive routine")
+				cs.Logger.Info("reached max steps. exiting receive routine")
 				cs.nSteps = 0
 				return
 			}
@@ -708,19 +660,19 @@ func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) {
 		// the peer is sending us CatchupCommit precommits.
 		// We could make note of this and help filter in broadcastHasVoteMessage().
 	default:
-		log.Warn("Unknown msg type", reflect.TypeOf(msg))
+		cs.Logger.Error("Unknown msg type", reflect.TypeOf(msg))
 	}
 	if err != nil {
-		log.Error("Error with msg", "type", reflect.TypeOf(msg), "peer", peerKey, "error", err, "msg", msg)
+		cs.Logger.Error("Error with msg", "type", reflect.TypeOf(msg), "peer", peerKey, "err", err, "msg", msg)
 	}
 }
 
 func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
-	log.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+	cs.Logger.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
 
 	// timeouts must be for current height, round, step
 	if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) {
-		log.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
+		cs.Logger.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
 		return
 	}
 
@@ -743,7 +695,7 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
 		types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
 		cs.enterNewRound(ti.Height, ti.Round+1)
 	default:
-		panic(Fmt("Invalid timeout step: %v", ti.Step))
+		panic(cmn.Fmt("Invalid timeout step: %v", ti.Step))
 	}
 
 }
@@ -758,16 +710,15 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
 // NOTE: cs.StartTime was already set for height.
 func (cs *ConsensusState) enterNewRound(height int, round int) {
 	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != RoundStepNewHeight) {
-		log.Debug(Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 
 	if now := time.Now(); cs.StartTime.After(now) {
-		log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now)
+		cs.Logger.Info("Need to set a buffer and log message here for sanity.", "startTime", cs.StartTime, "now", now)
 	}
-	// cs.stopTimer()
 
-	log.Notice(Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	// Increment validators if necessary
 	validators := cs.Validators
@@ -801,10 +752,10 @@ func (cs *ConsensusState) enterNewRound(height int, round int) {
 // Enter: from NewRound(height,round).
 func (cs *ConsensusState) enterPropose(height int, round int) {
 	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
-		log.Debug(Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
-	log.Info(Fmt("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
 		// Done enterPropose:
@@ -820,19 +771,25 @@ func (cs *ConsensusState) enterPropose(height int, round int) {
 	}()
 
 	// If we don't get the proposal and all block parts quick enough, enterPrevote
-	cs.scheduleTimeout(cs.timeoutParams.Propose(round), height, round, RoundStepPropose)
+	cs.scheduleTimeout(cs.config.Propose(round), height, round, RoundStepPropose)
 
 	// Nothing more to do if we're not a validator
 	if cs.privValidator == nil {
+		cs.Logger.Debug("This node is not a validator")
 		return
 	}
 
-	if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.GetAddress()) {
-		log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
+	if !bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress()) {
+		cs.Logger.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator)
+		if cs.Validators.HasAddress(cs.privValidator.GetAddress()) {
+			cs.Logger.Debug("This node is a validator")
+		} else {
+			cs.Logger.Debug("This node is not a validator")
+		}
 	} else {
-		log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
+		cs.Logger.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator)
+		cs.Logger.Debug("This node is a validator")
 		cs.decideProposal(height, round)
-
 	}
 }
 
@@ -870,11 +827,11 @@ func (cs *ConsensusState) defaultDecideProposal(height, round int) {
 			part := blockParts.GetPart(i)
 			cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""})
 		}
-		log.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
-		log.Debug(Fmt("Signed proposal block: %v", block))
+		cs.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
+		cs.Logger.Debug(cmn.Fmt("Signed proposal block: %v", block))
 	} else {
 		if !cs.replayMode {
-			log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err)
+			cs.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err)
 		}
 	}
 }
@@ -909,15 +866,15 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 		commit = cs.LastCommit.MakeCommit()
 	} else {
 		// This shouldn't happen.
-		log.Error("enterPropose: Cannot propose anything: No commit for the previous block.")
+		cs.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block.")
 		return
 	}
 
 	// Mempool validated transactions
-	txs := cs.mempool.Reap(cs.config.GetInt("block_size"))
+	txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs)
 
 	return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit,
-		cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.GetInt("block_part_size"))
+		cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.BlockPartSize)
 }
 
 // Enter: `timeoutPropose` after entering Propose.
@@ -927,7 +884,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 // Otherwise vote nil.
 func (cs *ConsensusState) enterPrevote(height int, round int) {
 	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
-		log.Debug(Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 
@@ -945,9 +902,7 @@ func (cs *ConsensusState) enterPrevote(height int, round int) {
 		// TODO: catchup event?
 	}
 
-	// cs.stopTimer()
-
-	log.Info(Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	// Sign and broadcast vote as necessary
 	cs.doPrevote(height, round)
@@ -959,14 +914,14 @@ func (cs *ConsensusState) enterPrevote(height int, round int) {
 func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
 	// If a block is locked, prevote that.
 	if cs.LockedBlock != nil {
-		log.Notice("enterPrevote: Block was locked")
+		cs.Logger.Info("enterPrevote: Block was locked")
 		cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header())
 		return
 	}
 
 	// If ProposalBlock is nil, prevote nil.
 	if cs.ProposalBlock == nil {
-		log.Warn("enterPrevote: ProposalBlock is nil")
+		cs.Logger.Info("enterPrevote: ProposalBlock is nil")
 		cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
 		return
 	}
@@ -975,7 +930,7 @@ func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
 	err := cs.state.ValidateBlock(cs.ProposalBlock)
 	if err != nil {
 		// ProposalBlock is invalid, prevote nil.
-		log.Warn("enterPrevote: ProposalBlock is invalid", "error", err)
+		cs.Logger.Error("enterPrevote: ProposalBlock is invalid", "err", err)
 		cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
 		return
 	}
@@ -984,19 +939,18 @@ func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
 	// NOTE: the proposal signature is validated when it is received,
 	// and the proposal block parts are validated as they are received (against the merkle hash in the proposal)
 	cs.signAddVote(types.VoteTypePrevote, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header())
-	return
 }
 
 // Enter: any +2/3 prevotes at next round.
 func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
 	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
-		log.Debug(Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 	if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
-		PanicSanity(Fmt("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
+		cmn.PanicSanity(cmn.Fmt("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
 	}
-	log.Info(Fmt("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
 		// Done enterPrevoteWait:
@@ -1005,7 +959,7 @@ func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
 	}()
 
 	// Wait for some more prevotes; enterPrecommit
-	cs.scheduleTimeout(cs.timeoutParams.Prevote(round), height, round, RoundStepPrevoteWait)
+	cs.scheduleTimeout(cs.config.Prevote(round), height, round, RoundStepPrevoteWait)
 }
 
 // Enter: +2/3 precomits for block or nil.
@@ -1016,13 +970,11 @@ func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
 // else, precommit nil otherwise.
 func (cs *ConsensusState) enterPrecommit(height int, round int) {
 	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
-		log.Debug(Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 
-	// cs.stopTimer()
-
-	log.Info(Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
 		// Done enterPrecommit:
@@ -1035,9 +987,9 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 	// If we don't have a polka, we must precommit nil
 	if !ok {
 		if cs.LockedBlock != nil {
-			log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
+			cs.Logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
 		} else {
-			log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
+			cs.Logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
 		}
 		cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
 		return
@@ -1049,15 +1001,15 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 	// the latest POLRound should be this round
 	polRound, _ := cs.Votes.POLInfo()
 	if polRound < round {
-		PanicSanity(Fmt("This POLRound should be %v but got %", round, polRound))
+		cmn.PanicSanity(cmn.Fmt("This POLRound should be %v but got %", round, polRound))
 	}
 
 	// +2/3 prevoted nil. Unlock and precommit nil.
 	if len(blockID.Hash) == 0 {
 		if cs.LockedBlock == nil {
-			log.Notice("enterPrecommit: +2/3 prevoted for nil.")
+			cs.Logger.Info("enterPrecommit: +2/3 prevoted for nil.")
 		} else {
-			log.Notice("enterPrecommit: +2/3 prevoted for nil. Unlocking")
+			cs.Logger.Info("enterPrecommit: +2/3 prevoted for nil. Unlocking")
 			cs.LockedRound = 0
 			cs.LockedBlock = nil
 			cs.LockedBlockParts = nil
@@ -1071,7 +1023,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 
 	// If we're already locked on that block, precommit it, and update the LockedRound
 	if cs.LockedBlock.HashesTo(blockID.Hash) {
-		log.Notice("enterPrecommit: +2/3 prevoted locked block. Relocking")
+		cs.Logger.Info("enterPrecommit: +2/3 prevoted locked block. Relocking")
 		cs.LockedRound = round
 		types.FireEventRelock(cs.evsw, cs.RoundStateEvent())
 		cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
@@ -1080,10 +1032,10 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 
 	// If +2/3 prevoted for proposal block, stage and precommit it
 	if cs.ProposalBlock.HashesTo(blockID.Hash) {
-		log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash)
+		cs.Logger.Info("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash)
 		// Validate the block.
 		if err := cs.state.ValidateBlock(cs.ProposalBlock); err != nil {
-			PanicConsensus(Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
+			cmn.PanicConsensus(cmn.Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
 		}
 		cs.LockedRound = round
 		cs.LockedBlock = cs.ProposalBlock
@@ -1106,19 +1058,18 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 	}
 	types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
 	cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
-	return
 }
 
 // Enter: any +2/3 precommits for next round.
 func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
 	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
-		log.Debug(Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 	if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
-		PanicSanity(Fmt("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
+		cmn.PanicSanity(cmn.Fmt("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
 	}
-	log.Info(Fmt("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
 		// Done enterPrecommitWait:
@@ -1127,17 +1078,17 @@ func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
 	}()
 
 	// Wait for some more precommits; enterNewRound
-	cs.scheduleTimeout(cs.timeoutParams.Precommit(round), height, round, RoundStepPrecommitWait)
+	cs.scheduleTimeout(cs.config.Precommit(round), height, round, RoundStepPrecommitWait)
 
 }
 
 // Enter: +2/3 precommits for block
 func (cs *ConsensusState) enterCommit(height int, commitRound int) {
 	if cs.Height != height || RoundStepCommit <= cs.Step {
-		log.Debug(Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
 		return
 	}
-	log.Info(Fmt("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
+	cs.Logger.Info(cmn.Fmt("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
 		// Done enterCommit:
@@ -1153,7 +1104,7 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) {
 
 	blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
 	if !ok {
-		PanicSanity("RunActionCommit() expects +2/3 precommits")
+		cmn.PanicSanity("RunActionCommit() expects +2/3 precommits")
 	}
 
 	// The Locked* fields no longer matter.
@@ -1180,20 +1131,21 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) {
 // If we have the block AND +2/3 commits for it, finalize.
 func (cs *ConsensusState) tryFinalizeCommit(height int) {
 	if cs.Height != height {
-		PanicSanity(Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
+		cmn.PanicSanity(cmn.Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
 	}
 
 	blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
 	if !ok || len(blockID.Hash) == 0 {
-		log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
+		cs.Logger.Error("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.", "height", height)
 		return
 	}
 	if !cs.ProposalBlock.HashesTo(blockID.Hash) {
 		// TODO: this happens every time if we're not a validator (ugly logs)
 		// TODO: ^^ wait, why does it matter that we're a validator?
-		log.Warn("Attempt to finalize failed. We don't have the commit block.")
+		cs.Logger.Error("Attempt to finalize failed. We don't have the commit block.", "height", height, "proposal-block", cs.ProposalBlock.Hash(), "commit-block", blockID.Hash)
 		return
 	}
+
 	//	go
 	cs.finalizeCommit(height)
 }
@@ -1201,7 +1153,7 @@ func (cs *ConsensusState) tryFinalizeCommit(height int) {
 // Increment height and goto RoundStepNewHeight
 func (cs *ConsensusState) finalizeCommit(height int) {
 	if cs.Height != height || cs.Step != RoundStepCommit {
-		log.Debug(Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
+		cs.Logger.Debug(cmn.Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
 		return
 	}
 
@@ -1209,21 +1161,21 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
 
 	if !ok {
-		PanicSanity(Fmt("Cannot finalizeCommit, commit does not have two thirds majority"))
+		cmn.PanicSanity(cmn.Fmt("Cannot finalizeCommit, commit does not have two thirds majority"))
 	}
 	if !blockParts.HasHeader(blockID.PartsHeader) {
-		PanicSanity(Fmt("Expected ProposalBlockParts header to be commit header"))
+		cmn.PanicSanity(cmn.Fmt("Expected ProposalBlockParts header to be commit header"))
 	}
 	if !block.HashesTo(blockID.Hash) {
-		PanicSanity(Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
+		cmn.PanicSanity(cmn.Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
 	}
 	if err := cs.state.ValidateBlock(block); err != nil {
-		PanicConsensus(Fmt("+2/3 committed an invalid block: %v", err))
+		cmn.PanicConsensus(cmn.Fmt("+2/3 committed an invalid block: %v", err))
 	}
 
-	log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs),
+	cs.Logger.Info(cmn.Fmt("Finalizing commit of block with %d txs", block.NumTxs),
 		"height", block.Height, "hash", block.Hash(), "root", block.AppHash)
-	log.Info(Fmt("%v", block))
+	cs.Logger.Info(cmn.Fmt("%v", block))
 
 	fail.Fail() // XXX
 
@@ -1235,7 +1187,21 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 		seenCommit := precommits.MakeCommit()
 		cs.blockStore.SaveBlock(block, blockParts, seenCommit)
 	} else {
-		log.Warn("Why are we finalizeCommitting a block height we already have?", "height", block.Height)
+		// Happens during replay if we already saved the block but didn't commit
+		cs.Logger.Info("Calling finalizeCommit on already stored block", "height", block.Height)
+	}
+
+	fail.Fail() // XXX
+
+	// Finish writing to the WAL for this height.
+	// NOTE: If we fail before writing this, we'll never write it,
+	// and just recover by running ApplyBlock in the Handshake.
+	// If we moved it before persisting the block, we'd have to allow
+	// WAL replay for blocks with an #ENDHEIGHT
+	// As is, ConsensusState should not be started again
+	// until we successfully call ApplyBlock (ie. here or in Handshake after restart)
+	if cs.wal != nil {
+		cs.wal.writeEndHeight(height)
 	}
 
 	fail.Fail() // XXX
@@ -1245,30 +1211,35 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	stateCopy := cs.state.Copy()
 	eventCache := types.NewEventCache(cs.evsw)
 
-	// Execute and commit the block, and update the mempool.
-	// All calls to the proxyAppConn should come here.
+	// Execute and commit the block, update and save the state, and update the mempool.
+	// All calls to the proxyAppConn come here.
 	// NOTE: the block.AppHash wont reflect these txs until the next block
 	err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
 	if err != nil {
-		// TODO!
+		cs.Logger.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "err", err)
+		return
 	}
 
 	fail.Fail() // XXX
 
-	// Fire off event for new block.
-	// TODO: Handle app failure.  See #177
+	// Fire event for new block.
+	// NOTE: If we fail before firing, these events will never fire
+	//
+	// TODO: Either
+	//	* Fire before persisting state, in ApplyBlock
+	//	* Fire on start up if we haven't written any new WAL msgs
+	//   Both options mean we may fire more than once. Is that fine ?
 	types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block})
 	types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header})
 	eventCache.Flush()
 
-	// Save the state.
-	stateCopy.Save()
-
 	fail.Fail() // XXX
 
 	// NewHeightStep!
 	cs.updateToState(stateCopy)
 
+	fail.Fail() // XXX
+
 	// cs.StartTime is already set.
 	// Schedule Round0 to start soon.
 	cs.scheduleRound0(&cs.RoundState)
@@ -1277,7 +1248,6 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	// * cs.Height has been increment to height+1
 	// * cs.Step is now RoundStepNewHeight
 	// * cs.StartTime is set to when we will start round0.
-	return
 }
 
 //-----------------------------------------------------------------------------
@@ -1306,7 +1276,7 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
 	}
 
 	// Verify signature
-	if !cs.Validators.Proposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
+	if !cs.Validators.GetProposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
 		return ErrInvalidProposalSignature
 	}
 
@@ -1338,7 +1308,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, ver
 		var err error
 		cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
 		// NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
-		log.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
+		cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
 		if cs.Step == RoundStepPropose && cs.isProposalComplete() {
 			// Move onto the next step
 			cs.enterPrevote(height, cs.Round)
@@ -1362,10 +1332,10 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error {
 			return err
 		} else if _, ok := err.(*types.ErrVoteConflictingVotes); ok {
 			if peerKey == "" {
-				log.Warn("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
+				cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
 				return err
 			}
-			log.Warn("Found conflicting vote. Publish evidence (TODO)")
+			cs.Logger.Error("Found conflicting vote. Publish evidence (TODO)")
 			/* TODO
 			evidenceTx := &types.DupeoutTx{
 				Address: address,
@@ -1377,7 +1347,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error {
 			return err
 		} else {
 			// Probably an invalid signature. Bad peer.
-			log.Warn("Error attempting to add vote", "error", err)
+			cs.Logger.Error("Error attempting to add vote", "err", err)
 			return ErrAddingVote
 		}
 	}
@@ -1387,7 +1357,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error {
 //-----------------------------------------------------------------------------
 
 func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, err error) {
-	log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height)
+	cs.Logger.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "valIndex", vote.ValidatorIndex, "csHeight", cs.Height)
 
 	// A precommit for the previous height?
 	// These come in while we wait timeoutCommit
@@ -1399,11 +1369,11 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 		}
 		added, err = cs.LastCommit.AddVote(vote)
 		if added {
-			log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
+			cs.Logger.Info(cmn.Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
 			types.FireEventVote(cs.evsw, types.EventDataVote{vote})
 
 			// if we can skip timeoutCommit and have all the votes now,
-			if cs.timeoutParams.SkipTimeoutCommit && cs.LastCommit.HasAll() {
+			if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() {
 				// go straight to new round (skip timeout commit)
 				// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
 				cs.enterNewRound(cs.Height, 0)
@@ -1423,7 +1393,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 			switch vote.Type {
 			case types.VoteTypePrevote:
 				prevotes := cs.Votes.Prevotes(vote.Round)
-				log.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
+				cs.Logger.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
 				// First, unlock if prevotes is a valid POL.
 				// >> lockRound < POLRound <= unlockOrChangeLockRound (see spec)
 				// NOTE: If (lockRound < POLRound) but !(POLRound <= unlockOrChangeLockRound),
@@ -1432,7 +1402,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 				if (cs.LockedBlock != nil) && (cs.LockedRound < vote.Round) && (vote.Round <= cs.Round) {
 					blockID, ok := prevotes.TwoThirdsMajority()
 					if ok && !cs.LockedBlock.HashesTo(blockID.Hash) {
-						log.Notice("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
+						cs.Logger.Info("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
 						cs.LockedRound = 0
 						cs.LockedBlock = nil
 						cs.LockedBlockParts = nil
@@ -1456,7 +1426,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 				}
 			case types.VoteTypePrecommit:
 				precommits := cs.Votes.Precommits(vote.Round)
-				log.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
+				cs.Logger.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
 				blockID, ok := precommits.TwoThirdsMajority()
 				if ok {
 					if len(blockID.Hash) == 0 {
@@ -1466,7 +1436,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 						cs.enterPrecommit(height, vote.Round)
 						cs.enterCommit(height, vote.Round)
 
-						if cs.timeoutParams.SkipTimeoutCommit && precommits.HasAll() {
+						if cs.config.SkipTimeoutCommit && precommits.HasAll() {
 							// if we have all the votes now,
 							// go straight to new round (skip timeout commit)
 							// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
@@ -1480,7 +1450,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 					cs.enterPrecommitWait(height, vote.Round)
 				}
 			default:
-				PanicSanity(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen.
+				cmn.PanicSanity(cmn.Fmt("Unexpected vote type %X", vote.Type)) // Should not happen.
 			}
 		}
 		// Either duplicate, or error upon cs.Votes.AddByIndex()
@@ -1490,7 +1460,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 	}
 
 	// Height mismatch, bad peer?
-	log.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "err", err)
+	cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "err", err)
 	return
 }
 
@@ -1518,11 +1488,11 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part
 	vote, err := cs.signVote(type_, hash, header)
 	if err == nil {
 		cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""})
-		log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
+		cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err)
 		return vote
 	} else {
 		//if !cs.replayMode {
-		log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
+		cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err)
 		//}
 		return nil
 	}

consensus/state_test.go

@@ -6,17 +6,16 @@ import (
 	"testing"
 	"time"
 
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/tendermint/config/tendermint_test"
 	"github.com/tendermint/tendermint/types"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 func init() {
-	config = tendermint_test.ResetConfig("consensus_state_test")
+	config = ResetConfig("consensus_state_test")
 }
 
-func (tp *TimeoutParams) ensureProposeTimeout() time.Duration {
-	return time.Duration(tp.Propose0*2) * time.Millisecond
+func ensureProposeTimeout(timeoutPropose int) time.Duration {
+	return time.Duration(timeoutPropose*2) * time.Millisecond
 }
 
 /*
@@ -65,7 +64,7 @@ func TestProposerSelection0(t *testing.T) {
 	<-newRoundCh
 
 	// lets commit a block and ensure proposer for the next height is correct
-	prop := cs1.GetRoundState().Validators.Proposer()
+	prop := cs1.GetRoundState().Validators.GetProposer()
 	if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) {
 		t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address)
 	}
@@ -79,7 +78,7 @@ func TestProposerSelection0(t *testing.T) {
 	// wait for new round so next validator is set
 	<-newRoundCh
 
-	prop = cs1.GetRoundState().Validators.Proposer()
+	prop = cs1.GetRoundState().Validators.GetProposer()
 	if !bytes.Equal(prop.Address, vss[1].Address) {
 		panic(Fmt("expected proposer to be validator %d. Got %X", 1, prop.Address))
 	}
@@ -100,7 +99,7 @@ func TestProposerSelection2(t *testing.T) {
 
 	// everyone just votes nil. we get a new proposer each round
 	for i := 0; i < len(vss); i++ {
-		prop := cs1.GetRoundState().Validators.Proposer()
+		prop := cs1.GetRoundState().Validators.GetProposer()
 		if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].Address) {
 			panic(Fmt("expected proposer to be validator %d. Got %X", (i+2)%len(vss), prop.Address))
 		}
@@ -126,7 +125,7 @@ func TestEnterProposeNoPrivValidator(t *testing.T) {
 	startTestRound(cs, height, round)
 
 	// if we're not a validator, EnterPropose should timeout
-	ticker := time.NewTicker(cs.timeoutParams.ensureProposeTimeout())
+	ticker := time.NewTicker(ensureProposeTimeout(cs.config.TimeoutPropose))
 	select {
 	case <-timeoutCh:
 	case <-ticker.C:
@@ -167,7 +166,7 @@ func TestEnterProposeYesPrivValidator(t *testing.T) {
 	}
 
 	// if we're a validator, enterPropose should not timeout
-	ticker := time.NewTicker(cs.timeoutParams.ensureProposeTimeout())
+	ticker := time.NewTicker(ensureProposeTimeout(cs.config.TimeoutPropose))
 	select {
 	case <-timeoutCh:
 		panic("Expected EnterPropose not to timeout")
@@ -181,7 +180,7 @@ func TestBadProposal(t *testing.T) {
 	height, round := cs1.Height, cs1.Round
 	vs2 := vss[1]
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
 	voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1)
@@ -201,7 +200,7 @@ func TestBadProposal(t *testing.T) {
 	propBlock.AppHash = stateHash
 	propBlockParts := propBlock.MakePartSet(partSize)
 	proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{})
-	if err := vs2.SignProposal(config.GetString("chain_id"), proposal); err != nil {
+	if err := vs2.SignProposal(config.ChainID, proposal); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
 
@@ -248,7 +247,7 @@ func TestFullRound1(t *testing.T) {
 
 	// grab proposal
 	re := <-propCh
-	propBlockHash := re.(types.EventDataRoundState).RoundState.(*RoundState).ProposalBlock.Hash()
+	propBlockHash := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState).ProposalBlock.Hash()
 
 	<-voteCh // wait for prevote
 	// NOTE: voteChan cap of 0 ensures we can complete this
@@ -328,7 +327,7 @@ func TestLockNoPOL(t *testing.T) {
 	vs2 := vss[1]
 	height := cs1.Height
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
 	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
@@ -345,7 +344,7 @@ func TestLockNoPOL(t *testing.T) {
 	cs1.startRoutines(0)
 
 	re := <-proposalCh
-	rs := re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -376,7 +375,7 @@ func TestLockNoPOL(t *testing.T) {
 	///
 
 	<-newRoundCh
-	log.Notice("#### ONTO ROUND 1")
+	t.Log("#### ONTO ROUND 1")
 	/*
 		Round2 (cs1, B) // B B2
 	*/
@@ -385,7 +384,7 @@ func TestLockNoPOL(t *testing.T) {
 
 	// now we're on a new round and not the proposer, so wait for timeout
 	re = <-timeoutProposeCh
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 
 	if rs.ProposalBlock != nil {
 		panic("Expected proposal block to be nil")
@@ -421,7 +420,7 @@ func TestLockNoPOL(t *testing.T) {
 	<-timeoutWaitCh
 
 	<-newRoundCh
-	log.Notice("#### ONTO ROUND 2")
+	t.Log("#### ONTO ROUND 2")
 	/*
 		Round3 (vs2, _) // B, B2
 	*/
@@ -429,7 +428,7 @@ func TestLockNoPOL(t *testing.T) {
 	incrementRound(vs2)
 
 	re = <-proposalCh
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 
 	// now we're on a new round and are the proposer
 	if !bytes.Equal(rs.ProposalBlock.Hash(), rs.LockedBlock.Hash()) {
@@ -462,7 +461,7 @@ func TestLockNoPOL(t *testing.T) {
 	incrementRound(vs2)
 
 	<-newRoundCh
-	log.Notice("#### ONTO ROUND 3")
+	t.Log("#### ONTO ROUND 3")
 	/*
 		Round4 (vs2, C) // B C // B C
 	*/
@@ -494,7 +493,7 @@ func TestLockPOLRelock(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
 	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
@@ -503,7 +502,7 @@ func TestLockPOLRelock(t *testing.T) {
 	newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1)
 	newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlockHeader(), 1)
 
-	log.Debug("vs2 last round", "lr", vs2.PrivValidator.LastRound)
+	t.Logf("vs2 last round %v", vs2.PrivValidator.LastRound)
 
 	// everything done from perspective of cs1
 
@@ -518,13 +517,14 @@ func TestLockPOLRelock(t *testing.T) {
 
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
 
 	signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2, vs3, vs4)
-	_, _, _ = <-voteCh, <-voteCh, <-voteCh // prevotes
+	// prevotes
+	discardFromChan(voteCh, 3)
 
 	<-voteCh // our precommit
 	// the proposed block should now be locked and our precommit added
@@ -533,7 +533,8 @@ func TestLockPOLRelock(t *testing.T) {
 	// add precommits from the rest
 	signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4)
 	signAddVotes(cs1, types.VoteTypePrecommit, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs3)
-	_, _, _ = <-voteCh, <-voteCh, <-voteCh // precommits
+	// precommites
+	discardFromChan(voteCh, 3)
 
 	// before we timeout to the new round set the new proposal
 	prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1)
@@ -545,11 +546,11 @@ func TestLockPOLRelock(t *testing.T) {
 	// timeout to new round
 	<-timeoutWaitCh
 
-	//XXX: this isnt gauranteed to get there before the timeoutPropose ...
+	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
 	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
 
 	<-newRoundCh
-	log.Notice("### ONTO ROUND 1")
+	t.Log("### ONTO ROUND 1")
 
 	/*
 		Round2 (vs2, C) // B C C C // C C C _)
@@ -571,7 +572,8 @@ func TestLockPOLRelock(t *testing.T) {
 
 	// now lets add prevotes from everyone else for the new block
 	signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4)
-	_, _, _ = <-voteCh, <-voteCh, <-voteCh // prevotes
+	// prevotes
+	discardFromChan(voteCh, 3)
 
 	// now either we go to PrevoteWait or Precommit
 	select {
@@ -586,12 +588,12 @@ func TestLockPOLRelock(t *testing.T) {
 	validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash, propBlockHash)
 
 	signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash, propBlockParts.Header(), vs2, vs3)
-	_, _ = <-voteCh, <-voteCh
+	discardFromChan(voteCh, 2)
 
 	be := <-newBlockCh
-	b := be.(types.EventDataNewBlockHeader)
+	b := be.(types.TMEventData).Unwrap().(types.EventDataNewBlockHeader)
 	re = <-newRoundCh
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	if rs.Height != 2 {
 		panic("Expected height to increment")
 	}
@@ -606,7 +608,7 @@ func TestLockPOLUnlock(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
 	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
@@ -627,7 +629,7 @@ func TestLockPOLUnlock(t *testing.T) {
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	theBlockHash := rs.ProposalBlock.Hash()
 
 	<-voteCh // prevote
@@ -653,14 +655,14 @@ func TestLockPOLUnlock(t *testing.T) {
 
 	// timeout to new round
 	re = <-timeoutWaitCh
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	lockedBlockHash := rs.LockedBlock.Hash()
 
-	//XXX: this isnt gauranteed to get there before the timeoutPropose ...
+	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
 	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
 
 	<-newRoundCh
-	log.Notice("#### ONTO ROUND 1")
+	t.Log("#### ONTO ROUND 1")
 	/*
 		Round2 (vs2, C) // B nil nil nil // nil nil nil _
 
@@ -701,7 +703,7 @@ func TestLockPOLSafety1(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
 	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
@@ -713,7 +715,7 @@ func TestLockPOLSafety1(t *testing.T) {
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	propBlock := rs.ProposalBlock
 
 	<-voteCh // prevote
@@ -732,7 +734,7 @@ func TestLockPOLSafety1(t *testing.T) {
 			panic("failed to update validator")
 		}*/
 
-	log.Warn("old prop", "hash", fmt.Sprintf("%X", propBlock.Hash()))
+	t.Logf("old prop hash %v", fmt.Sprintf("%X", propBlock.Hash()))
 
 	// we do see them precommit nil
 	signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3, vs4)
@@ -743,11 +745,11 @@ func TestLockPOLSafety1(t *testing.T) {
 
 	incrementRound(vs2, vs3, vs4)
 
-	//XXX: this isnt gauranteed to get there before the timeoutPropose ...
+	//XXX: this isnt guaranteed to get there before the timeoutPropose ...
 	cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer")
 
 	<-newRoundCh
-	log.Notice("### ONTO ROUND 1")
+	t.Log("### ONTO ROUND 1")
 	/*Round2
 	// we timeout and prevote our lock
 	// a polka happened but we didn't see it!
@@ -761,12 +763,12 @@ func TestLockPOLSafety1(t *testing.T) {
 		re = <-proposalCh
 	}
 
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 
 	if rs.LockedBlock != nil {
 		panic("we should not be locked!")
 	}
-	log.Warn("new prop", "hash", fmt.Sprintf("%X", propBlockHash))
+	t.Logf("new prop hash %v", fmt.Sprintf("%X", propBlockHash))
 	// go to prevote, prevote for proposal block
 	<-voteCh
 	validatePrevote(t, cs1, 1, vss[0], propBlockHash)
@@ -787,7 +789,7 @@ func TestLockPOLSafety1(t *testing.T) {
 
 	<-newRoundCh
 
-	log.Notice("### ONTO ROUND 2")
+	t.Log("### ONTO ROUND 2")
 	/*Round3
 	we see the polka from round 1 but we shouldn't unlock!
 	*/
@@ -806,7 +808,7 @@ func TestLockPOLSafety1(t *testing.T) {
 	// add prevotes from the earlier round
 	addVotes(cs1, prevotes...)
 
-	log.Warn("Done adding prevotes!")
+	t.Log("Done adding prevotes!")
 
 	ensureNoNewStep(newStepCh)
 }
@@ -822,7 +824,7 @@ func TestLockPOLSafety2(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
 	timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1)
@@ -850,7 +852,7 @@ func TestLockPOLSafety2(t *testing.T) {
 
 	cs1.updateRoundStep(0, RoundStepPrecommitWait)
 
-	log.Notice("### ONTO Round 1")
+	t.Log("### ONTO Round 1")
 	// jump in at round 1
 	height := cs1.Height
 	startTestRound(cs1, height, 1)
@@ -878,7 +880,7 @@ func TestLockPOLSafety2(t *testing.T) {
 
 	// in round 2 we see the polkad block from round 0
 	newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0, propBlockID1)
-	if err := vs3.SignProposal(config.GetString("chain_id"), newProp); err != nil {
+	if err := vs3.SignProposal(config.ChainID, newProp); err != nil {
 		t.Fatal(err)
 	}
 	cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer")
@@ -887,7 +889,7 @@ func TestLockPOLSafety2(t *testing.T) {
 	addVotes(cs1, prevotes...)
 
 	<-newRoundCh
-	log.Notice("### ONTO Round 2")
+	t.Log("### ONTO Round 2")
 	/*Round2
 	// now we see the polka from round 1, but we shouldnt unlock
 	*/
@@ -997,7 +999,7 @@ func TestHalt1(t *testing.T) {
 	cs1, vss := randConsensusState(4)
 	vs2, vs3, vs4 := vss[1], vss[2], vss[3]
 
-	partSize := config.GetInt("block_part_size")
+	partSize := config.Consensus.BlockPartSize
 
 	proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1)
 	timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1)
@@ -1009,7 +1011,7 @@ func TestHalt1(t *testing.T) {
 	startTestRound(cs1, cs1.Height, 0)
 	<-newRoundCh
 	re := <-proposalCh
-	rs := re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs := re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 	propBlock := rs.ProposalBlock
 	propBlockParts := propBlock.MakePartSet(partSize)
 
@@ -1032,9 +1034,9 @@ func TestHalt1(t *testing.T) {
 	// timeout to new round
 	<-timeoutWaitCh
 	re = <-newRoundCh
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 
-	log.Notice("### ONTO ROUND 1")
+	t.Log("### ONTO ROUND 1")
 	/*Round2
 	// we timeout and prevote our lock
 	// a polka happened but we didn't see it!
@@ -1050,7 +1052,7 @@ func TestHalt1(t *testing.T) {
 	// receiving that precommit should take us straight to commit
 	<-newBlockCh
 	re = <-newRoundCh
-	rs = re.(types.EventDataRoundState).RoundState.(*RoundState)
+	rs = re.(types.TMEventData).Unwrap().(types.EventDataRoundState).RoundState.(*RoundState)
 
 	if rs.Height != 2 {
 		panic("expected height to increment")

consensus/test_data/build.sh

@@ -1,58 +1,110 @@
-#! /bin/bash
+#!/usr/bin/env bash
 
-cd $GOPATH/src/github.com/tendermint/tendermint
+# XXX: removes tendermint dir
+
+cd "$GOPATH/src/github.com/tendermint/tendermint" || exit 1
+
+# Make sure we have a tendermint command.
+if ! hash tendermint 2>/dev/null; then
+	make install
+fi
 
 # specify a dir to copy
-# NOTE: eventually we should replace with `tendermint init --test`
-DIR=$HOME/.tendermint_test/consensus_state_test
+# TODO: eventually we should replace with `tendermint init --test`
+DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test
 
-# XXX: remove tendermint dir
-rm -rf $HOME/.tendermint
-cp -r $DIR $HOME/.tendermint
+TMHOME="$HOME/.tendermint"
+rm -rf "$TMHOME"
+cp -r "$DIR_TO_COPY" "$TMHOME"
+cp $TMHOME/config.toml $TMHOME/config.toml.bak
 
 function reset(){
-	rm -rf $HOME/.tendermint/data
-	tendermint unsafe_reset_priv_validator
+	tendermint unsafe_reset_all
+	cp $TMHOME/config.toml.bak $TMHOME/config.toml
 }
 
 reset
 
 # empty block
-tendermint node --proxy_app=dummy &> /dev/null &
+function empty_block(){
+tendermint node --proxy_app=persistent_dummy &> /dev/null &
 sleep 5
 killall tendermint
 
-# /q would print up to and including the match, then quit. 
-# /Q doesn't include the match. 
+# /q would print up to and including the match, then quit.
+# /Q doesn't include the match.
 # http://unix.stackexchange.com/questions/11305/grep-show-all-the-file-up-to-the-match
-sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/empty_block.cswal
+sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/empty_block.cswal
 
 reset
+}
 
-# small block 1
+# many blocks
+function many_blocks(){
 bash scripts/txs/random.sh 1000 36657 &> /dev/null &
 PID=$!
-tendermint node --proxy_app=dummy &> /dev/null &
-sleep 5
+tendermint node --proxy_app=persistent_dummy &> /dev/null &
+sleep 7
 killall tendermint
 kill -9 $PID
 
-sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block1.cswal
+sed '/ENDHEIGHT: 6/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/many_blocks.cswal
 
 reset
+}
+
+
+# small block 1
+function small_block1(){
+bash scripts/txs/random.sh 1000 36657 &> /dev/null &
+PID=$!
+tendermint node --proxy_app=persistent_dummy &> /dev/null &
+sleep 10
+killall tendermint
+kill -9 $PID
+
+sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block1.cswal
+
+reset
+}
 
 
 # small block 2 (part size = 512)
+function small_block2(){
 echo "" >> ~/.tendermint/config.toml
 echo "block_part_size = 512" >> ~/.tendermint/config.toml
 bash scripts/txs/random.sh 1000 36657 &> /dev/null &
 PID=$!
-tendermint node --proxy_app=dummy &> /dev/null &
+tendermint node --proxy_app=persistent_dummy &> /dev/null &
 sleep 5
 killall tendermint
 kill -9 $PID
 
-sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block2.cswal
+sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block2.cswal
 
 reset
+}
+
+
+
+case "$1" in
+	"small_block1")
+		small_block1
+		;;
+	"small_block2")
+		small_block2
+		;;
+	"empty_block")
+		empty_block
+		;;
+	"many_blocks")
+		many_blocks
+		;;
+	*)
+		small_block1
+		small_block2
+		empty_block
+		many_blocks
+esac
+
 

consensus/test_data/empty_block.cswal

@@ -1,10 +1,10 @@
-#HEIGHT: 1
-{"time":"2016-12-18T05:05:33.502Z","msg":[3,{"duration":974084551,"height":1,"round":0,"step":1}]}
-{"time":"2016-12-18T05:05:33.505Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
-{"time":"2016-12-18T05:05:33.505Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"62C0F2BCCB491399EEDAF8E85837ADDD4E25BAB7A84BFC4F0E88594531FBC6D4755DEC7E6427F04AD7EB8BB89502762AB4380C7BBA93A4C297E6180EC78E3504"]}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:33.506Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114914148D83E0DC00000000000000114354594CBFC1A7BCA1AD0050ED6AA010023EADA390001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:33.508Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
-{"time":"2016-12-18T05:05:33.508Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"3E83DF89A01C5F104912E095F32451C202F34717","parts":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"}},"signature":[1,"B64D0BB64B2E9AAFDD4EBEA679644F77AE774D69E3E2E1B042AB15FE4F84B1427AC6C8A25AFF58EA22011AE567FEA49D2EE7354382E915AD85BF40C58FA6130C"]}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:33.509Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
-{"time":"2016-12-18T05:05:33.509Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"3E83DF89A01C5F104912E095F32451C202F34717","parts":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"}},"signature":[1,"D83E968392D1BF09821E0D05079DAB5491CABD89BE128BD1CF573ED87148BA84667A56C0A069EFC90760F25EDAC62BC324DBB12EA63F44E6CB2D3500FE5E640F"]}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:33.509Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}
+#ENDHEIGHT: 0
+{"time":"2017-04-27T22:24:01.346Z","msg":[3,{"duration":972946821,"height":1,"round":0,"step":1}]}
+{"time":"2017-04-27T22:24:01.349Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-04-27T22:24:01.349Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"E785764AED6D92D7CC65C0A3A4ED9C8465198A05142C3E6C7F3EF601FDCD3A604900B77B7B87C046221EF99FD038A960398385BD5BBAA50EE4F86DE757B8F704"]}}],"peer_key":""}]}
+{"time":"2017-04-27T22:24:01.350Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114B96165CF4496C00000000000000114354594CBFC1A7BCA1AD0050ED6AA010023EADA390001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:24:01.351Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-04-27T22:24:01.351Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"F3BBFBE7E4A5D619E2C498C3D1B912883786DD71","parts":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"}},"signature":[1,"35C937C78D061ECDC3770982A1330C9AA7F6FEF00835C43DEB50B8FCF69A3EEF221E675EE5E469114F64E4FBBABA414EB9170E1025FC47D3F0EADE46767D2E00"]}}],"peer_key":""}]}
+{"time":"2017-04-27T22:24:01.352Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-04-27T22:24:01.352Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"F3BBFBE7E4A5D619E2C498C3D1B912883786DD71","parts":{"total":1,"hash":"ACED4A95DDEBD24E66A681F7EAB4CA22C4B8546D"}},"signature":[1,"D1A7D27FCD5D352F3A3EDA8DE368520BC5B796662E32BCD8D91CDB8209A88DAF37CB7C4C93143D3C12B37C1435229268098CFFD0AD1400D88DA7606454692301"]}}],"peer_key":""}]}
+{"time":"2017-04-27T22:24:01.352Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}

consensus/test_data/small_block2.cswal

@@ -1,14 +1,15 @@
-#HEIGHT: 1
-{"time":"2016-12-18T05:05:43.641Z","msg":[3,{"duration":969409681,"height":1,"round":0,"step":1}]}
-{"time":"2016-12-18T05:05:43.643Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
-{"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"F1A8E9928889C68FD393F3983B5362AECA4A95AA13FE3C78569B2515EC046893CB718071CAF54F3F1507DCD851B37CD5557EA17BB5471D2DC6FB5AC5FBB72E02"]}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F7465737401011491414B3483A8400190000000000114926EA77D30A4D19866159DE7E58AA9461F90F9D10114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010190010D6162636431323D646362613132010D6162636431333D646362613133010D6162636431343D646362613134010D6162636431353D646362613135010D6162636431363D646362613136010D6162636431373D646362613137010D6162636431383D646362613138010D6162636431393D646362613139010D6162636432303D646362613230010D6162636432313D646362613231010D6162636432323D646362613232010D6162636432333D646362613233010D6162636432343D646362613234010D6162636432353D646362613235010D6162636432363D646362613236010D6162636432373D646362613237010D6162636432383D646362613238010D6162636432393D646362613239010D6162636433303D646362613330010D6162636433313D646362613331010D6162636433323D646362613332010D6162636433333D646362613333010D6162636433343D646362613334010D6162636433353D646362613335010D6162636433363D646362613336010D6162636433373D646362613337010D6162636433383D646362613338010D6162636433393D646362613339010D6162636434303D","proof":{"aunts":["C9FBD66B63A976638196323F5B93494BDDFC9EED","47FD83BB7607E679EE5CF0783372D13C5A264056","FEEC97078A26B7F6057821C0660855170CC6F1D7"]}}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"646362613430010D6162636434313D646362613431010D6162636434323D646362613432010D6162636434333D646362613433010D6162636434343D646362613434010D6162636434353D646362613435010D6162636434363D646362613436010D6162636434373D646362613437010D6162636434383D646362613438010D6162636434393D646362613439010D6162636435303D646362613530010D6162636435313D646362613531010D6162636435323D646362613532010D6162636435333D646362613533010D6162636435343D646362613534010D6162636435353D646362613535010D6162636435363D646362613536010D6162636435373D646362613537010D6162636435383D646362613538010D6162636435393D646362613539010D6162636436303D646362613630010D6162636436313D646362613631010D6162636436323D646362613632010D6162636436333D646362613633010D6162636436343D646362613634010D6162636436353D646362613635010D6162636436363D646362613636010D6162636436373D646362613637010D6162636436383D646362613638010D6162636436393D646362613639010D6162636437303D646362613730010D6162636437313D646362613731010D6162636437323D646362613732010D6162636437333D646362613733010D6162636437343D6463","proof":{"aunts":["D7FB03B935B77C322064F8277823CDB5C7018597","47FD83BB7607E679EE5CF0783372D13C5A264056","FEEC97078A26B7F6057821C0660855170CC6F1D7"]}}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.644Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"62613734010D6162636437353D646362613735010D6162636437363D646362613736010D6162636437373D646362613737010D6162636437383D646362613738010D6162636437393D646362613739010D6162636438303D646362613830010D6162636438313D646362613831010D6162636438323D646362613832010D6162636438333D646362613833010D6162636438343D646362613834010D6162636438353D646362613835010D6162636438363D646362613836010D6162636438373D646362613837010D6162636438383D646362613838010D6162636438393D646362613839010D6162636439303D646362613930010D6162636439313D646362613931010D6162636439323D646362613932010D6162636439333D646362613933010D6162636439343D646362613934010D6162636439353D646362613935010D6162636439363D646362613936010D6162636439373D646362613937010D6162636439383D646362613938010D6162636439393D646362613939010F616263643130303D64636261313030010F616263643130313D64636261313031010F616263643130323D64636261313032010F616263643130333D64636261313033010F616263643130343D64636261313034010F616263643130353D64636261313035010F616263643130363D64636261313036010F616263643130373D64636261","proof":{"aunts":["A607D9BF5107E6C9FD19B6928D9CC7714B0730E4","FEEC97078A26B7F6057821C0660855170CC6F1D7"]}}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.644Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"313037010F616263643130383D64636261313038010F616263643130393D64636261313039010F616263643131303D64636261313130010F616263643131313D64636261313131010F616263643131323D64636261313132010F616263643131333D64636261313133010F616263643131343D64636261313134010F616263643131353D64636261313135010F616263643131363D64636261313136010F616263643131373D64636261313137010F616263643131383D64636261313138010F616263643131393D64636261313139010F616263643132303D64636261313230010F616263643132313D64636261313231010F616263643132323D64636261313232010F616263643132333D64636261313233010F616263643132343D64636261313234010F616263643132353D64636261313235010F616263643132363D64636261313236010F616263643132373D64636261313237010F616263643132383D64636261313238010F616263643132393D64636261313239010F616263643133303D64636261313330010F616263643133313D64636261313331010F616263643133323D64636261313332010F616263643133333D64636261313333010F616263643133343D64636261313334010F616263643133353D64636261313335010F616263643133363D64636261313336010F616263643133373D646362613133","proof":{"aunts":["0FD794B3506B9E92CDE3703F7189D42167E77095","86D455F542DA79F5A764B9DABDEABF01F4BAB2AB"]}}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.644Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"37010F616263643133383D64636261313338010F616263643133393D64636261313339010F616263643134303D64636261313430010F616263643134313D64636261313431010F616263643134323D64636261313432010F616263643134333D64636261313433010F616263643134343D64636261313434010F616263643134353D64636261313435010F616263643134363D64636261313436010F616263643134373D64636261313437010F616263643134383D64636261313438010F616263643134393D64636261313439010F616263643135303D64636261313530010F616263643135313D64636261313531010F616263643135323D64636261313532010F616263643135333D64636261313533010F616263643135343D64636261313534010F616263643135353D646362613135350100000000","proof":{"aunts":["50CBDC078A660EAE3442BA355BE10EE0D04408D1","86D455F542DA79F5A764B9DABDEABF01F4BAB2AB"]}}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.645Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
-{"time":"2016-12-18T05:05:43.645Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"6ADACDC2871C59A67337DAFD5045A982ED070C51","parts":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"}},"signature":[1,"E815E0A63B7EEE7894DE2D72372A7C393434AC8ACCC46B60C628910F73351806D55A59994F08B454BFD71EDAA0CA95733CA47E37FFDAF9AAA2431A8160176E01"]}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.647Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
-{"time":"2016-12-18T05:05:43.647Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"6ADACDC2871C59A67337DAFD5045A982ED070C51","parts":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"}},"signature":[1,"9AAC3F3A118EE039EB460E9E5308D490D671C7490309BD5D62B5F392205C7E420DFDAF90F08294FF36BE8A9AA5CC203C1F2088B42D2BB8EE40A45F2BB5C54D0A"]}}],"peer_key":""}]}
-{"time":"2016-12-18T05:05:43.648Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}
+#ENDHEIGHT: 0
+{"time":"2017-04-27T22:23:56.310Z","msg":[3,{"duration":969732098,"height":1,"round":0,"step":1}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"7624F6E943B7A207E16D1FA87EA099BD924E930F98E7DECBC01DB37735C619409588A67C2EABA9845FD6B80FDB65ECFCDA5F0DEFCEF74B8C34DB8E0540480203"]}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114B96164A30A118001620000000001141F6753D22BACA2180B1EADD722434EB28444D91D0114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010162011A3631363236333634333133303344363436333632363133313330011A3631363236333634333133313344363436333632363133313331011A3631363236333634333133323344363436333632363133313332011A3631363236333634333133333344363436333632363133313333011A3631363236333634333133343344363436333632363133313334011A3631363236333634333133353344363436333632363133313335011A3631363236333634333133363344363436333632363133313336011A3631363236333634333133373344363436333632363133313337011A3631363236333634333133383344363436333632363133313338011A3631363236333634333133393344363436333632363133313339011A3631363236333634333233303344363436333632363133323330011A3631363236333634333233313344363436333632363133323331011A3631363236333634333233323344363436333632363133323332011A3631363236333634333233333344363436333632363133323333011A3631363236333634333233343344363436333632363133323334011A36313632363336","proof":{"aunts":["49F4B71E3D7C457415069E2EA916DB12F67AA8D0","D35A72BEDAAAAC17045D7BFAAFA94C2EC0B0A4C2","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"34333233353344363436333632363133323335011A3631363236333634333233363344363436333632363133323336011A3631363236333634333233373344363436333632363133323337011A3631363236333634333233383344363436333632363133323338011A3631363236333634333233393344363436333632363133323339011A3631363236333634333333303344363436333632363133333330011A3631363236333634333333313344363436333632363133333331011A3631363236333634333333323344363436333632363133333332011A3631363236333634333333333344363436333632363133333333011A3631363236333634333333343344363436333632363133333334011A3631363236333634333333353344363436333632363133333335011A3631363236333634333333363344363436333632363133333336011A3631363236333634333333373344363436333632363133333337011A3631363236333634333333383344363436333632363133333338011A3631363236333634333333393344363436333632363133333339011A3631363236333634333433303344363436333632363133343330011A3631363236333634333433313344363436333632363133343331011A3631363236333634333433323344363436333632363133343332011A363136323633363433343333334436","proof":{"aunts":["5AD2A9A1A49A1FD6EF83F05FA4588F800B29DEF1","D35A72BEDAAAAC17045D7BFAAFA94C2EC0B0A4C2","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"3436333632363133343333011A3631363236333634333433343344363436333632363133343334011A3631363236333634333433353344363436333632363133343335011A3631363236333634333433363344363436333632363133343336011A3631363236333634333433373344363436333632363133343337011A3631363236333634333433383344363436333632363133343338011A3631363236333634333433393344363436333632363133343339011A3631363236333634333533303344363436333632363133353330011A3631363236333634333533313344363436333632363133353331011A3631363236333634333533323344363436333632363133353332011A3631363236333634333533333344363436333632363133353333011A3631363236333634333533343344363436333632363133353334011A3631363236333634333533353344363436333632363133353335011A3631363236333634333533363344363436333632363133353336011A3631363236333634333533373344363436333632363133353337011A3631363236333634333533383344363436333632363133353338011A3631363236333634333533393344363436333632363133353339011A3631363236333634333633303344363436333632363133363330011A3631363236333634333633313344363436333632363133","proof":{"aunts":["8B5786C3D871EE37B0F4B2DECAC39E157340DFBE","705BC647374F3495EE73C3F44C21E9BDB4731738"]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"363331011A3631363236333634333633323344363436333632363133363332011A3631363236333634333633333344363436333632363133363333011A3631363236333634333633343344363436333632363133363334011A3631363236333634333633353344363436333632363133363335011A3631363236333634333633363344363436333632363133363336011A3631363236333634333633373344363436333632363133363337011A3631363236333634333633383344363436333632363133363338011A3631363236333634333633393344363436333632363133363339011A3631363236333634333733303344363436333632363133373330011A3631363236333634333733313344363436333632363133373331011A3631363236333634333733323344363436333632363133373332011A3631363236333634333733333344363436333632363133373333011A3631363236333634333733343344363436333632363133373334011A3631363236333634333733353344363436333632363133373335011A3631363236333634333733363344363436333632363133373336011A3631363236333634333733373344363436333632363133373337011A3631363236333634333733383344363436333632363133373338011A3631363236333634333733393344363436333632363133373339011A363136","proof":{"aunts":["56097661A1B2707588100586B3B1C2C8A51057D1","6DE889147DF528EEB5F7422E95DC45900CAFB619","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.312Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"3236333634333833303344363436333632363133383330011A3631363236333634333833313344363436333632363133383331011A3631363236333634333833323344363436333632363133383332011A3631363236333634333833333344363436333632363133383333011A3631363236333634333833343344363436333632363133383334011A3631363236333634333833353344363436333632363133383335011A3631363236333634333833363344363436333632363133383336011A3631363236333634333833373344363436333632363133383337011A3631363236333634333833383344363436333632363133383338011A3631363236333634333833393344363436333632363133383339011A3631363236333634333933303344363436333632363133393330011A3631363236333634333933313344363436333632363133393331011A3631363236333634333933323344363436333632363133393332011A3631363236333634333933333344363436333632363133393333011A3631363236333634333933343344363436333632363133393334011A3631363236333634333933353344363436333632363133393335011A3631363236333634333933363344363436333632363133393336011A3631363236333634333933373344363436333632363133393337011A3631363236333634333933","proof":{"aunts":["081D3DC5F11850851D5F0D760B98EE87BFA6B8B0","6DE889147DF528EEB5F7422E95DC45900CAFB619","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.313Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":5,"bytes":"383344363436333632363133393338011A3631363236333634333933393344363436333632363133393339011E363136323633363433313330333033443634363336323631333133303330011E363136323633363433313330333133443634363336323631333133303331011E363136323633363433313330333233443634363336323631333133303332011E363136323633363433313330333333443634363336323631333133303333011E363136323633363433313330333433443634363336323631333133303334011E363136323633363433313330333533443634363336323631333133303335011E363136323633363433313330333633443634363336323631333133303336011E3631363236333634333133303337334436343633363236313331333033370100000000","proof":{"aunts":["6AA912328C2B52EFA0ECE71F523E137E400EC484","247C721D5CEB90BB1FE389BA74C43DF0955E1647"]}}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.314Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-04-27T22:23:56.314Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"62371CF72F8662378691706DB256C833CF1AF81B","parts":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"}},"signature":[1,"255906FAAA50C84E85DABF7DE73468E4F95DB4E46F598848145926E2FAD77CA682BF07E09E2F3EC81FFBD9A036B67914A3C02F819B69248D777AEBA792725907"]}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.315Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-04-27T22:23:56.315Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"62371CF72F8662378691706DB256C833CF1AF81B","parts":{"total":6,"hash":"A3C176F13F5CBC7C48EE27A472800410C9D487DC"}},"signature":[1,"056CC15C748434D0A59B64B45CB56EDC1A437A426E68FA63DC7D61A7C17B0F768F207D81340D129A57C5A64195F8AFDD03B6BF28D7B2286290D61BCE88FCA304"]}}],"peer_key":""}]}
+{"time":"2017-04-27T22:23:56.316Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}

consensus/ticker.go

@@ -3,7 +3,8 @@ package consensus
 import (
 	"time"
 
-	. "github.com/tendermint/go-common"
+	. "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 )
 
 var (
@@ -18,6 +19,8 @@ type TimeoutTicker interface {
 	Stop() bool
 	Chan() <-chan timeoutInfo       // on which to receive a timeout
 	ScheduleTimeout(ti timeoutInfo) // reset the timer
+
+	SetLogger(log.Logger)
 }
 
 // timeoutTicker wraps time.Timer,
@@ -39,13 +42,12 @@ func NewTimeoutTicker() TimeoutTicker {
 		tickChan: make(chan timeoutInfo, tickTockBufferSize),
 		tockChan: make(chan timeoutInfo, tickTockBufferSize),
 	}
+	tt.BaseService = *NewBaseService(nil, "TimeoutTicker", tt)
 	tt.stopTimer() // don't want to fire until the first scheduled timeout
-	tt.BaseService = *NewBaseService(log, "TimeoutTicker", tt)
 	return tt
 }
 
 func (t *timeoutTicker) OnStart() error {
-	t.BaseService.OnStart()
 
 	go t.timeoutRoutine()
 
@@ -76,7 +78,7 @@ func (t *timeoutTicker) stopTimer() {
 		select {
 		case <-t.timer.C:
 		default:
-			log.Debug("Timer already stopped")
+			t.Logger.Debug("Timer already stopped")
 		}
 	}
 }
@@ -85,12 +87,12 @@ func (t *timeoutTicker) stopTimer() {
 // timers are interupted and replaced by new ticks from later steps
 // timeouts of 0 on the tickChan will be immediately relayed to the tockChan
 func (t *timeoutTicker) timeoutRoutine() {
-	log.Debug("Starting timeout routine")
+	t.Logger.Debug("Starting timeout routine")
 	var ti timeoutInfo
 	for {
 		select {
 		case newti := <-t.tickChan:
-			log.Debug("Received tick", "old_ti", ti, "new_ti", newti)
+			t.Logger.Debug("Received tick", "old_ti", ti, "new_ti", newti)
 
 			// ignore tickers for old height/round/step
 			if newti.Height < ti.Height {
@@ -112,10 +114,10 @@ func (t *timeoutTicker) timeoutRoutine() {
 			// NOTE time.Timer allows duration to be non-positive
 			ti = newti
 			t.timer.Reset(ti.Duration)
-			log.Debug("Scheduled timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+			t.Logger.Debug("Scheduled timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
 		case <-t.timer.C:
-			log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
-			// go routine here gaurantees timeoutRoutine doesn't block.
+			t.Logger.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+			// go routine here guarantees timeoutRoutine doesn't block.
 			// Determinism comes from playback in the receiveRoutine.
 			// We can eliminate it by merging the timeoutRoutine into receiveRoutine
 			//  and managing the timeouts ourselves with a millisecond ticker

consensus/version.go

@@ -1,7 +1,7 @@
 package consensus
 
 import (
-	. "github.com/tendermint/go-common"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 // kind of arbitrary

consensus/wal.go

@@ -3,10 +3,10 @@ package consensus
 import (
 	"time"
 
-	auto "github.com/tendermint/go-autofile"
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-wire"
+	wire "github.com/tendermint/go-wire"
 	"github.com/tendermint/tendermint/types"
+	auto "github.com/tendermint/tmlibs/autofile"
+	. "github.com/tendermint/tmlibs/common"
 )
 
 //--------------------------------------------------------
@@ -40,8 +40,8 @@ type WAL struct {
 	light bool // ignore block parts
 }
 
-func NewWAL(walDir string, light bool) (*WAL, error) {
-	group, err := auto.OpenGroup(walDir + "/wal")
+func NewWAL(walFile string, light bool) (*WAL, error) {
+	group, err := auto.OpenGroup(walFile)
 	if err != nil {
 		return nil, err
 	}
@@ -49,18 +49,16 @@ func NewWAL(walDir string, light bool) (*WAL, error) {
 		group: group,
 		light: light,
 	}
-	wal.BaseService = *NewBaseService(log, "WAL", wal)
-	_, err = wal.Start()
-	return wal, err
+	wal.BaseService = *NewBaseService(nil, "WAL", wal)
+	return wal, nil
 }
 
 func (wal *WAL) OnStart() error {
-	wal.BaseService.OnStart()
 	size, err := wal.group.Head.Size()
 	if err != nil {
 		return err
 	} else if size == 0 {
-		wal.writeHeight(1)
+		wal.writeEndHeight(0)
 	}
 	_, err = wal.group.Start()
 	return err
@@ -84,12 +82,6 @@ func (wal *WAL) Save(wmsg WALMessage) {
 			}
 		}
 	}
-	// Write #HEIGHT: XYZ if new height
-	if edrs, ok := wmsg.(types.EventDataRoundState); ok {
-		if edrs.Step == RoundStepNewHeight.String() {
-			wal.writeHeight(edrs.Height)
-		}
-	}
 	// Write the wal message
 	var wmsgBytes = wire.JSONBytes(TimedWALMessage{time.Now(), wmsg})
 	err := wal.group.WriteLine(string(wmsgBytes))
@@ -102,8 +94,8 @@ func (wal *WAL) Save(wmsg WALMessage) {
 	}
 }
 
-func (wal *WAL) writeHeight(height int) {
-	wal.group.WriteLine(Fmt("#HEIGHT: %v", height))
+func (wal *WAL) writeEndHeight(height int) {
+	wal.group.WriteLine(Fmt("#ENDHEIGHT: %v", height))
 
 	// TODO: only flush when necessary
 	if err := wal.group.Flush(); err != nil {

docs/architecture/ABCI.md

@@ -0,0 +1,16 @@
+# ABCI
+
+ABCI is an interface between the consensus/blockchain engine known as tendermint, and the application-specific business logic, known as an ABCi app.
+
+The tendermint core should run unchanged for all apps.  Each app can customize it, the supported transactions, queries, even the validator sets and how to handle staking / slashing stake. This customization is achieved by implementing the ABCi app to send the proper information to the tendermint engine to perform as directed.
+
+To understand this decision better, think of the design of the tendermint engine.
+
+* A blockchain is simply consensus on a unique global ordering of events.
+* This consensus can efficiently be implemented using BFT and PoS
+* This code can be generalized to easily support a large number of blockchains
+* The block-chain specific code, the interpretation of the individual events, can be implemented by a 3rd party app without touching the consensus engine core
+* Use an efficient, language-agnostic layer to implement this (ABCi)
+
+
+Bucky, please make this doc real.

docs/architecture/README.md

@@ -0,0 +1,16 @@
+# Architecture Decision Records
+
+This is a location to record all high-level architecture decisions in the tendermint project.  Not the implementation details, but the reasoning that happened.  This should be refered to for guidance of the "right way" to extend the application.  And if we notice that the original decisions were lacking, we should have another open discussion, record the new decisions here, and then modify the code to match.
+
+This is like our guide and mentor when Jae and Bucky are offline.... The concept comes from a [blog post](https://product.reverb.com/documenting-architecture-decisions-the-reverb-way-a3563bb24bd0#.78xhdix6t) that resonated among the team when Anton shared it.
+
+Each section of the code can have it's own markdown file in this directory, and please add a link to the readme.
+
+## Sections
+
+* [ABCI](./ABCI.md)
+* [go-merkle / merkleeyes](./merkle.md)
+* [Frey's thoughts on the data store](./merkle-frey.md)
+* basecoin
+* tendermint core (multiple sections)
+* ???

docs/architecture/adr-001.md

@@ -0,0 +1,216 @@
+# ADR 1: Logging
+
+## Context
+
+Current logging system in Tendermint is very static and not flexible enough.
+
+Issues: [358](https://github.com/tendermint/tendermint/issues/358), [375](https://github.com/tendermint/tendermint/issues/375).
+
+What we want from the new system:
+
+- per package dynamic log levels
+- dynamic logger setting (logger tied to the processing struct)
+- conventions
+- be more visually appealing
+
+"dynamic" here means the ability to set smth in runtime.
+
+## Decision
+
+### 1) An interface
+
+First, we will need an interface for all of our libraries (`tmlibs`, Tendermint, etc.). My personal preference is go-kit `Logger` interface (see Appendix A.), but that is too much a bigger change. Plus we will still need levels.
+
+```go
+# log.go
+type Logger interface {
+    Debug(msg string, keyvals ...interface{}) error
+    Info(msg string, keyvals ...interface{}) error
+    Error(msg string, keyvals ...interface{}) error
+
+	  With(keyvals ...interface{}) Logger
+}
+```
+
+On a side note: difference between `Info` and `Notice` is subtle. We probably
+could do without `Notice`. Don't think we need `Panic` or `Fatal` as a part of
+the interface. These funcs could be implemented as helpers. In fact, we already
+have some in `tmlibs/common`.
+
+- `Debug` - extended output for devs
+- `Info` - all that is useful for a user
+- `Error` - errors
+
+`Notice` should become `Info`, `Warn` either `Error` or `Debug` depending on the message, `Crit` -> `Error`.
+
+This interface should go into `tmlibs/log`. All libraries which are part of the core (tendermint/tendermint) should obey it.
+
+### 2) Logger with our current formatting
+
+On top of this interface, we will need to implement a stdout logger, which will be used when Tendermint is configured to output logs to STDOUT.
+
+Many people say that they like the current output, so let's stick with it.
+
+```
+NOTE[04-25|14:45:08] ABCI Replay Blocks                       module=consensus appHeight=0 storeHeight=0 stateHeight=0
+```
+
+Couple of minor changes:
+
+```
+I[04-25|14:45:08.322] ABCI Replay Blocks            module=consensus appHeight=0 storeHeight=0 stateHeight=0
+```
+
+Notice the level is encoded using only one char plus milliseconds.
+
+Note: there are many other formats out there like [logfmt](https://brandur.org/logfmt).
+
+This logger could be implemented using any logger - [logrus](https://github.com/sirupsen/logrus), [go-kit/log](https://github.com/go-kit/kit/tree/master/log), [zap](https://github.com/uber-go/zap), log15 so far as it
+
+a) supports coloring output<br>
+b) is moderately fast (buffering) <br>
+c) conforms to the new interface or adapter could be written for it <br>
+d) is somewhat configurable<br>
+
+go-kit is my favorite so far. Check out how easy it is to color errors in red https://github.com/go-kit/kit/blob/master/log/term/example_test.go#L12. Although, coloring could only be applied to the whole string :(
+
+```
+go-kit +: flexible, modular
+go-kit “-”: logfmt format https://brandur.org/logfmt
+
+logrus +: popular, feature rich (hooks), API and output is more like what we want
+logrus -: not so flexible
+```
+
+```go
+# tm_logger.go
+// NewTmLogger returns a logger that encodes keyvals to the Writer in
+// tm format.
+func NewTmLogger(w io.Writer) Logger {
+  return &tmLogger{kitlog.NewLogfmtLogger(w)}
+}
+
+func (l tmLogger) SetLevel(level string() {
+  switch (level) {
+    case "debug":
+      l.sourceLogger = level.NewFilter(l.sourceLogger, level.AllowDebug())
+  }
+}
+
+func (l tmLogger) Info(msg string, keyvals ...interface{}) error {
+  l.sourceLogger.Log("msg", msg, keyvals...)
+}
+
+# log.go
+func With(logger Logger, keyvals ...interface{}) Logger {
+  kitlog.With(logger.sourceLogger, keyvals...)
+}
+```
+
+Usage:
+
+```go
+logger := log.NewTmLogger(os.Stdout)
+logger.SetLevel(config.GetString("log_level"))
+node.SetLogger(log.With(logger, "node", Name))
+```
+
+**Other log formatters**
+
+In the future, we may want other formatters like JSONFormatter.
+
+```
+{ "level": "notice", "time": "2017-04-25 14:45:08.562471297 -0400 EDT", "module": "consensus", "msg": "ABCI Replay Blocks", "appHeight": 0, "storeHeight": 0, "stateHeight": 0 }
+```
+
+### 3) Dynamic logger setting
+
+https://dave.cheney.net/2017/01/23/the-package-level-logger-anti-pattern
+
+This is the hardest part and where the most work will be done. logger should be tied to the processing struct, or the context if it adds some fields to the logger.
+
+```go
+type BaseService struct {
+    log     log15.Logger
+    name    string
+    started uint32 // atomic
+    stopped uint32 // atomic
+...
+}
+```
+
+BaseService already contains `log` field, so most of the structs embedding it should be fine. We should rename it to `logger`.
+
+The only thing missing is the ability to set logger:
+
+```
+func (bs *BaseService) SetLogger(l log.Logger) {
+  bs.logger = l
+}
+```
+
+### 4) Conventions
+
+Important keyvals should go first. Example:
+
+```
+correct
+I[04-25|14:45:08.322] ABCI Replay Blocks                       module=consensus instance=1 appHeight=0 storeHeight=0 stateHeight=0
+```
+
+not
+
+```
+wrong
+I[04-25|14:45:08.322] ABCI Replay Blocks                       module=consensus appHeight=0 storeHeight=0 stateHeight=0 instance=1
+```
+
+for that in most cases you'll need to add `instance` field to a logger upon creating, not when u log a particular message:
+
+```go
+colorFn := func(keyvals ...interface{}) term.FgBgColor {
+		for i := 1; i < len(keyvals); i += 2 {
+			if keyvals[i] == "instance" && keyvals[i+1] == "1" {
+				return term.FgBgColor{Fg: term.Blue}
+			} else if keyvals[i] == "instance" && keyvals[i+1] == "1" {
+				return term.FgBgColor{Fg: term.Red}
+			}
+		}
+		return term.FgBgColor{}
+	}
+logger := term.NewLogger(os.Stdout, log.NewTmLogger, colorFn)
+
+c1 := NewConsensusReactor(...)
+c1.SetLogger(log.With(logger, "instance", 1))
+
+c2 := NewConsensusReactor(...)
+c2.SetLogger(log.With(logger, "instance", 2))
+```
+
+## Status
+
+proposed
+
+## Consequences
+
+### Positive
+
+Dynamic logger, which could be turned off for some modules at runtime. Public interface for other projects using Tendermint libraries.
+
+### Negative
+
+We may loose the ability to color keys in keyvalue pairs. go-kit allow you to easily change foreground / background colors of the whole string, but not its parts.
+
+### Neutral
+
+## Appendix A.
+
+I really like a minimalistic approach go-kit took with his logger https://github.com/go-kit/kit/tree/master/log:
+
+```
+type Logger interface {
+    Log(keyvals ...interface{}) error
+}
+```
+
+See [The Hunt for a Logger Interface](https://go-talks.appspot.com/github.com/ChrisHines/talks/structured-logging/structured-logging.slide). The advantage is greater composability (check out how go-kit defines colored logging or log-leveled logging on top of this interface https://github.com/go-kit/kit/tree/master/log).

docs/architecture/merkle-frey.md

@@ -0,0 +1,240 @@
+# Merkle data stores - Frey's proposal
+
+## TL;DR
+
+To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implementation of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an app hash to the consensus engine, as well as a full proof to any client.
+
+This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adapter from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adapters to existence merkle-ized data stores, or modifying the stores to support this interface.
+
+We need to consider the API (both in-process and over the network), language bindings, maintaining handles to old state (and garbage collecting), persistence, security, providing merkle proofs, and general key-value store operations. To stay consistent with the blockchains "single global order of operations", this data store should only allow one connection at a time to have write access.
+
+## Overview
+
+* **State**
+  * There are two concepts of state, "committed state" and "working state"
+  * The working state is only accessible from the ABCi app, allows writing, but does not need to support proofs.
+  * When we commit the "working state", it becomes a new "committed state" and has an immutable root hash, provides proofs, and can be exposed to external clients.
+* **Transactions**
+  * The database always allows creating a read-only transaction at the last "committed state", this transaction can serve read queries and proofs.
+  * The database maintains all data to serve these read transactions until they are closed by the client (or time out).  This allows the client(s) to determine how much old info is needed
+  * The database can only support *maximal* one writable transaction at a time.  This makes it easy to enforce serializability, and attempting to start a second writable transaction may trigger a panic.
+* **Functionality**
+  * It must support efficient key-value operations (get/set/delete)
+  * It must support returning merkle proofs for any "committed state"
+  * It should support range queries on subsets of the key space if possible (ie. if the db doesn't hash keys)
+  * It should also support listening to changes to a desired key via pub-sub or similar method, so I can quickly notify you on a change to your balance without constant polling.
+  * It may support other db-specific query types as an extension to this interface, as long as all specified actions maintain their meaning.
+* **Interface**
+  * This interface should be domain-specific - ie. designed just for this use case
+  * It should present a simple go interface for embedding the data store in-process
+  * It should create a gRPC/protobuf API for calling from any client
+  * It should provide and maintain client adapters from our in-process interface to gRPC client calls for at least golang and Java (maybe more languages?)
+  * It should provide and maintain server adapters from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support)
+* **Persistence**
+  * It must support atomic persistence upon committing a new block.  That is, upon crash recovery, the state is guaranteed to represent the state at the end of a complete block (along with a note of which height it was).
+  * It must delay deletion of old data as long as there are open read-only transactions referring to it, thus we must maintain some sort of WAL to keep track of pending cleanup.
+  * When a transaction is closed, or when we recover from a crash, it should clean up all no longer needed data to avoid memory/storage leaks.
+* **Security and Auth**
+  * If we allow connections over gRPC, we must consider this issues and allow both encryption (SSL), and some basic auth rules to prevent undesired access to the DB
+  * This is client-specific and does not need to be supported in the in-process, embedded version.
+
+## Details
+
+Here we go more in-depth in each of the sections, explaining the reasoning and more details on the desired behavior. This document is only the high-level architecture and should support multiple implementations.  When building out a specific implementation, a similar document should be provided for that repo, showing how it implements these concepts, and details about memory usage, storage, efficiency, etc.
+
+
+### State
+
+The current ABCi interface avoids this question a bit and that has brought confusion.  If I use `merkleeyes` to store data, which state is returned from `Query`?  The current "working" state, which I would like to refer to in my ABCi application?  Or the last committed state, which I would like to return to a client's query?  Or an old state, which I may select based on height?
+
+Right now, `merkleeyes` implements `Query` like a normal ABCi app and only returns committed state, which has lead to problems and confusion.  Thus, we need to be explicit about which state we want to view.  Each viewer can then specify which state it wants to view.  This allows the app to query the working state in DeliverTx, but the committed state in Query.
+
+We can easily provide two global references for "last committed" and "current working" states.  However, if we want to also allow querying of older commits... then we need some way to keep track of which ones are still in use, so we can garbage collect the unneeded ones. There is a non-trivial overhead in holding references to all past states, but also a hard-coded solution (hold onto the last 5 commits) may not support all clients.  We should let the client define this somehow.
+
+### Transactions
+
+Transactions (in the typical database sense) are a clean and established solution to this issue.  We can look at the [isolations levels](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) which attempt to provide us things like "repeatable reads".  That means if we open a transaction, and query some data 100 times while other processes are writing to the db, we get the same result each time.  This transaction has a reference to its own local state from the time the transaction started. (We are referring to the highest isolation levels here, which correlate well this the blockchain use case).
+
+If we implement a read-only transaction as a reference to state at the time of creation of that transaction, we can then hold these references to various snapshots, one per block that we are interested, and allow the client to multiplex queries and proofs from these various blocks.
+
+If we continue using these concepts (which have informed 30+ years of server side design), we can add a few nice features to our write transactions. The first of which is `Rollback` and `Commit`.  That means all the changes we make in this transaction have no effect on the database until they are committed.  And until they are committed, we can always abort if we detect an anomaly, returning to the last committed state with a rollback.
+
+There is also a nice extension to this available on some database servers, basically, "nested" transactions or "savepoints".  This means that within one transaction, you can open a subtransaction/savepoint and continue work.  Later you have the option to commit or rollback all work since the savepoint/subtransaction.  And then continue with the main transaction.
+
+If you don't understand why this is useful, look at how basecoin needs to [hold cached state for AppTx](https://github.com/tendermint/basecoin/blob/master/state/execution.go#L126-L149), meaning that it rolls back all modifications if the AppTx returns an error. This was implemented as a wrapper in basecoin, but it is a reasonable thing to support in the DB interface itself (especially since the implementation becomes quite non-trivial as soon as you support range queries).
+
+To give a bit more reference to this concept in practice, read about [Savepoints in Postgresql](https://www.postgresql.org/docs/current/static/tutorial-transactions.html) ([reference](https://www.postgresql.org/docs/current/static/sql-savepoint.html)) or [Nesting transactions in SQL Server](http://dba-presents.com/index.php/databases/sql-server/43-nesting-transactions-and-save-transaction-command) (TL;DR: scroll to the bottom, section "Real nesting transactions with SAVE TRANSACTION")
+
+### Functionality
+
+Merkle trees work with key-value pairs, so we should most importantly focus on the basic Key-Value operations.  That is `Get`, `Set`, and `Remove`. We also need to return a merkle proof for any key, along with a root hash of the tree for committing state to the blockchain. This is just the basic merkle-tree stuff.
+
+If it is possible with the implementation, it is nice to provide access to Range Queries.  That is, return all values where the key is between X and Y.  If you construct your keys wisely, it is possible to store lists (1:N) relations this way.  Eg, storing blog posts and the key is blog:`poster_id`:`sequence`, then I could search for all blog posts by a given `poster_id`, or even return just posts 10-19 from the given poster.
+
+The construction of a tree that supports range queries was one of the [design decisions of go-merkle](https://github.com/tendermint/go-merkle/blob/master/README.md).  It is also kind of possible with [ethereum's patricia trie](https://github.com/ethereum/wiki/wiki/Patricia-Tree) as long as the key is less than 32 bytes.
+
+In addition to range queries, there is one more nice feature that we could add to our data store - listening to events. Depending on your context, this is "reactive programming", "event emitters", "notifications", etc... But the basic concept is that a client can listen for all changes to a given key (or set of keys), and receive a notification when this happens. This is very important to avoid [repeated polling and wasted queries](http://resthooks.org/) when a client simply wants to [detect changes](https://www.rethinkdb.com/blog/realtime-web/).
+
+If the database provides access to some "listener" functionality, the app can choose to expose this to the external client via websockets, web hooks, http2 push events, android push notifications, etc, etc etc.... But if we want to support modern client functionality, let's add support for this reactive paradigm in our DB interface.
+
+**TODO** support for more advanced backends, eg. Bolt....
+
+### Go Interface
+
+I will start with a simple go interface to illustrate the in-process interface.  Once there is agreement on how this looks, we can work out the gRPC bindings to support calling out of process. These interfaces are not finalized code, but I think the demonstrate the concepts better than text and provide a strawman to get feedback.
+
+```
+// DB represents the committed state of a merkle-ized key-value store
+type DB interface {
+  // Snapshot returns a reference to last committed state to use for
+  // providing proofs, you must close it at the end to garbage collect
+  // the historical state we hold on to to make these proofs
+  Snapshot() Prover
+
+  // Start a transaction - only way to change state
+  // This will return an error if there is an open Transaction
+  Begin() (Transaction, error)
+
+  // These callbacks are triggered when the Transaction is Committed
+  // to the DB.  They can be used to eg. notify clients via websockets when
+  // their account balance changes.
+  AddListener(key []byte, listener Listener)
+  RemoveListener(listener Listener)
+}
+
+// DBReader represents a read-only connection to a snapshot of the db
+type DBReader interface {
+  // Queries on my local view
+  Has(key []byte) (bool, error)
+  Get(key []byte) (Model, error)
+  GetRange(start, end []byte, ascending bool, limit int) ([]Model, error)
+  Closer
+}
+
+// Prover is an interface that lets one query for Proofs, holding the
+// data at a specific location in memory
+type Prover interface {
+  DBReader
+
+  // Hash is the AppHash (RootHash) for this block
+  Hash() (hash []byte)
+
+  // Prove returns the data along with a merkle Proof
+  // Model and Proof are nil if not found
+  Prove(key []byte) (Model, Proof, error)
+}
+
+// Transaction is a set of state changes to the DB to be applied atomically.
+// There can only be one open transaction at a time, which may only have
+// maximum one subtransaction at a time.
+// In short, at any time, there is exactly one object that can write to the
+// DB, and we can use Subtransactions to group operations and roll them back
+// together (kind of like `types.KVCache` from basecoin)
+type Transaction interface {
+  DBReader
+
+  // Change the state - will raise error immediately if this Transaction
+  // is not holding the exclusive write lock
+  Set(model Model) (err error)
+  Remove(key []byte) (removed bool, err error)
+
+  // Subtransaction starts a new subtransaction, rollback will not affect the
+  // parent.  Only on Commit are the changes applied to this transaction.
+  // While the subtransaction exists, no write allowed on the parent.
+  // (You must Commit or Rollback the child to continue)
+  Subtransaction() Transaction
+
+  // Commit this transaction (or subtransaction), the parent reference is
+  // now updated.
+  // This only updates persistant store if the top level transaction commits
+  // (You may have any number of nested sub transactions)
+  Commit() error
+
+  // Rollback ends the transaction and throw away all transaction-local state,
+  // allowing the tree to prune those elements.
+  // The parent transaction now recovers the write lock.
+  Rollback()
+}
+
+// Listener registers callbacks on changes to the data store
+type Listener interface {
+  OnSet(key, value, oldValue []byte)
+  OnRemove(key, oldValue []byte)
+}
+
+// Proof represents a merkle proof for a key
+type Proof interface {
+  RootHash() []byte
+  Verify(key, value, root []byte) bool
+}
+
+type Model interface {
+  Key() []byte
+  Value() []byte
+}
+
+// Closer releases the reference to this state, allowing us to garbage collect
+// Make sure to call it before discarding.
+type Closer interface {
+  Close()
+}
+```
+
+### Remote Interface
+
+The use-case of allowing out-of-process calls is very powerful.  Not just to provide a powerful merkle-ready data store to non-go applications.
+
+It we allow the ABCi app to maintain the only writable connections, we can guarantee that all transactions are only processed through the tendermint consensus engine.  We could then allow multiple "web server" machines "read-only" access and scale out the database reads, assuming the consensus engine, ABCi logic, and public key cryptography is more the bottleneck than the database. We could even place the consensus engine, ABCi app, and data store on one machine, connected with unix sockets for security, and expose a tcp/ssl interface for reading the data, to scale out query processing over multiple machines.
+
+But returning our focus directly to the ABCi app (which is the most important use case).  An app may well want to maintain 100 or 1000 snapshots of different heights to allow people to easily query many proofs at a given height without race conditions (very important for IBC, ask Jae).  Thus, we should not require a separate TCP connection for each height, as this gets quite awkward with so many connections. Also, if we want to use gRPC, we should consider the connections potentially transient (although they are more efficient with keep-alive).
+
+Thus, the wire encoding of a transaction or a snapshot should simply return a unique id.  All methods on a `Prover` or `Transaction` over the wire can send this id along with the arguments for the method call.  And we just need a hash map on the server to map this id to a state.
+
+The only negative of not requiring a persistent tcp connection for each snapshot is there is no auto-detection if the client crashes without explicitly closing the connections.  Thus, I would suggest adding a `Ping` thread in the gRPC interface which keeps the Snapshot alive.  If no ping is received within a server-defined time, it may automatically close those transactions.  And if we consider a client with 500 snapshots that needs to ping each every 10 seconds, that is a lot of overhead, so we should design the ping to accept a list of IDs for the client and update them all.  Or associate all snapshots with a clientID and then just send the clientID in the ping.  (Please add other ideas on how to detect client crashes without persistent connections).
+
+To encourage adoption, we should provide a nice client that uses this gRPC interface (like we do with ABCi).  For go, the client may have the exact same interface as the in-process version, just that the error call may return network errors, not just illegal operations.  We should also add a client with a clean API for Java, since that seems to be popular among app developers in the current tendermint community.  Other bindings as we see the need in the server space.
+
+### Persistence
+
+Any data store worth it's name should not lose all data on a crash.  Even [redis provides some persistence](https://redis.io/topics/persistence) these days. Ideally, if the system crashes and restarts, it should have the data at the last block N that was committed.  If the system crash during the commit of block N+1, then the recovered state should either be block N or completely committed block N+1, but no partial state between the two.  Basically, the commit must be an atomic operation (even if updating 100's of records).
+
+To avoid a lot of headaches ourselves, we can use an existing data store, such as leveldb, which provides `WriteBatch` to group all operations.
+
+The other issue is cleaning up old state.  We cannot delete any information from our persistent store, as long as any snapshot holds a reference to it (or else we get some panics when the data we query is not there).  So, we need to store the outstanding deletions that we can perform when the snapshot is `Close`d.  In addition, we must consider the case that the data store crashes with open snapshots.  Thus, the info on outstanding deletions must also be persisted somewhere.  Something like a "delete-behind log" (the opposite of a "write ahead log").
+
+This is not a concern of the generic interface, but each implementation should take care to handle this well to avoid accumulation of unused references in the data store and eventual data bloat.
+
+#### Backing stores
+
+It is way outside the scope of this project to build our own database that is capable of efficiently storing the data, provide multiple read-only snapshots at once, and save it atomically. The best approach seems to select an existing database (best a simple one) that provides this functionality and build upon it, much like the current `go-merkle` implementation builds upon `leveldb`.  After some research here are winners and losers:
+
+**Winners**
+
+* Leveldb - [provides consistent snapshots](https://ayende.com/blog/161705/reviewing-leveldb-part-xiii-smile-and-here-is-your-snapshot), and [provides tooling for building ACID compliance](http://codeofrob.com/entries/writing-a-transaction-manager-on-top-of-leveldb.html)
+  * Note there are at least two solid implementations available in go - [goleveldb](https://github.com/syndtr/goleveldb) - a pure go implementation, and [levigo](https://github.com/jmhodges/levigo) - a go wrapper around leveldb.
+  * Goleveldb is much easier to compile and cross-compile (not requiring cgo), while levigo (or cleveldb) seems to provide a significant performance boosts (but I had trouble even running benchmarks)
+* PostgreSQL - fully supports these ACID semantics if you call `SET TRANSACTION ISOLATION LEVEL SERIALIZABLE` at the beginning of a transaction (tested)
+  * This may be total overkill unless we also want to make use of other features, like storing data in multiple columns with secondary indexes.
+  * Trillian can show an example of [how to store a merkle tree in sql](https://github.com/google/trillian/blob/master/storage/mysql/tree_storage.go)
+
+**Losers**
+
+* Bolt - open [read-only snapshots can block writing](https://github.com/boltdb/bolt/issues/378)
+* Mongo - [barely even supports atomic operations](https://docs.mongodb.com/manual/core/write-operations-atomicity/), much less multiple snapshots
+
+**To investigate**
+
+* [Trillian](https://github.com/google/trillian) - has a [persistent merkle tree interface](https://github.com/google/trillian/blob/master/storage/tree_storage.go) along with [backend storage with mysql](https://github.com/google/trillian/blob/master/storage/mysql/tree_storage.go), good inspiration for our design if not directly using it
+* [Moss](https://github.com/couchbase/moss) - another key-value store in go, seems similar to leveldb, maybe compare with performance tests?
+
+### Security
+
+When allowing access out-of-process, we should provide different mechanisms to secure it.  The first is the choice of binding to a local unix socket or a tcp port.  The second is the optional use of ssl to encrypt the connection (very important over tcp).  The third is authentication to control access to the database.
+
+We may also want to consider the case of two server connections with different permissions, eg. a local unix socket that allows write access with no more credentials, and a public TCP connection with ssl and authentication that only provides read-only access.
+
+The use of ssl is quite easy in go, we just need to generate and sign a certificate, so it is nice to be able to disable it for dev machines, but it is very important for production.
+
+For authentication, let me sketch out a minimal solution. The server could just have a simple config file with key/bcrypt(password) pairs along with read/write permission level, and read that upon startup.  The client must provide a username and password in the HTTP headers when making the original HTTPS gRPC connection.
+
+This is super minimal to provide some protection. Things like LDAP, OAuth and single-sign on seem overkill and even potential security holes.  Maybe there is another solution somewhere in the middle.

docs/architecture/merkle.md

@@ -0,0 +1,17 @@
+# Merkle data stores
+
+To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client.
+
+This engine is currently implemented in `go-merkle` with `merkleeyes` providing a language-agnostic binding via ABCi.  It uses `tmlibs/db` bindings internally to persist data to leveldb.
+
+What are some of the requirements of this store:
+
+* It must support efficient key-value operations (get/set/delete)
+* It must support persistance.
+* We must only persist complete blocks, so when we come up after a crash we are at the state of block N or N+1, but not in-between these two states.
+* It must allow us to read/write from one uncommited state (working state), while serving other queries from the last commited state. And a way to determine which one to serve for each client.
+* It must allow us to hold references to old state, to allow providing proofs from 20 blocks ago.  We can define some limits as to the maximum time to hold this data.
+* We provide in process binding in Go
+* We provide language-agnostic bindings when running the data store as it's own process.
+
+

glide.lock

@@ -1,38 +1,82 @@
-hash: dcaf3fb1290b0d7942c86f0644a7431ac313247936eab9515b1ade9ffe579848
-updated: 2017-01-13T00:30:55.237750829-05:00
+hash: 2c988aae9517b386ee911e4da5deb9f5034359b7e2ccf448952a3ddb9771222d
+updated: 2017-06-28T13:04:20.907047164+02:00
 imports:
 - name: github.com/btcsuite/btcd
-  version: 153dca5c1e4b5d1ea1523592495e5bedfa503391
+  version: b8df516b4b267acf2de46be593a9d948d1d2c420
   subpackages:
   - btcec
-- name: github.com/BurntSushi/toml
-  version: 99064174e013895bbd9b025c31100bd1d9b590ca
+- name: github.com/btcsuite/fastsha256
+  version: 637e656429416087660c84436a2a035d69d54e2e
 - name: github.com/ebuchman/fail-test
-  version: c1eddaa09da2b4017351245b0d43234955276798
+  version: 95f809107225be108efcf10a3509e4ea6ceef3c4
+- name: github.com/fsnotify/fsnotify
+  version: 4da3e2cfbabc9f751898f250b49f2439785783a1
+- name: github.com/go-kit/kit
+  version: d67bb4c202e3b91377d1079b110a6c9ce23ab2f8
+  subpackages:
+  - log
+  - log/level
+  - log/term
+- name: github.com/go-logfmt/logfmt
+  version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: f9114dace7bd920b32f943b3c73fafbcbab2bf31
+  version: 9df9efe4c742f1a2bfdedf1c3b6902fc6e814c6b
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 8ee79997227bf9b34611aee7946ae64735e6fd93
+  version: 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8
   subpackages:
   - proto
+  - ptypes/any
 - name: github.com/golang/snappy
-  version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
+  version: 553a641470496b2327abcac10b36396bd98e45c9
 - name: github.com/gorilla/websocket
-  version: 17634340a83afe0cab595e40fbc63f6ffa1d8915
+  version: a91eba7f97777409bc2c443f5534d41dd20c5720
+- name: github.com/hashicorp/hcl
+  version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca
+  subpackages:
+  - hcl/ast
+  - hcl/parser
+  - hcl/scanner
+  - hcl/strconv
+  - hcl/token
+  - json/parser
+  - json/scanner
+  - json/token
+- name: github.com/inconshreveable/mousetrap
+  version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
-- name: github.com/mattn/go-colorable
-  version: d228849504861217f796da67fae4f6e347643f15
-- name: github.com/mattn/go-isatty
-  version: 30a891c33c7cde7b02a981314b4228ec99380cca
+- name: github.com/kr/logfmt
+  version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0
+- name: github.com/magiconair/properties
+  version: 51463bfca2576e06c62a8504b5c0f06d61312647
+- name: github.com/mitchellh/mapstructure
+  version: cc8532a8e9a55ea36402aa21efdf403a60d34096
+- name: github.com/pelletier/go-buffruneio
+  version: c37440a7cf42ac63b919c752ca73a85067e05992
+- name: github.com/pelletier/go-toml
+  version: 5ccdfb18c776b740aecaf085c4d9a2779199c279
+- name: github.com/pkg/errors
+  version: 645ef00459ed84a119197bfb8d8205042c6df63d
+- name: github.com/spf13/afero
+  version: 9be650865eab0c12963d8753212f4f9c66cdcf12
+  subpackages:
+  - mem
+- name: github.com/spf13/cast
+  version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4
+- name: github.com/spf13/cobra
+  version: 4cdb38c072b86bf795d2c81de50784d9fdd6eb77
+- name: github.com/spf13/jwalterweatherman
+  version: 8f07c835e5cc1450c082fe3a439cf87b0cbb2d99
 - name: github.com/spf13/pflag
-  version: 25f8b5b07aece3207895bf19f7ab517eb3b22a40
+  version: e57e3eeb33f795204c1ca35f56c44f83227c6e66
+- name: github.com/spf13/viper
+  version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2
 - name: github.com/syndtr/goleveldb
-  version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
+  version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4
   subpackages:
   - leveldb
   - leveldb/cache
@@ -47,12 +91,11 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 699d45bc678865b004b90213bf88a950f420973b
+  version: 864d1f80b36b440bde030a5c18d8ac3aa8c2949d
   subpackages:
   - client
   - example/counter
   - example/dummy
-  - example/nil
   - server
   - types
 - name: github.com/tendermint/ed25519
@@ -60,48 +103,36 @@ imports:
   subpackages:
   - edwards25519
   - extra25519
-- name: github.com/tendermint/go-autofile
-  version: 0416e0aa9c68205aa44844096f9f151ada9d0405
-- name: github.com/tendermint/go-clist
-  version: 3baa390bbaf7634251c42ad69a8682e7e3990552
-- name: github.com/tendermint/go-common
-  version: e289af53b6bf6af28da129d9ef64389a4cf7987f
-  subpackages:
-  - test
-- name: github.com/tendermint/go-config
-  version: e64b424499acd0eb9856b88e10c0dff41628c0d6
 - name: github.com/tendermint/go-crypto
-  version: 4b11d62bdb324027ea01554e5767b71174680ba0
-- name: github.com/tendermint/go-db
-  version: 72f6dacd22a686cdf7fcd60286503e3aceda77ba
-- name: github.com/tendermint/go-events
-  version: fddee66d90305fccb6f6d84d16c34fa65ea5b7f6
-- name: github.com/tendermint/go-flowrate
-  version: a20c98e61957faa93b4014fbd902f20ab9317a6a
-  subpackages:
-  - flowrate
-- name: github.com/tendermint/go-logger
-  version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2
-- name: github.com/tendermint/go-merkle
-  version: 7a86b4486f2cd84ac885c5bbc609fdee2905f5d1
-- name: github.com/tendermint/go-p2p
-  version: 3d98f675f30dc4796546b8b890f895926152fa8d
-  subpackages:
-  - upnp
-- name: github.com/tendermint/go-rpc
-  version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7
-  subpackages:
-  - client
-  - server
-  - types
+  version: 95b7c9e09c49b91bfbb71bb63dd514eb55450f16
 - name: github.com/tendermint/go-wire
-  version: 2f3b7aafe21c80b19b6ee3210ecb3e3d07c7a471
-- name: github.com/tendermint/log15
-  version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
+  version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb
   subpackages:
-  - term
+  - data
+  - data/base58
+- name: github.com/tendermint/merkleeyes
+  version: 102aaf5a8ffda1846413fb22805a94def2045b9f
+  subpackages:
+  - app
+  - client
+  - iavl
+  - testutil
+- name: github.com/tendermint/tmlibs
+  version: 7ce4da1eee6004d627e780c8fe91e96d9b99e459
+  subpackages:
+  - autofile
+  - cli
+  - cli/flags
+  - clist
+  - common
+  - db
+  - events
+  - flowrate
+  - log
+  - merkle
+  - test
 - name: golang.org/x/crypto
-  version: 7c6cc321c680f03b9ef0764448e780704f486b51
+  version: c7af5bf2638a1164f2eb5467c39c6cffbd13a02e
   subpackages:
   - curve25519
   - nacl/box
@@ -112,7 +143,7 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9
+  version: feeb485667d1fdabe727840fe00adc22431bc86e
   subpackages:
   - context
   - http2
@@ -122,27 +153,49 @@ imports:
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: d75a52659825e75fff6158388dddc6a5b04f9ba5
+  version: e62c3de784db939836898e5c19ffd41bece347da
   subpackages:
   - unix
 - name: golang.org/x/text
-  version: 44f4f658a783b0cee41fe0a23b8fc91d9c120558
+  version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4
   subpackages:
   - secure/bidirule
   - transform
   - unicode/bidi
   - unicode/norm
+- name: google.golang.org/genproto
+  version: 411e09b969b1170a9f0c467558eb4c4c110d9c77
+  subpackages:
+  - googleapis/rpc/status
 - name: google.golang.org/grpc
-  version: 50955793b0183f9de69bd78e2ec251cf20aab121
+  version: 844f573616520565fdc6fb4db242321b5456fd6d
   subpackages:
   - codes
   - credentials
+  - grpclb/grpc_lb_v1
   - grpclog
   - internal
+  - keepalive
   - metadata
   - naming
   - peer
   - stats
+  - status
   - tap
   - transport
-testImports: []
+- name: gopkg.in/yaml.v2
+  version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b
+testImports:
+- name: github.com/davecgh/go-spew
+  version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
+  subpackages:
+  - spew
+- name: github.com/pmezard/go-difflib
+  version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
+  subpackages:
+  - difflib
+- name: github.com/stretchr/testify
+  version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
+  subpackages:
+  - assert
+  - require

glide.yaml

@@ -1,29 +1,60 @@
 package: github.com/tendermint/tendermint
 import:
+- package: github.com/ebuchman/fail-test
 - package: github.com/gogo/protobuf
   subpackages:
   - proto
+- package: github.com/golang/protobuf
+  subpackages:
+  - proto
+- package: github.com/pelletier/go-toml
+  version: ^1.0.0
 - package: github.com/gorilla/websocket
-- package: github.com/spf13/pflag
-- package: github.com/tendermint/ed25519
-- package: github.com/tendermint/go-flowrate
-- package: github.com/tendermint/go-autofile
-- package: github.com/tendermint/go-clist
-- package: github.com/tendermint/go-common
-- package: github.com/tendermint/go-config
-- package: github.com/tendermint/go-crypto
-- package: github.com/tendermint/go-db
-- package: github.com/tendermint/go-events
-- package: github.com/tendermint/go-logger
-- package: github.com/tendermint/go-merkle
-- package: github.com/tendermint/go-p2p
-- package: github.com/tendermint/go-rpc
-- package: github.com/tendermint/go-wire
-- package: github.com/tendermint/log15
+- package: github.com/pkg/errors
+  version: ~0.8.0
+- package: github.com/spf13/cobra
+- package: github.com/spf13/viper
 - package: github.com/tendermint/abci
+  version: v0.5.0
+  subpackages:
+  - client
+  - example/dummy
+  - types
+- package: github.com/tendermint/go-crypto
+  version: ~0.2.2
+- package: github.com/tendermint/go-wire
+  version: ~0.6.2
+  subpackages:
+  - data
+- package: github.com/tendermint/merkleeyes
+  version: ~0.2.4
+  subpackages:
+  - app
+  - iavl
+  - testutil
+- package: github.com/tendermint/tmlibs
+  version: ~0.2.2
+  subpackages:
+  - autofile
+  - cli
+  - clist
+  - common
+  - db
+  - events
+  - flowrate
+  - log
+  - merkle
 - package: golang.org/x/crypto
   subpackages:
+  - nacl/box
+  - nacl/secretbox
   - ripemd160
-- package: github.com/tendermint/go-flowrate
+- package: golang.org/x/net
   subpackages:
-  - flowrate 
+  - context
+- package: google.golang.org/grpc
+testImport:
+- package: github.com/stretchr/testify
+  subpackages:
+  - assert
+  - require

mempool/log.go

@@ -1,18 +0,0 @@
-package mempool
-
-import (
-	"github.com/tendermint/go-logger"
-)
-
-var log = logger.New("module", "mempool")
-
-/*
-func init() {
-	log.SetHandler(
-		logger.LvlFilterHandler(
-			logger.LvlDebug,
-			logger.BypassHandler(),
-		),
-	)
-}
-*/

mempool/mempool.go

@@ -7,13 +7,17 @@ import (
 	"sync/atomic"
 	"time"
 
-	auto "github.com/tendermint/go-autofile"
-	"github.com/tendermint/go-clist"
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
+	"github.com/pkg/errors"
+
+	abci "github.com/tendermint/abci/types"
+	auto "github.com/tendermint/tmlibs/autofile"
+	"github.com/tendermint/tmlibs/clist"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+
+	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 )
 
 /*
@@ -47,7 +51,7 @@ TODO: Better handle abci client errors. (make it automatically handle connection
 const cacheSize = 100000
 
 type Mempool struct {
-	config cfg.Config
+	config *cfg.MempoolConfig
 
 	proxyMtx      sync.Mutex
 	proxyAppConn  proxy.AppConnMempool
@@ -64,9 +68,11 @@ type Mempool struct {
 
 	// A log of mempool txs
 	wal *auto.AutoFile
+
+	logger log.Logger
 }
 
-func NewMempool(config cfg.Config, proxyAppConn proxy.AppConnMempool) *Mempool {
+func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool) *Mempool {
 	mempool := &Mempool{
 		config:        config,
 		proxyAppConn:  proxyAppConn,
@@ -76,26 +82,29 @@ func NewMempool(config cfg.Config, proxyAppConn proxy.AppConnMempool) *Mempool {
 		rechecking:    0,
 		recheckCursor: nil,
 		recheckEnd:    nil,
-
-		cache: newTxCache(cacheSize),
+		logger:        log.NewNopLogger(),
+		cache:         newTxCache(cacheSize),
 	}
 	mempool.initWAL()
 	proxyAppConn.SetResponseCallback(mempool.resCb)
 	return mempool
 }
 
+// SetLogger allows you to set your own Logger.
+func (mem *Mempool) SetLogger(l log.Logger) {
+	mem.logger = l
+}
+
 func (mem *Mempool) initWAL() {
-	walDir := mem.config.GetString("mempool_wal_dir")
+	walDir := mem.config.WalDir()
 	if walDir != "" {
-		err := EnsureDir(walDir, 0700)
+		err := cmn.EnsureDir(walDir, 0700)
 		if err != nil {
-			log.Error("Error ensuring Mempool wal dir", "error", err)
-			PanicSanity(err)
+			cmn.PanicSanity(errors.Wrap(err, "Error ensuring Mempool wal dir"))
 		}
 		af, err := auto.OpenAutoFile(walDir + "/wal")
 		if err != nil {
-			log.Error("Error opening Mempool wal file", "error", err)
-			PanicSanity(err)
+			cmn.PanicSanity(errors.Wrap(err, "Error opening Mempool wal file"))
 		}
 		mem.wal = af
 	}
@@ -202,7 +211,7 @@ func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) {
 			mem.txs.PushBack(memTx)
 		} else {
 			// ignore bad transaction
-			log.Info("Bad Transaction", "res", r)
+			mem.logger.Info("Bad Transaction", "res", r)
 
 			// remove from cache (it might be good later)
 			mem.cache.Remove(req.GetCheckTx().Tx)
@@ -219,7 +228,7 @@ func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
 	case *abci.Response_CheckTx:
 		memTx := mem.recheckCursor.Value.(*mempoolTx)
 		if !bytes.Equal(req.GetCheckTx().Tx, memTx.tx) {
-			PanicSanity(Fmt("Unexpected tx response from proxy during recheck\n"+
+			cmn.PanicSanity(cmn.Fmt("Unexpected tx response from proxy during recheck\n"+
 				"Expected %X, got %X", r.CheckTx.Data, memTx.tx))
 		}
 		if r.CheckTx.Code == abci.CodeType_OK {
@@ -240,7 +249,7 @@ func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
 		if mem.recheckCursor == nil {
 			// Done!
 			atomic.StoreInt32(&mem.rechecking, 0)
-			log.Info("Done rechecking txs")
+			mem.logger.Info("Done rechecking txs")
 		}
 	default:
 		// ignore other messages
@@ -249,7 +258,7 @@ func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
 
 // Get the valid transactions remaining
 // If maxTxs is -1, there is no cap on returned transactions.
-func (mem *Mempool) Reap(maxTxs int) []types.Tx {
+func (mem *Mempool) Reap(maxTxs int) types.Txs {
 	mem.proxyMtx.Lock()
 	defer mem.proxyMtx.Unlock()
 
@@ -263,13 +272,13 @@ func (mem *Mempool) Reap(maxTxs int) []types.Tx {
 }
 
 // maxTxs: -1 means uncapped, 0 means none
-func (mem *Mempool) collectTxs(maxTxs int) []types.Tx {
+func (mem *Mempool) collectTxs(maxTxs int) types.Txs {
 	if maxTxs == 0 {
 		return []types.Tx{}
 	} else if maxTxs < 0 {
 		maxTxs = mem.txs.Len()
 	}
-	txs := make([]types.Tx, 0, MinInt(mem.txs.Len(), maxTxs))
+	txs := make([]types.Tx, 0, cmn.MinInt(mem.txs.Len(), maxTxs))
 	for e := mem.txs.Front(); e != nil && len(txs) < maxTxs; e = e.Next() {
 		memTx := e.Value.(*mempoolTx)
 		txs = append(txs, memTx.tx)
@@ -281,7 +290,7 @@ func (mem *Mempool) collectTxs(maxTxs int) []types.Tx {
 // Mempool will discard these txs.
 // NOTE: this should be called *after* block is committed by consensus.
 // NOTE: unsafe; Lock/Unlock must be managed by caller
-func (mem *Mempool) Update(height int, txs []types.Tx) {
+func (mem *Mempool) Update(height int, txs types.Txs) {
 	// TODO: check err ?
 	mem.proxyAppConn.FlushSync() // To flush async resCb calls e.g. from CheckTx
 
@@ -298,9 +307,8 @@ func (mem *Mempool) Update(height int, txs []types.Tx) {
 	// Recheck mempool txs if any txs were committed in the block
 	// NOTE/XXX: in some apps a tx could be invalidated due to EndBlock,
 	//	so we really still do need to recheck, but this is for debugging
-	if mem.config.GetBool("mempool_recheck") &&
-		(mem.config.GetBool("mempool_recheck_empty") || len(txs) > 0) {
-		log.Info("Recheck txs", "numtxs", len(goodTxs))
+	if mem.config.Recheck && (mem.config.RecheckEmpty || len(txs) > 0) {
+		mem.logger.Info("Recheck txs", "numtxs", len(goodTxs))
 		mem.recheckTxs(goodTxs)
 		// At this point, mem.txs are being rechecked.
 		// mem.recheckCursor re-scans mem.txs and possibly removes some txs.
@@ -401,7 +409,7 @@ func (cache *txCache) Push(tx types.Tx) bool {
 		popped := cache.list.Front()
 		poppedTx := popped.Value.(types.Tx)
 		// NOTE: the tx may have already been removed from the map
-		// but deleting a non-existant element is fine
+		// but deleting a non-existent element is fine
 		delete(cache.map_, string(poppedTx))
 		cache.list.Remove(popped)
 	}

mempool/mempool_test.go

@@ -4,21 +4,31 @@ import (
 	"encoding/binary"
 	"testing"
 
-	"github.com/tendermint/tendermint/config/tendermint_test"
+	"github.com/tendermint/abci/example/counter"
+	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
-	"github.com/tendermint/abci/example/counter"
+	"github.com/tendermint/tmlibs/log"
 )
 
 func TestSerialReap(t *testing.T) {
-	config := tendermint_test.ResetConfig("mempool_mempool_test")
+	config := cfg.ResetTestRoot("mempool_test")
 
 	app := counter.NewCounterApplication(true)
 	app.SetOption("serial", "on")
 	cc := proxy.NewLocalClientCreator(app)
 	appConnMem, _ := cc.NewABCIClient()
+	appConnMem.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "mempool"))
+	if _, err := appConnMem.Start(); err != nil {
+		t.Fatalf("Error starting ABCI client: %v", err.Error())
+	}
 	appConnCon, _ := cc.NewABCIClient()
-	mempool := NewMempool(config, appConnMem)
+	appConnCon.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "consensus"))
+	if _, err := appConnCon.Start(); err != nil {
+		t.Fatalf("Error starting ABCI client: %v", err.Error())
+	}
+	mempool := NewMempool(config.Mempool, appConnMem)
+	mempool.SetLogger(log.TestingLogger())
 
 	deliverTxsRange := func(start, end int) {
 		// Deliver some txs.

mempool/reactor.go

@@ -6,13 +6,13 @@ import (
 	"reflect"
 	"time"
 
-	"github.com/tendermint/go-clist"
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/go-wire"
-	"github.com/tendermint/tendermint/types"
 	abci "github.com/tendermint/abci/types"
+	wire "github.com/tendermint/go-wire"
+	"github.com/tendermint/tmlibs/clist"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
 )
 
 const (
@@ -25,17 +25,17 @@ const (
 // MempoolReactor handles mempool tx broadcasting amongst peers.
 type MempoolReactor struct {
 	p2p.BaseReactor
-	config  cfg.Config
+	config  *cfg.MempoolConfig
 	Mempool *Mempool
 	evsw    types.EventSwitch
 }
 
-func NewMempoolReactor(config cfg.Config, mempool *Mempool) *MempoolReactor {
+func NewMempoolReactor(config *cfg.MempoolConfig, mempool *Mempool) *MempoolReactor {
 	memR := &MempoolReactor{
 		config:  config,
 		Mempool: mempool,
 	}
-	memR.BaseReactor = *p2p.NewBaseReactor(log, "MempoolReactor", memR)
+	memR.BaseReactor = *p2p.NewBaseReactor("MempoolReactor", memR)
 	return memR
 }
 
@@ -63,24 +63,24 @@ func (memR *MempoolReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
 func (memR *MempoolReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
 	_, msg, err := DecodeMessage(msgBytes)
 	if err != nil {
-		log.Warn("Error decoding message", "error", err)
+		memR.Logger.Error("Error decoding message", "err", err)
 		return
 	}
-	log.Debug("Receive", "src", src, "chId", chID, "msg", msg)
+	memR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg)
 
 	switch msg := msg.(type) {
 	case *TxMessage:
 		err := memR.Mempool.CheckTx(msg.Tx, nil)
 		if err != nil {
 			// Bad, seen, or conflicting tx.
-			log.Info("Could not add tx", "tx", msg.Tx)
+			memR.Logger.Info("Could not add tx", "tx", msg.Tx)
 			return
 		} else {
-			log.Info("Added valid tx", "tx", msg.Tx)
+			memR.Logger.Info("Added valid tx", "tx", msg.Tx)
 		}
 		// broadcasting happens from go routines per peer
 	default:
-		log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+		memR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg)))
 	}
 }
 
@@ -103,7 +103,7 @@ type Peer interface {
 // TODO: Handle mempool or reactor shutdown?
 // As is this routine may block forever if no new txs come in.
 func (memR *MempoolReactor) broadcastTxRoutine(peer Peer) {
-	if !memR.config.GetBool("mempool_broadcast") {
+	if !memR.config.Broadcast {
 		return
 	}
 

node/log.go

@@ -1,7 +0,0 @@
-package node
-
-import (
-	"github.com/tendermint/go-logger"
-)
-
-var log = logger.New("module", "node")

node/node.go

@@ -2,90 +2,121 @@ package node
 
 import (
 	"bytes"
-	"io/ioutil"
+	"errors"
 	"net"
 	"net/http"
 	"strings"
-	"time"
 
-	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-crypto"
-	dbm "github.com/tendermint/go-db"
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/go-rpc"
-	"github.com/tendermint/go-rpc/server"
-	"github.com/tendermint/go-wire"
+	abci "github.com/tendermint/abci/types"
+	crypto "github.com/tendermint/go-crypto"
+	wire "github.com/tendermint/go-wire"
 	bc "github.com/tendermint/tendermint/blockchain"
+	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/consensus"
 	mempl "github.com/tendermint/tendermint/mempool"
+	p2p "github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/proxy"
 	rpccore "github.com/tendermint/tendermint/rpc/core"
 	grpccore "github.com/tendermint/tendermint/rpc/grpc"
+	rpc "github.com/tendermint/tendermint/rpc/lib"
+	rpcserver "github.com/tendermint/tendermint/rpc/lib/server"
 	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/state/txindex"
+	"github.com/tendermint/tendermint/state/txindex/kv"
+	"github.com/tendermint/tendermint/state/txindex/null"
 	"github.com/tendermint/tendermint/types"
 	"github.com/tendermint/tendermint/version"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+
+	_ "net/http/pprof"
 )
 
-import _ "net/http/pprof"
-
 type Node struct {
-	config           cfg.Config
-	sw               *p2p.Switch
-	evsw             types.EventSwitch
-	blockStore       *bc.BlockStore
-	bcReactor        *bc.BlockchainReactor
-	mempoolReactor   *mempl.MempoolReactor
-	consensusState   *consensus.ConsensusState
-	consensusReactor *consensus.ConsensusReactor
-	privValidator    *types.PrivValidator
-	genesisDoc       *types.GenesisDoc
-	privKey          crypto.PrivKeyEd25519
-	proxyApp         proxy.AppConns
+	cmn.BaseService
+
+	// config
+	config        *cfg.Config
+	genesisDoc    *types.GenesisDoc    // initial validator set
+	privValidator *types.PrivValidator // local node's validator key
+
+	// network
+	privKey  crypto.PrivKeyEd25519 // local node's p2p key
+	sw       *p2p.Switch           // p2p connections
+	addrBook *p2p.AddrBook         // known peers
+
+	// services
+	evsw             types.EventSwitch           // pub/sub for services
+	blockStore       *bc.BlockStore              // store the blockchain to disk
+	bcReactor        *bc.BlockchainReactor       // for fast-syncing
+	mempoolReactor   *mempl.MempoolReactor       // for gossipping transactions
+	consensusState   *consensus.ConsensusState   // latest consensus state
+	consensusReactor *consensus.ConsensusReactor // for participating in the consensus
+	proxyApp         proxy.AppConns              // connection to the application
+	rpcListeners     []net.Listener              // rpc servers
+	txIndexer        txindex.TxIndexer
 }
 
-func NewNodeDefault(config cfg.Config) *Node {
+func NewNodeDefault(config *cfg.Config, logger log.Logger) *Node {
 	// Get PrivValidator
-	privValidatorFile := config.GetString("priv_validator_file")
-	privValidator := types.LoadOrGenPrivValidator(privValidatorFile)
-	return NewNode(config, privValidator, proxy.DefaultClientCreator(config))
+	privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger)
+	return NewNode(config, privValidator,
+		proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger)
 }
 
-func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node {
-
+func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator, logger log.Logger) *Node {
 	// Get BlockStore
-	blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir"))
+	blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir())
 	blockStore := bc.NewBlockStore(blockStoreDB)
 
-	// Get State db
-	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
+	consensusLogger := logger.With("module", "consensus")
+	stateLogger := logger.With("module", "state")
 
 	// Get State
-	state := sm.GetState(config, stateDB)
+	stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir())
+	state := sm.GetState(stateDB, config.GenesisFile())
+	state.SetLogger(stateLogger)
 
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore))
+	// and sync tendermint and the app by replaying any necessary blocks
+	handshaker := consensus.NewHandshaker(state, blockStore)
+	handshaker.SetLogger(consensusLogger)
+	proxyApp := proxy.NewAppConns(clientCreator, handshaker)
+	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if _, err := proxyApp.Start(); err != nil {
-		Exit(Fmt("Error starting proxy app connections: %v", err))
+		cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err))
 	}
 
-	// add the chainid and number of validators to the global config
-	config.Set("chain_id", state.ChainID)
-	config.Set("num_vals", state.Validators.Size())
+	// reload the state (it may have been updated by the handshake)
+	state = sm.LoadState(stateDB)
+	state.SetLogger(stateLogger)
+
+	// Transaction indexing
+	var txIndexer txindex.TxIndexer
+	switch config.TxIndex {
+	case "kv":
+		store := dbm.NewDB("tx_index", config.DBBackend, config.DBDir())
+		txIndexer = kv.NewTxIndex(store)
+	default:
+		txIndexer = &null.TxIndex{}
+	}
+	state.TxIndexer = txIndexer
 
 	// Generate node PrivKey
 	privKey := crypto.GenPrivKeyEd25519()
 
 	// Make event switch
 	eventSwitch := types.NewEventSwitch()
+	eventSwitch.SetLogger(logger.With("module", "types"))
 	_, err := eventSwitch.Start()
 	if err != nil {
-		Exit(Fmt("Failed to start switch: %v", err))
+		cmn.Exit(cmn.Fmt("Failed to start switch: %v", err))
 	}
 
 	// Decide whether to fast-sync or not
 	// We don't fast-sync when the only validator is us.
-	fastSync := config.GetBool("fast_sync")
+	fastSync := config.FastSync
 	if state.Validators.Size() == 1 {
 		addr, _ := state.Validators.GetByIndex(0)
 		if bytes.Equal(privValidator.Address, addr) {
@@ -93,53 +124,75 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 		}
 	}
 
+	// Log whether this node is a validator or an observer
+	if state.Validators.HasAddress(privValidator.Address) {
+		consensusLogger.Info("This node is a validator")
+	} else {
+		consensusLogger.Info("This node is not a validator")
+	}
+
 	// Make BlockchainReactor
-	bcReactor := bc.NewBlockchainReactor(config, state.Copy(), proxyApp.Consensus(), blockStore, fastSync)
+	bcReactor := bc.NewBlockchainReactor(state.Copy(), proxyApp.Consensus(), blockStore, fastSync)
+	bcReactor.SetLogger(logger.With("module", "blockchain"))
 
 	// Make MempoolReactor
-	mempool := mempl.NewMempool(config, proxyApp.Mempool())
-	mempoolReactor := mempl.NewMempoolReactor(config, mempool)
+	mempoolLogger := logger.With("module", "mempool")
+	mempool := mempl.NewMempool(config.Mempool, proxyApp.Mempool())
+	mempool.SetLogger(mempoolLogger)
+	mempoolReactor := mempl.NewMempoolReactor(config.Mempool, mempool)
+	mempoolReactor.SetLogger(mempoolLogger)
 
 	// Make ConsensusReactor
-	consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
+	consensusState := consensus.NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
+	consensusState.SetLogger(consensusLogger)
 	if privValidator != nil {
 		consensusState.SetPrivValidator(privValidator)
 	}
 	consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync)
+	consensusReactor.SetLogger(consensusLogger)
 
-	// Make p2p network switch
-	sw := p2p.NewSwitch(config.GetConfig("p2p"))
+	p2pLogger := logger.With("module", "p2p")
+
+	sw := p2p.NewSwitch(config.P2P)
+	sw.SetLogger(p2pLogger)
 	sw.AddReactor("MEMPOOL", mempoolReactor)
 	sw.AddReactor("BLOCKCHAIN", bcReactor)
 	sw.AddReactor("CONSENSUS", consensusReactor)
 
 	// Optionally, start the pex reactor
-	// TODO: this is a dev feature, it needs some love
-	if config.GetBool("pex_reactor") {
-		addrBook := p2p.NewAddrBook(config.GetString("addrbook_file"), config.GetBool("addrbook_strict"))
-		addrBook.Start()
+	var addrBook *p2p.AddrBook
+	if config.P2P.PexReactor {
+		addrBook = p2p.NewAddrBook(config.P2P.AddrBookFile(), config.P2P.AddrBookStrict)
+		addrBook.SetLogger(p2pLogger.With("book", config.P2P.AddrBookFile()))
 		pexReactor := p2p.NewPEXReactor(addrBook)
+		pexReactor.SetLogger(p2pLogger)
 		sw.AddReactor("PEX", pexReactor)
 	}
 
-	// filter peers by addr or pubkey with a abci query.
-	// if the query return code is OK, add peer
-	// XXX: query format subject to change
-	if config.GetBool("filter_peers") {
+	// Filter peers by addr or pubkey with an ABCI query.
+	// If the query return code is OK, add peer.
+	// XXX: Query format subject to change
+	if config.FilterPeers {
 		// NOTE: addr is ip:port
 		sw.SetAddrFilter(func(addr net.Addr) error {
-			res := proxyApp.Query().QuerySync([]byte(Fmt("p2p/filter/addr/%s", addr.String())))
-			if res.IsOK() {
+			resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/addr/%s", addr.String())})
+			if err != nil {
+				return err
+			}
+			if resQuery.Code.IsOK() {
 				return nil
 			}
-			return res
+			return errors.New(resQuery.Code.String())
 		})
 		sw.SetPubKeyFilter(func(pubkey crypto.PubKeyEd25519) error {
-			res := proxyApp.Query().QuerySync([]byte(Fmt("p2p/filter/pubkey/%X", pubkey.Bytes())))
-			if res.IsOK() {
+			resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/pubkey/%X", pubkey.Bytes())})
+			if err != nil {
+				return err
+			}
+			if resQuery.Code.IsOK() {
 				return nil
 			}
-			return res
+			return errors.New(resQuery.Code.String())
 		})
 	}
 
@@ -148,42 +201,91 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	SetEventSwitch(eventSwitch, bcReactor, mempoolReactor, consensusReactor)
 
 	// run the profile server
-	profileHost := config.GetString("prof_laddr")
+	profileHost := config.ProfListenAddress
 	if profileHost != "" {
 
 		go func() {
-			log.Warn("Profile server", "error", http.ListenAndServe(profileHost, nil))
+			logger.Error("Profile server", "err", http.ListenAndServe(profileHost, nil))
 		}()
 	}
 
-	return &Node{
-		config:           config,
-		sw:               sw,
+	node := &Node{
+		config:        config,
+		genesisDoc:    state.GenesisDoc,
+		privValidator: privValidator,
+
+		privKey:  privKey,
+		sw:       sw,
+		addrBook: addrBook,
+
 		evsw:             eventSwitch,
 		blockStore:       blockStore,
 		bcReactor:        bcReactor,
 		mempoolReactor:   mempoolReactor,
 		consensusState:   consensusState,
 		consensusReactor: consensusReactor,
-		privValidator:    privValidator,
-		genesisDoc:       state.GenesisDoc,
-		privKey:          privKey,
 		proxyApp:         proxyApp,
+		txIndexer:        txIndexer,
+	}
+	node.BaseService = *cmn.NewBaseService(logger, "Node", node)
+	return node
+}
+
+func (n *Node) OnStart() error {
+	// Create & add listener
+	protocol, address := ProtocolAndAddress(n.config.P2P.ListenAddress)
+	l := p2p.NewDefaultListener(protocol, address, n.config.P2P.SkipUPNP, n.Logger.With("module", "p2p"))
+	n.sw.AddListener(l)
+
+	// Start the switch
+	n.sw.SetNodeInfo(n.makeNodeInfo())
+	n.sw.SetNodePrivKey(n.privKey)
+	_, err := n.sw.Start()
+	if err != nil {
+		return err
+	}
+
+	// If seeds exist, add them to the address book and dial out
+	if n.config.P2P.Seeds != "" {
+		// dial out
+		seeds := strings.Split(n.config.P2P.Seeds, ",")
+		if err := n.DialSeeds(seeds); err != nil {
+			return err
+		}
+	}
+
+	// Run the RPC server
+	if n.config.RPC.ListenAddress != "" {
+		listeners, err := n.startRPC()
+		if err != nil {
+			return err
+		}
+		n.rpcListeners = listeners
+	}
+
+	return nil
+}
+
+func (n *Node) OnStop() {
+	n.BaseService.OnStop()
+
+	n.Logger.Info("Stopping Node")
+	// TODO: gracefully disconnect from peers.
+	n.sw.Stop()
+
+	for _, l := range n.rpcListeners {
+		n.Logger.Info("Closing rpc listener", "listener", l)
+		if err := l.Close(); err != nil {
+			n.Logger.Error("Error closing listener", "listener", l, "err", err)
+		}
 	}
 }
 
-// Call Start() after adding the listeners.
-func (n *Node) Start() error {
-	n.sw.SetNodeInfo(makeNodeInfo(n.config, n.sw, n.privKey))
-	n.sw.SetNodePrivKey(n.privKey)
-	_, err := n.sw.Start()
-	return err
-}
-
-func (n *Node) Stop() {
-	log.Notice("Stopping Node")
-	// TODO: gracefully disconnect from peers.
-	n.sw.Stop()
+func (n *Node) RunForever() {
+	// Sleep forever and then...
+	cmn.TrapSignal(func() {
+		n.Stop()
+	})
 }
 
 // Add the event switch to reactors, mempool, etc.
@@ -197,13 +299,12 @@ func SetEventSwitch(evsw types.EventSwitch, eventables ...types.Eventable) {
 // Add listeners before starting the Node.
 // The first listener is the primary listener (in NodeInfo)
 func (n *Node) AddListener(l p2p.Listener) {
-	log.Notice(Fmt("Added %v", l))
 	n.sw.AddListener(l)
 }
 
-func (n *Node) StartRPC() ([]net.Listener, error) {
-	rpccore.SetConfig(n.config)
-
+// ConfigureRPC sets all variables in rpccore so they will serve
+// rpc calls from this node
+func (n *Node) ConfigureRPC() {
 	rpccore.SetEventSwitch(n.evsw)
 	rpccore.SetBlockStore(n.blockStore)
 	rpccore.SetConsensusState(n.consensusState)
@@ -211,18 +312,30 @@ func (n *Node) StartRPC() ([]net.Listener, error) {
 	rpccore.SetSwitch(n.sw)
 	rpccore.SetPubKey(n.privValidator.PubKey)
 	rpccore.SetGenesisDoc(n.genesisDoc)
+	rpccore.SetAddrBook(n.addrBook)
 	rpccore.SetProxyAppQuery(n.proxyApp.Query())
+	rpccore.SetTxIndexer(n.txIndexer)
+	rpccore.SetLogger(n.Logger.With("module", "rpc"))
+}
 
-	listenAddrs := strings.Split(n.config.GetString("rpc_laddr"), ",")
+func (n *Node) startRPC() ([]net.Listener, error) {
+	n.ConfigureRPC()
+	listenAddrs := strings.Split(n.config.RPC.ListenAddress, ",")
+
+	if n.config.RPC.Unsafe {
+		rpccore.AddUnsafeRoutes()
+	}
 
 	// we may expose the rpc over both a unix and tcp socket
 	listeners := make([]net.Listener, len(listenAddrs))
 	for i, listenAddr := range listenAddrs {
 		mux := http.NewServeMux()
 		wm := rpcserver.NewWebsocketManager(rpccore.Routes, n.evsw)
+		rpcLogger := n.Logger.With("module", "rpc-server")
+		wm.SetLogger(rpcLogger)
 		mux.HandleFunc("/websocket", wm.WebsocketHandler)
-		rpcserver.RegisterRPCFuncs(mux, rpccore.Routes)
-		listener, err := rpcserver.StartHTTPServer(listenAddr, mux)
+		rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, rpcLogger)
+		listener, err := rpcserver.StartHTTPServer(listenAddr, mux, rpcLogger)
 		if err != nil {
 			return nil, err
 		}
@@ -230,7 +343,7 @@ func (n *Node) StartRPC() ([]net.Listener, error) {
 	}
 
 	// we expose a simplified api over grpc for convenience to app devs
-	grpcListenAddr := n.config.GetString("grpc_laddr")
+	grpcListenAddr := n.config.RPC.GRPCListenAddress
 	if grpcListenAddr != "" {
 		listener, err := grpccore.StartGRPCServer(grpcListenAddr)
 		if err != nil {
@@ -279,165 +392,57 @@ func (n *Node) ProxyApp() proxy.AppConns {
 	return n.proxyApp
 }
 
-func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd25519) *p2p.NodeInfo {
+func (n *Node) makeNodeInfo() *p2p.NodeInfo {
+	txIndexerStatus := "on"
+	if _, ok := n.txIndexer.(*null.TxIndex); ok {
+		txIndexerStatus = "off"
+	}
 
 	nodeInfo := &p2p.NodeInfo{
-		PubKey:  privKey.PubKey().(crypto.PubKeyEd25519),
-		Moniker: config.GetString("moniker"),
-		Network: config.GetString("chain_id"),
+		PubKey:  n.privKey.PubKey().Unwrap().(crypto.PubKeyEd25519),
+		Moniker: n.config.Moniker,
+		Network: n.consensusState.GetState().ChainID,
 		Version: version.Version,
 		Other: []string{
-			Fmt("wire_version=%v", wire.Version),
-			Fmt("p2p_version=%v", p2p.Version),
-			Fmt("consensus_version=%v", consensus.Version),
-			Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version),
+			cmn.Fmt("wire_version=%v", wire.Version),
+			cmn.Fmt("p2p_version=%v", p2p.Version),
+			cmn.Fmt("consensus_version=%v", consensus.Version),
+			cmn.Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version),
+			cmn.Fmt("tx_index=%v", txIndexerStatus),
 		},
 	}
 
 	// include git hash in the nodeInfo if available
-	if rev, err := ReadFile(config.GetString("revision_file")); err == nil {
-		nodeInfo.Other = append(nodeInfo.Other, Fmt("revision=%v", string(rev)))
-	}
+	// TODO: use ld-flags
+	/*if rev, err := cmn.ReadFile(n.config.GetString("revision_file")); err == nil {
+		nodeInfo.Other = append(nodeInfo.Other, cmn.Fmt("revision=%v", string(rev)))
+	}*/
 
-	if !sw.IsListening() {
+	if !n.sw.IsListening() {
 		return nodeInfo
 	}
 
-	p2pListener := sw.Listeners()[0]
+	p2pListener := n.sw.Listeners()[0]
 	p2pHost := p2pListener.ExternalAddress().IP.String()
 	p2pPort := p2pListener.ExternalAddress().Port
-	rpcListenAddr := config.GetString("rpc_laddr")
+	rpcListenAddr := n.config.RPC.ListenAddress
 
 	// We assume that the rpcListener has the same ExternalAddress.
 	// This is probably true because both P2P and RPC listeners use UPnP,
 	// except of course if the rpc is only bound to localhost
-	nodeInfo.ListenAddr = Fmt("%v:%v", p2pHost, p2pPort)
-	nodeInfo.Other = append(nodeInfo.Other, Fmt("rpc_addr=%v", rpcListenAddr))
+	nodeInfo.ListenAddr = cmn.Fmt("%v:%v", p2pHost, p2pPort)
+	nodeInfo.Other = append(nodeInfo.Other, cmn.Fmt("rpc_addr=%v", rpcListenAddr))
 	return nodeInfo
 }
 
 //------------------------------------------------------------------------------
 
-// Users wishing to:
-//	* use an external signer for their validators
-//	* supply an in-proc abci app
-// should fork tendermint/tendermint and implement RunNode to
-// call NewNode with their custom priv validator and/or custom
-// proxy.ClientCreator interface
-func RunNode(config cfg.Config) {
-	// Wait until the genesis doc becomes available
-	genDocFile := config.GetString("genesis_file")
-	if !FileExists(genDocFile) {
-		log.Notice(Fmt("Waiting for genesis file %v...", genDocFile))
-		for {
-			time.Sleep(time.Second)
-			if !FileExists(genDocFile) {
-				continue
-			}
-			jsonBlob, err := ioutil.ReadFile(genDocFile)
-			if err != nil {
-				Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
-			}
-			genDoc := types.GenesisDocFromJSON(jsonBlob)
-			if genDoc.ChainID == "" {
-				PanicSanity(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
-			}
-			config.Set("chain_id", genDoc.ChainID)
-		}
-	}
-
-	// Create & start node
-	n := NewNodeDefault(config)
-
-	protocol, address := ProtocolAndAddress(config.GetString("node_laddr"))
-	l := p2p.NewDefaultListener(protocol, address, config.GetBool("skip_upnp"))
-	n.AddListener(l)
-	err := n.Start()
-	if err != nil {
-		Exit(Fmt("Failed to start node: %v", err))
-	}
-
-	log.Notice("Started node", "nodeInfo", n.sw.NodeInfo())
-
-	// If seedNode is provided by config, dial out.
-	if config.GetString("seeds") != "" {
-		seeds := strings.Split(config.GetString("seeds"), ",")
-		n.sw.DialSeeds(seeds)
-	}
-
-	// Run the RPC server.
-	if config.GetString("rpc_laddr") != "" {
-		_, err := n.StartRPC()
-		if err != nil {
-			PanicCrisis(err)
-		}
-	}
-
-	// Sleep forever and then...
-	TrapSignal(func() {
-		n.Stop()
-	})
-}
-
 func (n *Node) NodeInfo() *p2p.NodeInfo {
 	return n.sw.NodeInfo()
 }
 
-func (n *Node) DialSeeds(seeds []string) {
-	n.sw.DialSeeds(seeds)
-}
-
-//------------------------------------------------------------------------------
-// replay
-
-// convenience for replay mode
-func newConsensusState(config cfg.Config) *consensus.ConsensusState {
-	// Get BlockStore
-	blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir"))
-	blockStore := bc.NewBlockStore(blockStoreDB)
-
-	// Get State
-	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
-	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
-
-	// Create proxyAppConn connection (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore))
-	_, err := proxyApp.Start()
-	if err != nil {
-		Exit(Fmt("Error starting proxy app conns: %v", err))
-	}
-
-	// add the chainid to the global config
-	config.Set("chain_id", state.ChainID)
-
-	// Make event switch
-	eventSwitch := types.NewEventSwitch()
-	if _, err := eventSwitch.Start(); err != nil {
-		Exit(Fmt("Failed to start event switch: %v", err))
-	}
-
-	mempool := mempl.NewMempool(config, proxyApp.Mempool())
-
-	consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
-	consensusState.SetEventSwitch(eventSwitch)
-	return consensusState
-}
-
-func RunReplayConsole(config cfg.Config, walFile string) {
-	consensusState := newConsensusState(config)
-
-	if err := consensusState.ReplayConsole(walFile); err != nil {
-		Exit(Fmt("Error during consensus replay: %v", err))
-	}
-}
-
-func RunReplay(config cfg.Config, walFile string) {
-	consensusState := newConsensusState(config)
-
-	if err := consensusState.ReplayMessages(walFile); err != nil {
-		Exit(Fmt("Error during consensus replay: %v", err))
-	}
-	log.Notice("Replay run successfully")
+func (n *Node) DialSeeds(seeds []string) error {
+	return n.sw.DialSeeds(n.addrBook, seeds)
 }
 
 // Defaults to tcp
@@ -449,3 +454,5 @@ func ProtocolAndAddress(listenAddr string) (string, string) {
 	}
 	return protocol, address
 }
+
+//------------------------------------------------------------------------------

node/node_test.go

@@ -4,21 +4,22 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/tendermint/config/tendermint_test"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tmlibs/log"
 )
 
 func TestNodeStartStop(t *testing.T) {
-	config := tendermint_test.ResetConfig("node_node_test")
+	config := cfg.ResetTestRoot("node_node_test")
 
 	// Create & start node
-	n := NewNodeDefault(config)
-	protocol, address := ProtocolAndAddress(config.GetString("node_laddr"))
-	l := p2p.NewDefaultListener(protocol, address, config.GetBool("skip_upnp"))
-	n.AddListener(l)
+	n := NewNodeDefault(config, log.TestingLogger())
 	n.Start()
-	log.Notice("Started node", "nodeInfo", n.sw.NodeInfo())
+	t.Logf("Started node %v", n.sw.NodeInfo())
+
+	// Wait a bit to initialize
+	// TODO remove time.Sleep(), make asynchronous.
 	time.Sleep(time.Second * 2)
+
 	ch := make(chan struct{}, 1)
 	go func() {
 		n.Stop()

p2p/CHANGELOG.md

@@ -0,0 +1,78 @@
+# Changelog
+
+## 0.5.0 (April 21, 2017)
+
+BREAKING CHANGES: 
+
+- Remove or unexport methods from FuzzedConnection: Active, Mode, ProbDropRW, ProbDropConn, ProbSleep, MaxDelayMilliseconds, Fuzz
+- switch.AddPeerWithConnection is unexported and replaced by switch.AddPeer
+- switch.DialPeerWithAddress takes a bool, setting the peer as persistent or not
+
+FEATURES:
+
+- Persistent peers: any peer considered a "seed" will be reconnected to when the connection is dropped
+
+
+IMPROVEMENTS:
+
+- Many more tests and comments
+- Refactor configurations for less dependence on go-config. Introduces new structs PeerConfig, MConnConfig, FuzzConnConfig
+- New methods on peer: CloseConn, HandshakeTimeout, IsPersistent, Addr, PubKey
+- NewNetAddress supports a testing mode where the address defaults to 0.0.0.0:0
+
+
+## 0.4.0 (March 6, 2017)
+
+BREAKING CHANGES: 
+
+- DialSeeds now takes an AddrBook and returns an error: `DialSeeds(*AddrBook, []string) error`
+- NewNetAddressString now returns an error: `NewNetAddressString(string) (*NetAddress, error)`
+
+FEATURES:
+
+- `NewNetAddressStrings([]string) ([]*NetAddress, error)`
+- `AddrBook.Save()`
+
+IMPROVEMENTS:
+
+- PexReactor responsible for starting and stopping the AddrBook
+
+BUG FIXES:
+
+- DialSeeds returns an error instead of panicking on bad addresses
+
+## 0.3.5 (January 12, 2017)
+
+FEATURES
+
+- Toggle strict routability in the AddrBook 
+
+BUG FIXES
+
+- Close filtered out connections
+- Fixes for MakeConnectedSwitches and Connect2Switches
+
+## 0.3.4 (August 10, 2016)
+
+FEATURES:
+
+- Optionally filter connections by address or public key
+
+## 0.3.3 (May 12, 2016)
+
+FEATURES:
+
+- FuzzConn
+
+## 0.3.2 (March 12, 2016)
+
+IMPROVEMENTS:
+
+- Memory optimizations
+
+## 0.3.1 ()
+
+FEATURES: 
+
+- Configurable parameters
+

p2p/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:latest
+
+RUN curl https://glide.sh/get | sh
+
+RUN mkdir -p /go/src/github.com/tendermint/tendermint/p2p
+WORKDIR /go/src/github.com/tendermint/tendermint/p2p
+
+COPY glide.yaml /go/src/github.com/tendermint/tendermint/p2p/
+COPY glide.lock /go/src/github.com/tendermint/tendermint/p2p/
+
+RUN glide install
+
+COPY . /go/src/github.com/tendermint/tendermint/p2p

p2p/README.md

@@ -0,0 +1,79 @@
+# `tendermint/tendermint/p2p`
+
+[![CircleCI](https://circleci.com/gh/tendermint/tendermint/p2p.svg?style=svg)](https://circleci.com/gh/tendermint/tendermint/p2p)
+
+`tendermint/tendermint/p2p` provides an abstraction around peer-to-peer communication.<br/>
+
+## Peer/MConnection/Channel
+
+Each peer has one `MConnection` (multiplex connection) instance.
+
+__multiplex__ *noun* a system or signal involving simultaneous transmission of
+several messages along a single channel of communication.
+
+Each `MConnection` handles message transmission on multiple abstract communication
+`Channel`s.  Each channel has a globally unique byte id.
+The byte id and the relative priorities of each `Channel` are configured upon
+initialization of the connection.
+
+There are two methods for sending messages:
+```go
+func (m MConnection) Send(chID byte, msg interface{}) bool {}
+func (m MConnection) TrySend(chID byte, msg interface{}) bool {}
+```
+
+`Send(chID, msg)` is a blocking call that waits until `msg` is successfully queued
+for the channel with the given id byte `chID`.  The message `msg` is serialized
+using the `tendermint/wire` submodule's `WriteBinary()` reflection routine.
+
+`TrySend(chID, msg)` is a nonblocking call that returns false if the channel's
+queue is full.
+
+`Send()` and `TrySend()` are also exposed for each `Peer`.
+
+## Switch/Reactor
+
+The `Switch` handles peer connections and exposes an API to receive incoming messages
+on `Reactors`.  Each `Reactor` is responsible for handling incoming messages of one
+or more `Channels`.  So while sending outgoing messages is typically performed on the peer,
+incoming messages are received on the reactor.
+
+```go
+// Declare a MyReactor reactor that handles messages on MyChannelID.
+type MyReactor struct{}
+
+func (reactor MyReactor) GetChannels() []*ChannelDescriptor {
+    return []*ChannelDescriptor{ChannelDescriptor{ID:MyChannelID, Priority: 1}}
+}
+
+func (reactor MyReactor) Receive(chID byte, peer *Peer, msgBytes []byte) {
+    r, n, err := bytes.NewBuffer(msgBytes), new(int64), new(error)
+    msgString := ReadString(r, n, err)
+    fmt.Println(msgString)
+}
+
+// Other Reactor methods omitted for brevity
+...
+
+switch := NewSwitch([]Reactor{MyReactor{}})
+
+...
+
+// Send a random message to all outbound connections
+for _, peer := range switch.Peers().List() {
+    if peer.IsOutbound() {
+        peer.Send(MyChannelID, "Here's a random message")
+    }
+}
+```
+
+### PexReactor/AddrBook
+
+A `PEXReactor` reactor implementation is provided to automate peer discovery.
+
+```go
+book := p2p.NewAddrBook(addrBookFilePath)
+pexReactor := p2p.NewPEXReactor(book)
+...
+switch := NewSwitch([]Reactor{pexReactor, myReactor, ...})
+```

p2p/addrbook.go

@@ -0,0 +1,838 @@
+// Modified for Tendermint
+// Originally Copyright (c) 2013-2014 Conformal Systems LLC.
+// https://github.com/conformal/btcd/blob/master/LICENSE
+
+package p2p
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"math"
+	"math/rand"
+	"net"
+	"os"
+	"sync"
+	"time"
+
+	crypto "github.com/tendermint/go-crypto"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+const (
+	// addresses under which the address manager will claim to need more addresses.
+	needAddressThreshold = 1000
+
+	// interval used to dump the address cache to disk for future use.
+	dumpAddressInterval = time.Minute * 2
+
+	// max addresses in each old address bucket.
+	oldBucketSize = 64
+
+	// buckets we split old addresses over.
+	oldBucketCount = 64
+
+	// max addresses in each new address bucket.
+	newBucketSize = 64
+
+	// buckets that we spread new addresses over.
+	newBucketCount = 256
+
+	// old buckets over which an address group will be spread.
+	oldBucketsPerGroup = 4
+
+	// new buckets over which an source address group will be spread.
+	newBucketsPerGroup = 32
+
+	// buckets a frequently seen new address may end up in.
+	maxNewBucketsPerAddress = 4
+
+	// days before which we assume an address has vanished
+	// if we have not seen it announced in that long.
+	numMissingDays = 30
+
+	// tries without a single success before we assume an address is bad.
+	numRetries = 3
+
+	// max failures we will accept without a success before considering an address bad.
+	maxFailures = 10
+
+	// days since the last success before we will consider evicting an address.
+	minBadDays = 7
+
+	// % of total addresses known returned by GetSelection.
+	getSelectionPercent = 23
+
+	// min addresses that must be returned by GetSelection. Useful for bootstrapping.
+	minGetSelection = 32
+
+	// max addresses returned by GetSelection
+	// NOTE: this must match "maxPexMessageSize"
+	maxGetSelection = 250
+)
+
+const (
+	bucketTypeNew = 0x01
+	bucketTypeOld = 0x02
+)
+
+// AddrBook - concurrency safe peer address manager.
+type AddrBook struct {
+	cmn.BaseService
+
+	mtx               sync.Mutex
+	filePath          string
+	routabilityStrict bool
+	rand              *rand.Rand
+	key               string
+	ourAddrs          map[string]*NetAddress
+	addrLookup        map[string]*knownAddress // new & old
+	addrNew           []map[string]*knownAddress
+	addrOld           []map[string]*knownAddress
+	wg                sync.WaitGroup
+	nOld              int
+	nNew              int
+}
+
+// NewAddrBook creates a new address book.
+// Use Start to begin processing asynchronous address updates.
+func NewAddrBook(filePath string, routabilityStrict bool) *AddrBook {
+	am := &AddrBook{
+		rand:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		ourAddrs:          make(map[string]*NetAddress),
+		addrLookup:        make(map[string]*knownAddress),
+		filePath:          filePath,
+		routabilityStrict: routabilityStrict,
+	}
+	am.init()
+	am.BaseService = *cmn.NewBaseService(nil, "AddrBook", am)
+	return am
+}
+
+// When modifying this, don't forget to update loadFromFile()
+func (a *AddrBook) init() {
+	a.key = crypto.CRandHex(24) // 24/2 * 8 = 96 bits
+	// New addr buckets
+	a.addrNew = make([]map[string]*knownAddress, newBucketCount)
+	for i := range a.addrNew {
+		a.addrNew[i] = make(map[string]*knownAddress)
+	}
+	// Old addr buckets
+	a.addrOld = make([]map[string]*knownAddress, oldBucketCount)
+	for i := range a.addrOld {
+		a.addrOld[i] = make(map[string]*knownAddress)
+	}
+}
+
+// OnStart implements Service.
+func (a *AddrBook) OnStart() error {
+	a.BaseService.OnStart()
+	a.loadFromFile(a.filePath)
+	a.wg.Add(1)
+	go a.saveRoutine()
+	return nil
+}
+
+// OnStop implements Service.
+func (a *AddrBook) OnStop() {
+	a.BaseService.OnStop()
+}
+
+func (a *AddrBook) Wait() {
+	a.wg.Wait()
+}
+
+func (a *AddrBook) AddOurAddress(addr *NetAddress) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	a.Logger.Info("Add our address to book", "addr", addr)
+	a.ourAddrs[addr.String()] = addr
+}
+
+func (a *AddrBook) OurAddresses() []*NetAddress {
+	addrs := []*NetAddress{}
+	for _, addr := range a.ourAddrs {
+		addrs = append(addrs, addr)
+	}
+	return addrs
+}
+
+// NOTE: addr must not be nil
+func (a *AddrBook) AddAddress(addr *NetAddress, src *NetAddress) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	a.Logger.Info("Add address to book", "addr", addr, "src", src)
+	a.addAddress(addr, src)
+}
+
+func (a *AddrBook) NeedMoreAddrs() bool {
+	return a.Size() < needAddressThreshold
+}
+
+func (a *AddrBook) Size() int {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	return a.size()
+}
+
+func (a *AddrBook) size() int {
+	return a.nNew + a.nOld
+}
+
+// Pick an address to connect to with new/old bias.
+func (a *AddrBook) PickAddress(newBias int) *NetAddress {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+
+	if a.size() == 0 {
+		return nil
+	}
+	if newBias > 100 {
+		newBias = 100
+	}
+	if newBias < 0 {
+		newBias = 0
+	}
+
+	// Bias between new and old addresses.
+	oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(newBias))
+	newCorrelation := math.Sqrt(float64(a.nNew)) * float64(newBias)
+
+	if (newCorrelation+oldCorrelation)*a.rand.Float64() < oldCorrelation {
+		// pick random Old bucket.
+		var bucket map[string]*knownAddress = nil
+		for len(bucket) == 0 {
+			bucket = a.addrOld[a.rand.Intn(len(a.addrOld))]
+		}
+		// pick a random ka from bucket.
+		randIndex := a.rand.Intn(len(bucket))
+		for _, ka := range bucket {
+			if randIndex == 0 {
+				return ka.Addr
+			}
+			randIndex--
+		}
+		cmn.PanicSanity("Should not happen")
+	} else {
+		// pick random New bucket.
+		var bucket map[string]*knownAddress = nil
+		for len(bucket) == 0 {
+			bucket = a.addrNew[a.rand.Intn(len(a.addrNew))]
+		}
+		// pick a random ka from bucket.
+		randIndex := a.rand.Intn(len(bucket))
+		for _, ka := range bucket {
+			if randIndex == 0 {
+				return ka.Addr
+			}
+			randIndex--
+		}
+		cmn.PanicSanity("Should not happen")
+	}
+	return nil
+}
+
+func (a *AddrBook) MarkGood(addr *NetAddress) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	ka := a.addrLookup[addr.String()]
+	if ka == nil {
+		return
+	}
+	ka.markGood()
+	if ka.isNew() {
+		a.moveToOld(ka)
+	}
+}
+
+func (a *AddrBook) MarkAttempt(addr *NetAddress) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	ka := a.addrLookup[addr.String()]
+	if ka == nil {
+		return
+	}
+	ka.markAttempt()
+}
+
+// MarkBad currently just ejects the address. In the future, consider
+// blacklisting.
+func (a *AddrBook) MarkBad(addr *NetAddress) {
+	a.RemoveAddress(addr)
+}
+
+// RemoveAddress removes the address from the book.
+func (a *AddrBook) RemoveAddress(addr *NetAddress) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	ka := a.addrLookup[addr.String()]
+	if ka == nil {
+		return
+	}
+	a.Logger.Info("Remove address from book", "addr", addr)
+	a.removeFromAllBuckets(ka)
+}
+
+/* Peer exchange */
+
+// GetSelection randomly selects some addresses (old & new). Suitable for peer-exchange protocols.
+func (a *AddrBook) GetSelection() []*NetAddress {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+
+	if a.size() == 0 {
+		return nil
+	}
+
+	allAddr := make([]*NetAddress, a.size())
+	i := 0
+	for _, v := range a.addrLookup {
+		allAddr[i] = v.Addr
+		i++
+	}
+
+	numAddresses := cmn.MaxInt(
+		cmn.MinInt(minGetSelection, len(allAddr)),
+		len(allAddr)*getSelectionPercent/100)
+	numAddresses = cmn.MinInt(maxGetSelection, numAddresses)
+
+	// Fisher-Yates shuffle the array. We only need to do the first
+	// `numAddresses' since we are throwing the rest.
+	for i := 0; i < numAddresses; i++ {
+		// pick a number between current index and the end
+		j := rand.Intn(len(allAddr)-i) + i
+		allAddr[i], allAddr[j] = allAddr[j], allAddr[i]
+	}
+
+	// slice off the limit we are willing to share.
+	return allAddr[:numAddresses]
+}
+
+/* Loading & Saving */
+
+type addrBookJSON struct {
+	Key   string
+	Addrs []*knownAddress
+}
+
+func (a *AddrBook) saveToFile(filePath string) {
+	a.Logger.Info("Saving AddrBook to file", "size", a.Size())
+
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+	// Compile Addrs
+	addrs := []*knownAddress{}
+	for _, ka := range a.addrLookup {
+		addrs = append(addrs, ka)
+	}
+
+	aJSON := &addrBookJSON{
+		Key:   a.key,
+		Addrs: addrs,
+	}
+
+	jsonBytes, err := json.MarshalIndent(aJSON, "", "\t")
+	if err != nil {
+		a.Logger.Error("Failed to save AddrBook to file", "err", err)
+		return
+	}
+	err = cmn.WriteFileAtomic(filePath, jsonBytes, 0644)
+	if err != nil {
+		a.Logger.Error("Failed to save AddrBook to file", "file", filePath, "err", err)
+	}
+}
+
+// Returns false if file does not exist.
+// cmn.Panics if file is corrupt.
+func (a *AddrBook) loadFromFile(filePath string) bool {
+	// If doesn't exist, do nothing.
+	_, err := os.Stat(filePath)
+	if os.IsNotExist(err) {
+		return false
+	}
+
+	// Load addrBookJSON{}
+	r, err := os.Open(filePath)
+	if err != nil {
+		cmn.PanicCrisis(cmn.Fmt("Error opening file %s: %v", filePath, err))
+	}
+	defer r.Close()
+	aJSON := &addrBookJSON{}
+	dec := json.NewDecoder(r)
+	err = dec.Decode(aJSON)
+	if err != nil {
+		cmn.PanicCrisis(cmn.Fmt("Error reading file %s: %v", filePath, err))
+	}
+
+	// Restore all the fields...
+	// Restore the key
+	a.key = aJSON.Key
+	// Restore .addrNew & .addrOld
+	for _, ka := range aJSON.Addrs {
+		for _, bucketIndex := range ka.Buckets {
+			bucket := a.getBucket(ka.BucketType, bucketIndex)
+			bucket[ka.Addr.String()] = ka
+		}
+		a.addrLookup[ka.Addr.String()] = ka
+		if ka.BucketType == bucketTypeNew {
+			a.nNew++
+		} else {
+			a.nOld++
+		}
+	}
+	return true
+}
+
+// Save saves the book.
+func (a *AddrBook) Save() {
+	a.Logger.Info("Saving AddrBook to file", "size", a.Size())
+	a.saveToFile(a.filePath)
+}
+
+/* Private methods */
+
+func (a *AddrBook) saveRoutine() {
+	dumpAddressTicker := time.NewTicker(dumpAddressInterval)
+out:
+	for {
+		select {
+		case <-dumpAddressTicker.C:
+			a.saveToFile(a.filePath)
+		case <-a.Quit:
+			break out
+		}
+	}
+	dumpAddressTicker.Stop()
+	a.saveToFile(a.filePath)
+	a.wg.Done()
+	a.Logger.Info("Address handler done")
+}
+
+func (a *AddrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAddress {
+	switch bucketType {
+	case bucketTypeNew:
+		return a.addrNew[bucketIdx]
+	case bucketTypeOld:
+		return a.addrOld[bucketIdx]
+	default:
+		cmn.PanicSanity("Should not happen")
+		return nil
+	}
+}
+
+// Adds ka to new bucket. Returns false if it couldn't do it cuz buckets full.
+// NOTE: currently it always returns true.
+func (a *AddrBook) addToNewBucket(ka *knownAddress, bucketIdx int) bool {
+	// Sanity check
+	if ka.isOld() {
+		a.Logger.Error(cmn.Fmt("Cannot add address already in old bucket to a new bucket: %v", ka))
+		return false
+	}
+
+	addrStr := ka.Addr.String()
+	bucket := a.getBucket(bucketTypeNew, bucketIdx)
+
+	// Already exists?
+	if _, ok := bucket[addrStr]; ok {
+		return true
+	}
+
+	// Enforce max addresses.
+	if len(bucket) > newBucketSize {
+		a.Logger.Info("new bucket is full, expiring old ")
+		a.expireNew(bucketIdx)
+	}
+
+	// Add to bucket.
+	bucket[addrStr] = ka
+	if ka.addBucketRef(bucketIdx) == 1 {
+		a.nNew++
+	}
+
+	// Ensure in addrLookup
+	a.addrLookup[addrStr] = ka
+
+	return true
+}
+
+// Adds ka to old bucket. Returns false if it couldn't do it cuz buckets full.
+func (a *AddrBook) addToOldBucket(ka *knownAddress, bucketIdx int) bool {
+	// Sanity check
+	if ka.isNew() {
+		a.Logger.Error(cmn.Fmt("Cannot add new address to old bucket: %v", ka))
+		return false
+	}
+	if len(ka.Buckets) != 0 {
+		a.Logger.Error(cmn.Fmt("Cannot add already old address to another old bucket: %v", ka))
+		return false
+	}
+
+	addrStr := ka.Addr.String()
+	bucket := a.getBucket(bucketTypeNew, bucketIdx)
+
+	// Already exists?
+	if _, ok := bucket[addrStr]; ok {
+		return true
+	}
+
+	// Enforce max addresses.
+	if len(bucket) > oldBucketSize {
+		return false
+	}
+
+	// Add to bucket.
+	bucket[addrStr] = ka
+	if ka.addBucketRef(bucketIdx) == 1 {
+		a.nOld++
+	}
+
+	// Ensure in addrLookup
+	a.addrLookup[addrStr] = ka
+
+	return true
+}
+
+func (a *AddrBook) removeFromBucket(ka *knownAddress, bucketType byte, bucketIdx int) {
+	if ka.BucketType != bucketType {
+		a.Logger.Error(cmn.Fmt("Bucket type mismatch: %v", ka))
+		return
+	}
+	bucket := a.getBucket(bucketType, bucketIdx)
+	delete(bucket, ka.Addr.String())
+	if ka.removeBucketRef(bucketIdx) == 0 {
+		if bucketType == bucketTypeNew {
+			a.nNew--
+		} else {
+			a.nOld--
+		}
+		delete(a.addrLookup, ka.Addr.String())
+	}
+}
+
+func (a *AddrBook) removeFromAllBuckets(ka *knownAddress) {
+	for _, bucketIdx := range ka.Buckets {
+		bucket := a.getBucket(ka.BucketType, bucketIdx)
+		delete(bucket, ka.Addr.String())
+	}
+	ka.Buckets = nil
+	if ka.BucketType == bucketTypeNew {
+		a.nNew--
+	} else {
+		a.nOld--
+	}
+	delete(a.addrLookup, ka.Addr.String())
+}
+
+func (a *AddrBook) pickOldest(bucketType byte, bucketIdx int) *knownAddress {
+	bucket := a.getBucket(bucketType, bucketIdx)
+	var oldest *knownAddress
+	for _, ka := range bucket {
+		if oldest == nil || ka.LastAttempt.Before(oldest.LastAttempt) {
+			oldest = ka
+		}
+	}
+	return oldest
+}
+
+func (a *AddrBook) addAddress(addr, src *NetAddress) {
+	if a.routabilityStrict && !addr.Routable() {
+		a.Logger.Error(cmn.Fmt("Cannot add non-routable address %v", addr))
+		return
+	}
+	if _, ok := a.ourAddrs[addr.String()]; ok {
+		// Ignore our own listener address.
+		return
+	}
+
+	ka := a.addrLookup[addr.String()]
+
+	if ka != nil {
+		// Already old.
+		if ka.isOld() {
+			return
+		}
+		// Already in max new buckets.
+		if len(ka.Buckets) == maxNewBucketsPerAddress {
+			return
+		}
+		// The more entries we have, the less likely we are to add more.
+		factor := int32(2 * len(ka.Buckets))
+		if a.rand.Int31n(factor) != 0 {
+			return
+		}
+	} else {
+		ka = newKnownAddress(addr, src)
+	}
+
+	bucket := a.calcNewBucket(addr, src)
+	a.addToNewBucket(ka, bucket)
+
+	a.Logger.Info("Added new address", "address", addr, "total", a.size())
+}
+
+// Make space in the new buckets by expiring the really bad entries.
+// If no bad entries are available we remove the oldest.
+func (a *AddrBook) expireNew(bucketIdx int) {
+	for addrStr, ka := range a.addrNew[bucketIdx] {
+		// If an entry is bad, throw it away
+		if ka.isBad() {
+			a.Logger.Info(cmn.Fmt("expiring bad address %v", addrStr))
+			a.removeFromBucket(ka, bucketTypeNew, bucketIdx)
+			return
+		}
+	}
+
+	// If we haven't thrown out a bad entry, throw out the oldest entry
+	oldest := a.pickOldest(bucketTypeNew, bucketIdx)
+	a.removeFromBucket(oldest, bucketTypeNew, bucketIdx)
+}
+
+// Promotes an address from new to old.
+// TODO: Move to old probabilistically.
+// The better a node is, the less likely it should be evicted from an old bucket.
+func (a *AddrBook) moveToOld(ka *knownAddress) {
+	// Sanity check
+	if ka.isOld() {
+		a.Logger.Error(cmn.Fmt("Cannot promote address that is already old %v", ka))
+		return
+	}
+	if len(ka.Buckets) == 0 {
+		a.Logger.Error(cmn.Fmt("Cannot promote address that isn't in any new buckets %v", ka))
+		return
+	}
+
+	// Remember one of the buckets in which ka is in.
+	freedBucket := ka.Buckets[0]
+	// Remove from all (new) buckets.
+	a.removeFromAllBuckets(ka)
+	// It's officially old now.
+	ka.BucketType = bucketTypeOld
+
+	// Try to add it to its oldBucket destination.
+	oldBucketIdx := a.calcOldBucket(ka.Addr)
+	added := a.addToOldBucket(ka, oldBucketIdx)
+	if !added {
+		// No room, must evict something
+		oldest := a.pickOldest(bucketTypeOld, oldBucketIdx)
+		a.removeFromBucket(oldest, bucketTypeOld, oldBucketIdx)
+		// Find new bucket to put oldest in
+		newBucketIdx := a.calcNewBucket(oldest.Addr, oldest.Src)
+		added := a.addToNewBucket(oldest, newBucketIdx)
+		// No space in newBucket either, just put it in freedBucket from above.
+		if !added {
+			added := a.addToNewBucket(oldest, freedBucket)
+			if !added {
+				a.Logger.Error(cmn.Fmt("Could not migrate oldest %v to freedBucket %v", oldest, freedBucket))
+			}
+		}
+		// Finally, add to bucket again.
+		added = a.addToOldBucket(ka, oldBucketIdx)
+		if !added {
+			a.Logger.Error(cmn.Fmt("Could not re-add ka %v to oldBucketIdx %v", ka, oldBucketIdx))
+		}
+	}
+}
+
+// doublesha256(  key + sourcegroup +
+//                int64(doublesha256(key + group + sourcegroup))%bucket_per_group  ) % num_new_buckets
+func (a *AddrBook) calcNewBucket(addr, src *NetAddress) int {
+	data1 := []byte{}
+	data1 = append(data1, []byte(a.key)...)
+	data1 = append(data1, []byte(a.groupKey(addr))...)
+	data1 = append(data1, []byte(a.groupKey(src))...)
+	hash1 := doubleSha256(data1)
+	hash64 := binary.BigEndian.Uint64(hash1)
+	hash64 %= newBucketsPerGroup
+	var hashbuf [8]byte
+	binary.BigEndian.PutUint64(hashbuf[:], hash64)
+	data2 := []byte{}
+	data2 = append(data2, []byte(a.key)...)
+	data2 = append(data2, a.groupKey(src)...)
+	data2 = append(data2, hashbuf[:]...)
+
+	hash2 := doubleSha256(data2)
+	return int(binary.BigEndian.Uint64(hash2) % newBucketCount)
+}
+
+// doublesha256(  key + group +
+//                int64(doublesha256(key + addr))%buckets_per_group  ) % num_old_buckets
+func (a *AddrBook) calcOldBucket(addr *NetAddress) int {
+	data1 := []byte{}
+	data1 = append(data1, []byte(a.key)...)
+	data1 = append(data1, []byte(addr.String())...)
+	hash1 := doubleSha256(data1)
+	hash64 := binary.BigEndian.Uint64(hash1)
+	hash64 %= oldBucketsPerGroup
+	var hashbuf [8]byte
+	binary.BigEndian.PutUint64(hashbuf[:], hash64)
+	data2 := []byte{}
+	data2 = append(data2, []byte(a.key)...)
+	data2 = append(data2, a.groupKey(addr)...)
+	data2 = append(data2, hashbuf[:]...)
+
+	hash2 := doubleSha256(data2)
+	return int(binary.BigEndian.Uint64(hash2) % oldBucketCount)
+}
+
+// Return a string representing the network group of this address.
+// This is the /16 for IPv6, the /32 (/36 for he.net) for IPv6, the string
+// "local" for a local address and the string "unroutable for an unroutable
+// address.
+func (a *AddrBook) groupKey(na *NetAddress) string {
+	if a.routabilityStrict && na.Local() {
+		return "local"
+	}
+	if a.routabilityStrict && !na.Routable() {
+		return "unroutable"
+	}
+
+	if ipv4 := na.IP.To4(); ipv4 != nil {
+		return (&net.IPNet{IP: na.IP, Mask: net.CIDRMask(16, 32)}).String()
+	}
+	if na.RFC6145() || na.RFC6052() {
+		// last four bytes are the ip address
+		ip := net.IP(na.IP[12:16])
+		return (&net.IPNet{IP: ip, Mask: net.CIDRMask(16, 32)}).String()
+	}
+
+	if na.RFC3964() {
+		ip := net.IP(na.IP[2:7])
+		return (&net.IPNet{IP: ip, Mask: net.CIDRMask(16, 32)}).String()
+
+	}
+	if na.RFC4380() {
+		// teredo tunnels have the last 4 bytes as the v4 address XOR
+		// 0xff.
+		ip := net.IP(make([]byte, 4))
+		for i, byte := range na.IP[12:16] {
+			ip[i] = byte ^ 0xff
+		}
+		return (&net.IPNet{IP: ip, Mask: net.CIDRMask(16, 32)}).String()
+	}
+
+	// OK, so now we know ourselves to be a IPv6 address.
+	// bitcoind uses /32 for everything, except for Hurricane Electric's
+	// (he.net) IP range, which it uses /36 for.
+	bits := 32
+	heNet := &net.IPNet{IP: net.ParseIP("2001:470::"),
+		Mask: net.CIDRMask(32, 128)}
+	if heNet.Contains(na.IP) {
+		bits = 36
+	}
+
+	return (&net.IPNet{IP: na.IP, Mask: net.CIDRMask(bits, 128)}).String()
+}
+
+//-----------------------------------------------------------------------------
+
+/*
+   knownAddress
+
+   tracks information about a known network address that is used
+   to determine how viable an address is.
+*/
+type knownAddress struct {
+	Addr        *NetAddress
+	Src         *NetAddress
+	Attempts    int32
+	LastAttempt time.Time
+	LastSuccess time.Time
+	BucketType  byte
+	Buckets     []int
+}
+
+func newKnownAddress(addr *NetAddress, src *NetAddress) *knownAddress {
+	return &knownAddress{
+		Addr:        addr,
+		Src:         src,
+		Attempts:    0,
+		LastAttempt: time.Now(),
+		BucketType:  bucketTypeNew,
+		Buckets:     nil,
+	}
+}
+
+func (ka *knownAddress) isOld() bool {
+	return ka.BucketType == bucketTypeOld
+}
+
+func (ka *knownAddress) isNew() bool {
+	return ka.BucketType == bucketTypeNew
+}
+
+func (ka *knownAddress) markAttempt() {
+	now := time.Now()
+	ka.LastAttempt = now
+	ka.Attempts += 1
+}
+
+func (ka *knownAddress) markGood() {
+	now := time.Now()
+	ka.LastAttempt = now
+	ka.Attempts = 0
+	ka.LastSuccess = now
+}
+
+func (ka *knownAddress) addBucketRef(bucketIdx int) int {
+	for _, bucket := range ka.Buckets {
+		if bucket == bucketIdx {
+			// TODO refactor to return error?
+			// log.Warn(Fmt("Bucket already exists in ka.Buckets: %v", ka))
+			return -1
+		}
+	}
+	ka.Buckets = append(ka.Buckets, bucketIdx)
+	return len(ka.Buckets)
+}
+
+func (ka *knownAddress) removeBucketRef(bucketIdx int) int {
+	buckets := []int{}
+	for _, bucket := range ka.Buckets {
+		if bucket != bucketIdx {
+			buckets = append(buckets, bucket)
+		}
+	}
+	if len(buckets) != len(ka.Buckets)-1 {
+		// TODO refactor to return error?
+		// log.Warn(Fmt("bucketIdx not found in ka.Buckets: %v", ka))
+		return -1
+	}
+	ka.Buckets = buckets
+	return len(ka.Buckets)
+}
+
+/*
+   An address is bad if the address in question has not been tried in the last
+   minute and meets one of the following criteria:
+
+   1) It claims to be from the future
+   2) It hasn't been seen in over a month
+   3) It has failed at least three times and never succeeded
+   4) It has failed ten times in the last week
+
+   All addresses that meet these criteria are assumed to be worthless and not
+   worth keeping hold of.
+*/
+func (ka *knownAddress) isBad() bool {
+	// Has been attempted in the last minute --> good
+	if ka.LastAttempt.Before(time.Now().Add(-1 * time.Minute)) {
+		return false
+	}
+
+	// Over a month old?
+	if ka.LastAttempt.After(time.Now().Add(-1 * numMissingDays * time.Hour * 24)) {
+		return true
+	}
+
+	// Never succeeded?
+	if ka.LastSuccess.IsZero() && ka.Attempts >= numRetries {
+		return true
+	}
+
+	// Hasn't succeeded in too long?
+	if ka.LastSuccess.Before(time.Now().Add(-1*minBadDays*time.Hour*24)) &&
+		ka.Attempts >= maxFailures {
+		return true
+	}
+
+	return false
+}

p2p/addrbook_test.go

@@ -0,0 +1,174 @@
+package p2p
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tendermint/tmlibs/log"
+)
+
+func createTempFileName(prefix string) string {
+	f, err := ioutil.TempFile("", prefix)
+	if err != nil {
+		panic(err)
+	}
+	fname := f.Name()
+	err = f.Close()
+	if err != nil {
+		panic(err)
+	}
+	return fname
+}
+
+func TestAddrBookSaveLoad(t *testing.T) {
+	fname := createTempFileName("addrbook_test")
+
+	// 0 addresses
+	book := NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+	book.saveToFile(fname)
+
+	book = NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+	book.loadFromFile(fname)
+
+	assert.Zero(t, book.Size())
+
+	// 100 addresses
+	randAddrs := randNetAddressPairs(t, 100)
+
+	for _, addrSrc := range randAddrs {
+		book.AddAddress(addrSrc.addr, addrSrc.src)
+	}
+
+	assert.Equal(t, 100, book.Size())
+	book.saveToFile(fname)
+
+	book = NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+	book.loadFromFile(fname)
+
+	assert.Equal(t, 100, book.Size())
+}
+
+func TestAddrBookLookup(t *testing.T) {
+	fname := createTempFileName("addrbook_test")
+
+	randAddrs := randNetAddressPairs(t, 100)
+
+	book := NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+	for _, addrSrc := range randAddrs {
+		addr := addrSrc.addr
+		src := addrSrc.src
+		book.AddAddress(addr, src)
+
+		ka := book.addrLookup[addr.String()]
+		assert.NotNil(t, ka, "Expected to find KnownAddress %v but wasn't there.", addr)
+
+		if !(ka.Addr.Equals(addr) && ka.Src.Equals(src)) {
+			t.Fatalf("KnownAddress doesn't match addr & src")
+		}
+	}
+}
+
+func TestAddrBookPromoteToOld(t *testing.T) {
+	fname := createTempFileName("addrbook_test")
+
+	randAddrs := randNetAddressPairs(t, 100)
+
+	book := NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+	for _, addrSrc := range randAddrs {
+		book.AddAddress(addrSrc.addr, addrSrc.src)
+	}
+
+	// Attempt all addresses.
+	for _, addrSrc := range randAddrs {
+		book.MarkAttempt(addrSrc.addr)
+	}
+
+	// Promote half of them
+	for i, addrSrc := range randAddrs {
+		if i%2 == 0 {
+			book.MarkGood(addrSrc.addr)
+		}
+	}
+
+	// TODO: do more testing :)
+
+	selection := book.GetSelection()
+	t.Logf("selection: %v", selection)
+
+	if len(selection) > book.Size() {
+		t.Errorf("selection could not be bigger than the book")
+	}
+}
+
+func TestAddrBookHandlesDuplicates(t *testing.T) {
+	fname := createTempFileName("addrbook_test")
+
+	book := NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+
+	randAddrs := randNetAddressPairs(t, 100)
+
+	differentSrc := randIPv4Address(t)
+	for _, addrSrc := range randAddrs {
+		book.AddAddress(addrSrc.addr, addrSrc.src)
+		book.AddAddress(addrSrc.addr, addrSrc.src)  // duplicate
+		book.AddAddress(addrSrc.addr, differentSrc) // different src
+	}
+
+	assert.Equal(t, 100, book.Size())
+}
+
+type netAddressPair struct {
+	addr *NetAddress
+	src  *NetAddress
+}
+
+func randNetAddressPairs(t *testing.T, n int) []netAddressPair {
+	randAddrs := make([]netAddressPair, n)
+	for i := 0; i < n; i++ {
+		randAddrs[i] = netAddressPair{addr: randIPv4Address(t), src: randIPv4Address(t)}
+	}
+	return randAddrs
+}
+
+func randIPv4Address(t *testing.T) *NetAddress {
+	for {
+		ip := fmt.Sprintf("%v.%v.%v.%v",
+			rand.Intn(254)+1,
+			rand.Intn(255),
+			rand.Intn(255),
+			rand.Intn(255),
+		)
+		port := rand.Intn(65535-1) + 1
+		addr, err := NewNetAddressString(fmt.Sprintf("%v:%v", ip, port))
+		assert.Nil(t, err, "error generating rand network address")
+		if addr.Routable() {
+			return addr
+		}
+	}
+}
+
+func TestAddrBookRemoveAddress(t *testing.T) {
+	fname := createTempFileName("addrbook_test")
+	book := NewAddrBook(fname, true)
+	book.SetLogger(log.TestingLogger())
+
+	addr := randIPv4Address(t)
+	book.AddAddress(addr, addr)
+	assert.Equal(t, 1, book.Size())
+
+	book.RemoveAddress(addr)
+	assert.Equal(t, 0, book.Size())
+
+	nonExistingAddr := randIPv4Address(t)
+	book.RemoveAddress(nonExistingAddr)
+	assert.Equal(t, 0, book.Size())
+}

p2p/connection.go

@@ -0,0 +1,686 @@
+package p2p
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"runtime/debug"
+	"sync/atomic"
+	"time"
+
+	wire "github.com/tendermint/go-wire"
+	cmn "github.com/tendermint/tmlibs/common"
+	flow "github.com/tendermint/tmlibs/flowrate"
+)
+
+const (
+	numBatchMsgPackets = 10
+	minReadBufferSize  = 1024
+	minWriteBufferSize = 65536
+	updateState        = 2 * time.Second
+	pingTimeout        = 40 * time.Second
+	flushThrottle      = 100 * time.Millisecond
+
+	defaultSendQueueCapacity   = 1
+	defaultSendRate            = int64(512000) // 500KB/s
+	defaultRecvBufferCapacity  = 4096
+	defaultRecvMessageCapacity = 22020096      // 21MB
+	defaultRecvRate            = int64(512000) // 500KB/s
+	defaultSendTimeout         = 10 * time.Second
+)
+
+type receiveCbFunc func(chID byte, msgBytes []byte)
+type errorCbFunc func(interface{})
+
+/*
+Each peer has one `MConnection` (multiplex connection) instance.
+
+__multiplex__ *noun* a system or signal involving simultaneous transmission of
+several messages along a single channel of communication.
+
+Each `MConnection` handles message transmission on multiple abstract communication
+`Channel`s.  Each channel has a globally unique byte id.
+The byte id and the relative priorities of each `Channel` are configured upon
+initialization of the connection.
+
+There are two methods for sending messages:
+	func (m MConnection) Send(chID byte, msg interface{}) bool {}
+	func (m MConnection) TrySend(chID byte, msg interface{}) bool {}
+
+`Send(chID, msg)` is a blocking call that waits until `msg` is successfully queued
+for the channel with the given id byte `chID`, or until the request times out.
+The message `msg` is serialized using the `tendermint/wire` submodule's
+`WriteBinary()` reflection routine.
+
+`TrySend(chID, msg)` is a nonblocking call that returns false if the channel's
+queue is full.
+
+Inbound message bytes are handled with an onReceive callback function.
+*/
+type MConnection struct {
+	cmn.BaseService
+
+	conn        net.Conn
+	bufReader   *bufio.Reader
+	bufWriter   *bufio.Writer
+	sendMonitor *flow.Monitor
+	recvMonitor *flow.Monitor
+	send        chan struct{}
+	pong        chan struct{}
+	channels    []*Channel
+	channelsIdx map[byte]*Channel
+	onReceive   receiveCbFunc
+	onError     errorCbFunc
+	errored     uint32
+	config      *MConnConfig
+
+	quit         chan struct{}
+	flushTimer   *cmn.ThrottleTimer // flush writes as necessary but throttled.
+	pingTimer    *cmn.RepeatTimer   // send pings periodically
+	chStatsTimer *cmn.RepeatTimer   // update channel stats periodically
+
+	LocalAddress  *NetAddress
+	RemoteAddress *NetAddress
+}
+
+// MConnConfig is a MConnection configuration.
+type MConnConfig struct {
+	SendRate int64 `mapstructure:"send_rate"`
+	RecvRate int64 `mapstructure:"recv_rate"`
+}
+
+// DefaultMConnConfig returns the default config.
+func DefaultMConnConfig() *MConnConfig {
+	return &MConnConfig{
+		SendRate: defaultSendRate,
+		RecvRate: defaultRecvRate,
+	}
+}
+
+// NewMConnection wraps net.Conn and creates multiplex connection
+func NewMConnection(conn net.Conn, chDescs []*ChannelDescriptor, onReceive receiveCbFunc, onError errorCbFunc) *MConnection {
+	return NewMConnectionWithConfig(
+		conn,
+		chDescs,
+		onReceive,
+		onError,
+		DefaultMConnConfig())
+}
+
+// NewMConnectionWithConfig wraps net.Conn and creates multiplex connection with a config
+func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onReceive receiveCbFunc, onError errorCbFunc, config *MConnConfig) *MConnection {
+	mconn := &MConnection{
+		conn:        conn,
+		bufReader:   bufio.NewReaderSize(conn, minReadBufferSize),
+		bufWriter:   bufio.NewWriterSize(conn, minWriteBufferSize),
+		sendMonitor: flow.New(0, 0),
+		recvMonitor: flow.New(0, 0),
+		send:        make(chan struct{}, 1),
+		pong:        make(chan struct{}),
+		onReceive:   onReceive,
+		onError:     onError,
+		config:      config,
+
+		LocalAddress:  NewNetAddress(conn.LocalAddr()),
+		RemoteAddress: NewNetAddress(conn.RemoteAddr()),
+	}
+
+	// Create channels
+	var channelsIdx = map[byte]*Channel{}
+	var channels = []*Channel{}
+
+	for _, desc := range chDescs {
+		descCopy := *desc // copy the desc else unsafe access across connections
+		channel := newChannel(mconn, &descCopy)
+		channelsIdx[channel.id] = channel
+		channels = append(channels, channel)
+	}
+	mconn.channels = channels
+	mconn.channelsIdx = channelsIdx
+
+	mconn.BaseService = *cmn.NewBaseService(nil, "MConnection", mconn)
+
+	return mconn
+}
+
+func (c *MConnection) OnStart() error {
+	c.BaseService.OnStart()
+	c.quit = make(chan struct{})
+	c.flushTimer = cmn.NewThrottleTimer("flush", flushThrottle)
+	c.pingTimer = cmn.NewRepeatTimer("ping", pingTimeout)
+	c.chStatsTimer = cmn.NewRepeatTimer("chStats", updateState)
+	go c.sendRoutine()
+	go c.recvRoutine()
+	return nil
+}
+
+func (c *MConnection) OnStop() {
+	c.BaseService.OnStop()
+	c.flushTimer.Stop()
+	c.pingTimer.Stop()
+	c.chStatsTimer.Stop()
+	if c.quit != nil {
+		close(c.quit)
+	}
+	c.conn.Close()
+	// We can't close pong safely here because
+	// recvRoutine may write to it after we've stopped.
+	// Though it doesn't need to get closed at all,
+	// we close it @ recvRoutine.
+	// close(c.pong)
+}
+
+func (c *MConnection) String() string {
+	return fmt.Sprintf("MConn{%v}", c.conn.RemoteAddr())
+}
+
+func (c *MConnection) flush() {
+	c.Logger.Debug("Flush", "conn", c)
+	err := c.bufWriter.Flush()
+	if err != nil {
+		c.Logger.Error("MConnection flush failed", "err", err)
+	}
+}
+
+// Catch panics, usually caused by remote disconnects.
+func (c *MConnection) _recover() {
+	if r := recover(); r != nil {
+		stack := debug.Stack()
+		err := cmn.StackError{r, stack}
+		c.stopForError(err)
+	}
+}
+
+func (c *MConnection) stopForError(r interface{}) {
+	c.Stop()
+	if atomic.CompareAndSwapUint32(&c.errored, 0, 1) {
+		if c.onError != nil {
+			c.onError(r)
+		}
+	}
+}
+
+// Queues a message to be sent to channel.
+func (c *MConnection) Send(chID byte, msg interface{}) bool {
+	if !c.IsRunning() {
+		return false
+	}
+
+	c.Logger.Debug("Send", "channel", chID, "conn", c, "msg", msg) //, "bytes", wire.BinaryBytes(msg))
+
+	// Send message to channel.
+	channel, ok := c.channelsIdx[chID]
+	if !ok {
+		c.Logger.Error(cmn.Fmt("Cannot send bytes, unknown channel %X", chID))
+		return false
+	}
+
+	success := channel.sendBytes(wire.BinaryBytes(msg))
+	if success {
+		// Wake up sendRoutine if necessary
+		select {
+		case c.send <- struct{}{}:
+		default:
+		}
+	} else {
+		c.Logger.Error("Send failed", "channel", chID, "conn", c, "msg", msg)
+	}
+	return success
+}
+
+// Queues a message to be sent to channel.
+// Nonblocking, returns true if successful.
+func (c *MConnection) TrySend(chID byte, msg interface{}) bool {
+	if !c.IsRunning() {
+		return false
+	}
+
+	c.Logger.Debug("TrySend", "channel", chID, "conn", c, "msg", msg)
+
+	// Send message to channel.
+	channel, ok := c.channelsIdx[chID]
+	if !ok {
+		c.Logger.Error(cmn.Fmt("Cannot send bytes, unknown channel %X", chID))
+		return false
+	}
+
+	ok = channel.trySendBytes(wire.BinaryBytes(msg))
+	if ok {
+		// Wake up sendRoutine if necessary
+		select {
+		case c.send <- struct{}{}:
+		default:
+		}
+	}
+
+	return ok
+}
+
+// CanSend returns true if you can send more data onto the chID, false
+// otherwise. Use only as a heuristic.
+func (c *MConnection) CanSend(chID byte) bool {
+	if !c.IsRunning() {
+		return false
+	}
+
+	channel, ok := c.channelsIdx[chID]
+	if !ok {
+		c.Logger.Error(cmn.Fmt("Unknown channel %X", chID))
+		return false
+	}
+	return channel.canSend()
+}
+
+// sendRoutine polls for packets to send from channels.
+func (c *MConnection) sendRoutine() {
+	defer c._recover()
+
+FOR_LOOP:
+	for {
+		var n int
+		var err error
+		select {
+		case <-c.flushTimer.Ch:
+			// NOTE: flushTimer.Set() must be called every time
+			// something is written to .bufWriter.
+			c.flush()
+		case <-c.chStatsTimer.Ch:
+			for _, channel := range c.channels {
+				channel.updateStats()
+			}
+		case <-c.pingTimer.Ch:
+			c.Logger.Debug("Send Ping")
+			wire.WriteByte(packetTypePing, c.bufWriter, &n, &err)
+			c.sendMonitor.Update(int(n))
+			c.flush()
+		case <-c.pong:
+			c.Logger.Debug("Send Pong")
+			wire.WriteByte(packetTypePong, c.bufWriter, &n, &err)
+			c.sendMonitor.Update(int(n))
+			c.flush()
+		case <-c.quit:
+			break FOR_LOOP
+		case <-c.send:
+			// Send some msgPackets
+			eof := c.sendSomeMsgPackets()
+			if !eof {
+				// Keep sendRoutine awake.
+				select {
+				case c.send <- struct{}{}:
+				default:
+				}
+			}
+		}
+
+		if !c.IsRunning() {
+			break FOR_LOOP
+		}
+		if err != nil {
+			c.Logger.Error("Connection failed @ sendRoutine", "conn", c, "err", err)
+			c.stopForError(err)
+			break FOR_LOOP
+		}
+	}
+
+	// Cleanup
+}
+
+// Returns true if messages from channels were exhausted.
+// Blocks in accordance to .sendMonitor throttling.
+func (c *MConnection) sendSomeMsgPackets() bool {
+	// Block until .sendMonitor says we can write.
+	// Once we're ready we send more than we asked for,
+	// but amortized it should even out.
+	c.sendMonitor.Limit(maxMsgPacketTotalSize, atomic.LoadInt64(&c.config.SendRate), true)
+
+	// Now send some msgPackets.
+	for i := 0; i < numBatchMsgPackets; i++ {
+		if c.sendMsgPacket() {
+			return true
+		}
+	}
+	return false
+}
+
+// Returns true if messages from channels were exhausted.
+func (c *MConnection) sendMsgPacket() bool {
+	// Choose a channel to create a msgPacket from.
+	// The chosen channel will be the one whose recentlySent/priority is the least.
+	var leastRatio float32 = math.MaxFloat32
+	var leastChannel *Channel
+	for _, channel := range c.channels {
+		// If nothing to send, skip this channel
+		if !channel.isSendPending() {
+			continue
+		}
+		// Get ratio, and keep track of lowest ratio.
+		ratio := float32(channel.recentlySent) / float32(channel.priority)
+		if ratio < leastRatio {
+			leastRatio = ratio
+			leastChannel = channel
+		}
+	}
+
+	// Nothing to send?
+	if leastChannel == nil {
+		return true
+	} else {
+		// c.Logger.Info("Found a msgPacket to send")
+	}
+
+	// Make & send a msgPacket from this channel
+	n, err := leastChannel.writeMsgPacketTo(c.bufWriter)
+	if err != nil {
+		c.Logger.Error("Failed to write msgPacket", "err", err)
+		c.stopForError(err)
+		return true
+	}
+	c.sendMonitor.Update(int(n))
+	c.flushTimer.Set()
+	return false
+}
+
+// recvRoutine reads msgPackets and reconstructs the message using the channels' "recving" buffer.
+// After a whole message has been assembled, it's pushed to onReceive().
+// Blocks depending on how the connection is throttled.
+func (c *MConnection) recvRoutine() {
+	defer c._recover()
+
+FOR_LOOP:
+	for {
+		// Block until .recvMonitor says we can read.
+		c.recvMonitor.Limit(maxMsgPacketTotalSize, atomic.LoadInt64(&c.config.RecvRate), true)
+
+		/*
+			// Peek into bufReader for debugging
+			if numBytes := c.bufReader.Buffered(); numBytes > 0 {
+				log.Info("Peek connection buffer", "numBytes", numBytes, "bytes", log15.Lazy{func() []byte {
+					bytes, err := c.bufReader.Peek(MinInt(numBytes, 100))
+					if err == nil {
+						return bytes
+					} else {
+						log.Warn("Error peeking connection buffer", "err", err)
+						return nil
+					}
+				}})
+			}
+		*/
+
+		// Read packet type
+		var n int
+		var err error
+		pktType := wire.ReadByte(c.bufReader, &n, &err)
+		c.recvMonitor.Update(int(n))
+		if err != nil {
+			if c.IsRunning() {
+				c.Logger.Error("Connection failed @ recvRoutine (reading byte)", "conn", c, "err", err)
+				c.stopForError(err)
+			}
+			break FOR_LOOP
+		}
+
+		// Read more depending on packet type.
+		switch pktType {
+		case packetTypePing:
+			// TODO: prevent abuse, as they cause flush()'s.
+			c.Logger.Debug("Receive Ping")
+			c.pong <- struct{}{}
+		case packetTypePong:
+			// do nothing
+			c.Logger.Debug("Receive Pong")
+		case packetTypeMsg:
+			pkt, n, err := msgPacket{}, int(0), error(nil)
+			wire.ReadBinaryPtr(&pkt, c.bufReader, maxMsgPacketTotalSize, &n, &err)
+			c.recvMonitor.Update(int(n))
+			if err != nil {
+				if c.IsRunning() {
+					c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err)
+					c.stopForError(err)
+				}
+				break FOR_LOOP
+			}
+			channel, ok := c.channelsIdx[pkt.ChannelID]
+			if !ok || channel == nil {
+				cmn.PanicQ(cmn.Fmt("Unknown channel %X", pkt.ChannelID))
+			}
+			msgBytes, err := channel.recvMsgPacket(pkt)
+			if err != nil {
+				if c.IsRunning() {
+					c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err)
+					c.stopForError(err)
+				}
+				break FOR_LOOP
+			}
+			if msgBytes != nil {
+				c.Logger.Debug("Received bytes", "chID", pkt.ChannelID, "msgBytes", msgBytes)
+				c.onReceive(pkt.ChannelID, msgBytes)
+			}
+		default:
+			cmn.PanicSanity(cmn.Fmt("Unknown message type %X", pktType))
+		}
+
+		// TODO: shouldn't this go in the sendRoutine?
+		// Better to send a ping packet when *we* haven't sent anything for a while.
+		c.pingTimer.Reset()
+	}
+
+	// Cleanup
+	close(c.pong)
+	for range c.pong {
+		// Drain
+	}
+}
+
+type ConnectionStatus struct {
+	SendMonitor flow.Status
+	RecvMonitor flow.Status
+	Channels    []ChannelStatus
+}
+
+type ChannelStatus struct {
+	ID                byte
+	SendQueueCapacity int
+	SendQueueSize     int
+	Priority          int
+	RecentlySent      int64
+}
+
+func (c *MConnection) Status() ConnectionStatus {
+	var status ConnectionStatus
+	status.SendMonitor = c.sendMonitor.Status()
+	status.RecvMonitor = c.recvMonitor.Status()
+	status.Channels = make([]ChannelStatus, len(c.channels))
+	for i, channel := range c.channels {
+		status.Channels[i] = ChannelStatus{
+			ID:                channel.id,
+			SendQueueCapacity: cap(channel.sendQueue),
+			SendQueueSize:     int(channel.sendQueueSize), // TODO use atomic
+			Priority:          channel.priority,
+			RecentlySent:      channel.recentlySent,
+		}
+	}
+	return status
+}
+
+//-----------------------------------------------------------------------------
+
+type ChannelDescriptor struct {
+	ID                  byte
+	Priority            int
+	SendQueueCapacity   int
+	RecvBufferCapacity  int
+	RecvMessageCapacity int
+}
+
+func (chDesc *ChannelDescriptor) FillDefaults() {
+	if chDesc.SendQueueCapacity == 0 {
+		chDesc.SendQueueCapacity = defaultSendQueueCapacity
+	}
+	if chDesc.RecvBufferCapacity == 0 {
+		chDesc.RecvBufferCapacity = defaultRecvBufferCapacity
+	}
+	if chDesc.RecvMessageCapacity == 0 {
+		chDesc.RecvMessageCapacity = defaultRecvMessageCapacity
+	}
+}
+
+// TODO: lowercase.
+// NOTE: not goroutine-safe.
+type Channel struct {
+	conn          *MConnection
+	desc          *ChannelDescriptor
+	id            byte
+	sendQueue     chan []byte
+	sendQueueSize int32 // atomic.
+	recving       []byte
+	sending       []byte
+	priority      int
+	recentlySent  int64 // exponential moving average
+}
+
+func newChannel(conn *MConnection, desc *ChannelDescriptor) *Channel {
+	desc.FillDefaults()
+	if desc.Priority <= 0 {
+		cmn.PanicSanity("Channel default priority must be a postive integer")
+	}
+	return &Channel{
+		conn:      conn,
+		desc:      desc,
+		id:        desc.ID,
+		sendQueue: make(chan []byte, desc.SendQueueCapacity),
+		recving:   make([]byte, 0, desc.RecvBufferCapacity),
+		priority:  desc.Priority,
+	}
+}
+
+// Queues message to send to this channel.
+// Goroutine-safe
+// Times out (and returns false) after defaultSendTimeout
+func (ch *Channel) sendBytes(bytes []byte) bool {
+	select {
+	case ch.sendQueue <- bytes:
+		atomic.AddInt32(&ch.sendQueueSize, 1)
+		return true
+	case <-time.After(defaultSendTimeout):
+		return false
+	}
+}
+
+// Queues message to send to this channel.
+// Nonblocking, returns true if successful.
+// Goroutine-safe
+func (ch *Channel) trySendBytes(bytes []byte) bool {
+	select {
+	case ch.sendQueue <- bytes:
+		atomic.AddInt32(&ch.sendQueueSize, 1)
+		return true
+	default:
+		return false
+	}
+}
+
+// Goroutine-safe
+func (ch *Channel) loadSendQueueSize() (size int) {
+	return int(atomic.LoadInt32(&ch.sendQueueSize))
+}
+
+// Goroutine-safe
+// Use only as a heuristic.
+func (ch *Channel) canSend() bool {
+	return ch.loadSendQueueSize() < defaultSendQueueCapacity
+}
+
+// Returns true if any msgPackets are pending to be sent.
+// Call before calling nextMsgPacket()
+// Goroutine-safe
+func (ch *Channel) isSendPending() bool {
+	if len(ch.sending) == 0 {
+		if len(ch.sendQueue) == 0 {
+			return false
+		}
+		ch.sending = <-ch.sendQueue
+	}
+	return true
+}
+
+// Creates a new msgPacket to send.
+// Not goroutine-safe
+func (ch *Channel) nextMsgPacket() msgPacket {
+	packet := msgPacket{}
+	packet.ChannelID = byte(ch.id)
+	packet.Bytes = ch.sending[:cmn.MinInt(maxMsgPacketPayloadSize, len(ch.sending))]
+	if len(ch.sending) <= maxMsgPacketPayloadSize {
+		packet.EOF = byte(0x01)
+		ch.sending = nil
+		atomic.AddInt32(&ch.sendQueueSize, -1) // decrement sendQueueSize
+	} else {
+		packet.EOF = byte(0x00)
+		ch.sending = ch.sending[cmn.MinInt(maxMsgPacketPayloadSize, len(ch.sending)):]
+	}
+	return packet
+}
+
+// Writes next msgPacket to w.
+// Not goroutine-safe
+func (ch *Channel) writeMsgPacketTo(w io.Writer) (n int, err error) {
+	packet := ch.nextMsgPacket()
+	// log.Debug("Write Msg Packet", "conn", ch.conn, "packet", packet)
+	wire.WriteByte(packetTypeMsg, w, &n, &err)
+	wire.WriteBinary(packet, w, &n, &err)
+	if err == nil {
+		ch.recentlySent += int64(n)
+	}
+	return
+}
+
+// Handles incoming msgPackets. Returns a msg bytes if msg is complete.
+// Not goroutine-safe
+func (ch *Channel) recvMsgPacket(packet msgPacket) ([]byte, error) {
+	// log.Debug("Read Msg Packet", "conn", ch.conn, "packet", packet)
+	if ch.desc.RecvMessageCapacity < len(ch.recving)+len(packet.Bytes) {
+		return nil, wire.ErrBinaryReadOverflow
+	}
+	ch.recving = append(ch.recving, packet.Bytes...)
+	if packet.EOF == byte(0x01) {
+		msgBytes := ch.recving
+		// clear the slice without re-allocating.
+		// http://stackoverflow.com/questions/16971741/how-do-you-clear-a-slice-in-go
+		//   suggests this could be a memory leak, but we might as well keep the memory for the channel until it closes,
+		//	at which point the recving slice stops being used and should be garbage collected
+		ch.recving = ch.recving[:0] // make([]byte, 0, ch.desc.RecvBufferCapacity)
+		return msgBytes, nil
+	}
+	return nil, nil
+}
+
+// Call this periodically to update stats for throttling purposes.
+// Not goroutine-safe
+func (ch *Channel) updateStats() {
+	// Exponential decay of stats.
+	// TODO: optimize.
+	ch.recentlySent = int64(float64(ch.recentlySent) * 0.8)
+}
+
+//-----------------------------------------------------------------------------
+
+const (
+	maxMsgPacketPayloadSize  = 1024
+	maxMsgPacketOverheadSize = 10 // It's actually lower but good enough
+	maxMsgPacketTotalSize    = maxMsgPacketPayloadSize + maxMsgPacketOverheadSize
+	packetTypePing           = byte(0x01)
+	packetTypePong           = byte(0x02)
+	packetTypeMsg            = byte(0x03)
+)
+
+// Messages in channels are chopped into smaller msgPackets for multiplexing.
+type msgPacket struct {
+	ChannelID byte
+	EOF       byte // 1 means message ends here.
+	Bytes     []byte
+}
+
+func (p msgPacket) String() string {
+	return fmt.Sprintf("MsgPacket{%X:%X T:%X}", p.ChannelID, p.Bytes, p.EOF)
+}

p2p/connection_test.go

@@ -0,0 +1,144 @@
+package p2p_test
+
+import (
+	"net"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	p2p "github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tmlibs/log"
+)
+
+func createMConnection(conn net.Conn) *p2p.MConnection {
+	onReceive := func(chID byte, msgBytes []byte) {
+	}
+	onError := func(r interface{}) {
+	}
+	c := createMConnectionWithCallbacks(conn, onReceive, onError)
+	c.SetLogger(log.TestingLogger())
+	return c
+}
+
+func createMConnectionWithCallbacks(conn net.Conn, onReceive func(chID byte, msgBytes []byte), onError func(r interface{})) *p2p.MConnection {
+	chDescs := []*p2p.ChannelDescriptor{&p2p.ChannelDescriptor{ID: 0x01, Priority: 1, SendQueueCapacity: 1}}
+	c := p2p.NewMConnection(conn, chDescs, onReceive, onError)
+	c.SetLogger(log.TestingLogger())
+	return c
+}
+
+func TestMConnectionSend(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+
+	mconn := createMConnection(client)
+	_, err := mconn.Start()
+	require.Nil(err)
+	defer mconn.Stop()
+
+	msg := "Ant-Man"
+	assert.True(mconn.Send(0x01, msg))
+	// Note: subsequent Send/TrySend calls could pass because we are reading from
+	// the send queue in a separate goroutine.
+	server.Read(make([]byte, len(msg)))
+	assert.True(mconn.CanSend(0x01))
+
+	msg = "Spider-Man"
+	assert.True(mconn.TrySend(0x01, msg))
+	server.Read(make([]byte, len(msg)))
+
+	assert.False(mconn.CanSend(0x05), "CanSend should return false because channel is unknown")
+	assert.False(mconn.Send(0x05, "Absorbing Man"), "Send should return false because channel is unknown")
+}
+
+func TestMConnectionReceive(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+
+	receivedCh := make(chan []byte)
+	errorsCh := make(chan interface{})
+	onReceive := func(chID byte, msgBytes []byte) {
+		receivedCh <- msgBytes
+	}
+	onError := func(r interface{}) {
+		errorsCh <- r
+	}
+	mconn1 := createMConnectionWithCallbacks(client, onReceive, onError)
+	_, err := mconn1.Start()
+	require.Nil(err)
+	defer mconn1.Stop()
+
+	mconn2 := createMConnection(server)
+	_, err = mconn2.Start()
+	require.Nil(err)
+	defer mconn2.Stop()
+
+	msg := "Cyclops"
+	assert.True(mconn2.Send(0x01, msg))
+
+	select {
+	case receivedBytes := <-receivedCh:
+		assert.Equal([]byte(msg), receivedBytes[2:]) // first 3 bytes are internal
+	case err := <-errorsCh:
+		t.Fatalf("Expected %s, got %+v", msg, err)
+	case <-time.After(500 * time.Millisecond):
+		t.Fatalf("Did not receive %s message in 500ms", msg)
+	}
+}
+
+func TestMConnectionStatus(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+
+	mconn := createMConnection(client)
+	_, err := mconn.Start()
+	require.Nil(err)
+	defer mconn.Stop()
+
+	status := mconn.Status()
+	assert.NotNil(status)
+	assert.Zero(status.Channels[0].SendQueueSize)
+}
+
+func TestMConnectionStopsAndReturnsError(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+
+	receivedCh := make(chan []byte)
+	errorsCh := make(chan interface{})
+	onReceive := func(chID byte, msgBytes []byte) {
+		receivedCh <- msgBytes
+	}
+	onError := func(r interface{}) {
+		errorsCh <- r
+	}
+	mconn := createMConnectionWithCallbacks(client, onReceive, onError)
+	_, err := mconn.Start()
+	require.Nil(err)
+	defer mconn.Stop()
+
+	client.Close()
+
+	select {
+	case receivedBytes := <-receivedCh:
+		t.Fatalf("Expected error, got %v", receivedBytes)
+	case err := <-errorsCh:
+		assert.NotNil(err)
+		assert.False(mconn.IsRunning())
+	case <-time.After(500 * time.Millisecond):
+		t.Fatal("Did not receive error in 500ms")
+	}
+}

p2p/fuzz.go

@@ -0,0 +1,173 @@
+package p2p
+
+import (
+	"math/rand"
+	"net"
+	"sync"
+	"time"
+)
+
+const (
+	// FuzzModeDrop is a mode in which we randomly drop reads/writes, connections or sleep
+	FuzzModeDrop = iota
+	// FuzzModeDelay is a mode in which we randomly sleep
+	FuzzModeDelay
+)
+
+// FuzzedConnection wraps any net.Conn and depending on the mode either delays
+// reads/writes or randomly drops reads/writes/connections.
+type FuzzedConnection struct {
+	conn net.Conn
+
+	mtx    sync.Mutex
+	start  <-chan time.Time
+	active bool
+
+	config *FuzzConnConfig
+}
+
+// FuzzConnConfig is a FuzzedConnection configuration.
+type FuzzConnConfig struct {
+	Mode         int
+	MaxDelay     time.Duration
+	ProbDropRW   float64
+	ProbDropConn float64
+	ProbSleep    float64
+}
+
+// DefaultFuzzConnConfig returns the default config.
+func DefaultFuzzConnConfig() *FuzzConnConfig {
+	return &FuzzConnConfig{
+		Mode:         FuzzModeDrop,
+		MaxDelay:     3 * time.Second,
+		ProbDropRW:   0.2,
+		ProbDropConn: 0.00,
+		ProbSleep:    0.00,
+	}
+}
+
+// FuzzConn creates a new FuzzedConnection. Fuzzing starts immediately.
+func FuzzConn(conn net.Conn) net.Conn {
+	return FuzzConnFromConfig(conn, DefaultFuzzConnConfig())
+}
+
+// FuzzConnFromConfig creates a new FuzzedConnection from a config. Fuzzing
+// starts immediately.
+func FuzzConnFromConfig(conn net.Conn, config *FuzzConnConfig) net.Conn {
+	return &FuzzedConnection{
+		conn:   conn,
+		start:  make(<-chan time.Time),
+		active: true,
+		config: config,
+	}
+}
+
+// FuzzConnAfter creates a new FuzzedConnection. Fuzzing starts when the
+// duration elapses.
+func FuzzConnAfter(conn net.Conn, d time.Duration) net.Conn {
+	return FuzzConnAfterFromConfig(conn, d, DefaultFuzzConnConfig())
+}
+
+// FuzzConnAfterFromConfig creates a new FuzzedConnection from a config.
+// Fuzzing starts when the duration elapses.
+func FuzzConnAfterFromConfig(conn net.Conn, d time.Duration, config *FuzzConnConfig) net.Conn {
+	return &FuzzedConnection{
+		conn:   conn,
+		start:  time.After(d),
+		active: false,
+		config: config,
+	}
+}
+
+// Config returns the connection's config.
+func (fc *FuzzedConnection) Config() *FuzzConnConfig {
+	return fc.config
+}
+
+// Read implements net.Conn.
+func (fc *FuzzedConnection) Read(data []byte) (n int, err error) {
+	if fc.fuzz() {
+		return 0, nil
+	}
+	return fc.conn.Read(data)
+}
+
+// Write implements net.Conn.
+func (fc *FuzzedConnection) Write(data []byte) (n int, err error) {
+	if fc.fuzz() {
+		return 0, nil
+	}
+	return fc.conn.Write(data)
+}
+
+// Close implements net.Conn.
+func (fc *FuzzedConnection) Close() error { return fc.conn.Close() }
+
+// LocalAddr implements net.Conn.
+func (fc *FuzzedConnection) LocalAddr() net.Addr { return fc.conn.LocalAddr() }
+
+// RemoteAddr implements net.Conn.
+func (fc *FuzzedConnection) RemoteAddr() net.Addr { return fc.conn.RemoteAddr() }
+
+// SetDeadline implements net.Conn.
+func (fc *FuzzedConnection) SetDeadline(t time.Time) error { return fc.conn.SetDeadline(t) }
+
+// SetReadDeadline implements net.Conn.
+func (fc *FuzzedConnection) SetReadDeadline(t time.Time) error {
+	return fc.conn.SetReadDeadline(t)
+}
+
+// SetWriteDeadline implements net.Conn.
+func (fc *FuzzedConnection) SetWriteDeadline(t time.Time) error {
+	return fc.conn.SetWriteDeadline(t)
+}
+
+func (fc *FuzzedConnection) randomDuration() time.Duration {
+	maxDelayMillis := int(fc.config.MaxDelay.Nanoseconds() / 1000)
+	return time.Millisecond * time.Duration(rand.Int()%maxDelayMillis)
+}
+
+// implements the fuzz (delay, kill conn)
+// and returns whether or not the read/write should be ignored
+func (fc *FuzzedConnection) fuzz() bool {
+	if !fc.shouldFuzz() {
+		return false
+	}
+
+	switch fc.config.Mode {
+	case FuzzModeDrop:
+		// randomly drop the r/w, drop the conn, or sleep
+		r := rand.Float64()
+		if r <= fc.config.ProbDropRW {
+			return true
+		} else if r < fc.config.ProbDropRW+fc.config.ProbDropConn {
+			// XXX: can't this fail because machine precision?
+			// XXX: do we need an error?
+			fc.Close()
+			return true
+		} else if r < fc.config.ProbDropRW+fc.config.ProbDropConn+fc.config.ProbSleep {
+			time.Sleep(fc.randomDuration())
+		}
+	case FuzzModeDelay:
+		// sleep a bit
+		time.Sleep(fc.randomDuration())
+	}
+	return false
+}
+
+func (fc *FuzzedConnection) shouldFuzz() bool {
+	if fc.active {
+		return true
+	}
+
+	fc.mtx.Lock()
+	defer fc.mtx.Unlock()
+
+	select {
+	case <-fc.start:
+		fc.active = true
+		return true
+	default:
+		return false
+	}
+}

p2p/ip_range_counter.go

@@ -0,0 +1,29 @@
+package p2p
+
+import (
+	"strings"
+)
+
+// TODO Test
+func AddToIPRangeCounts(counts map[string]int, ip string) map[string]int {
+	changes := make(map[string]int)
+	ipParts := strings.Split(ip, ":")
+	for i := 1; i < len(ipParts); i++ {
+		prefix := strings.Join(ipParts[:i], ":")
+		counts[prefix] += 1
+		changes[prefix] = counts[prefix]
+	}
+	return changes
+}
+
+// TODO Test
+func CheckIPRangeCounts(counts map[string]int, limits []int) bool {
+	for prefix, count := range counts {
+		ipParts := strings.Split(prefix, ":")
+		numParts := len(ipParts)
+		if limits[numParts] < count {
+			return false
+		}
+	}
+	return true
+}

p2p/listener.go

@@ -0,0 +1,218 @@
+package p2p
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"time"
+
+	"github.com/tendermint/tendermint/p2p/upnp"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+)
+
+type Listener interface {
+	Connections() <-chan net.Conn
+	InternalAddress() *NetAddress
+	ExternalAddress() *NetAddress
+	String() string
+	Stop() bool
+}
+
+// Implements Listener
+type DefaultListener struct {
+	cmn.BaseService
+
+	listener    net.Listener
+	intAddr     *NetAddress
+	extAddr     *NetAddress
+	connections chan net.Conn
+}
+
+const (
+	numBufferedConnections = 10
+	defaultExternalPort    = 8770
+	tryListenSeconds       = 5
+)
+
+func splitHostPort(addr string) (host string, port int) {
+	host, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		cmn.PanicSanity(err)
+	}
+	port, err = strconv.Atoi(portStr)
+	if err != nil {
+		cmn.PanicSanity(err)
+	}
+	return host, port
+}
+
+// skipUPNP: If true, does not try getUPNPExternalAddress()
+func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log.Logger) Listener {
+	// Local listen IP & port
+	lAddrIP, lAddrPort := splitHostPort(lAddr)
+
+	// Create listener
+	var listener net.Listener
+	var err error
+	for i := 0; i < tryListenSeconds; i++ {
+		listener, err = net.Listen(protocol, lAddr)
+		if err == nil {
+			break
+		} else if i < tryListenSeconds-1 {
+			time.Sleep(time.Second * 1)
+		}
+	}
+	if err != nil {
+		cmn.PanicCrisis(err)
+	}
+	// Actual listener local IP & port
+	listenerIP, listenerPort := splitHostPort(listener.Addr().String())
+	logger.Info("Local listener", "ip", listenerIP, "port", listenerPort)
+
+	// Determine internal address...
+	var intAddr *NetAddress
+	intAddr, err = NewNetAddressString(lAddr)
+	if err != nil {
+		cmn.PanicCrisis(err)
+	}
+
+	// Determine external address...
+	var extAddr *NetAddress
+	if !skipUPNP {
+		// If the lAddrIP is INADDR_ANY, try UPnP
+		if lAddrIP == "" || lAddrIP == "0.0.0.0" {
+			extAddr = getUPNPExternalAddress(lAddrPort, listenerPort, logger)
+		}
+	}
+	// Otherwise just use the local address...
+	if extAddr == nil {
+		extAddr = getNaiveExternalAddress(listenerPort)
+	}
+	if extAddr == nil {
+		cmn.PanicCrisis("Could not determine external address!")
+	}
+
+	dl := &DefaultListener{
+		listener:    listener,
+		intAddr:     intAddr,
+		extAddr:     extAddr,
+		connections: make(chan net.Conn, numBufferedConnections),
+	}
+	dl.BaseService = *cmn.NewBaseService(logger, "DefaultListener", dl)
+	dl.Start() // Started upon construction
+	return dl
+}
+
+func (l *DefaultListener) OnStart() error {
+	l.BaseService.OnStart()
+	go l.listenRoutine()
+	return nil
+}
+
+func (l *DefaultListener) OnStop() {
+	l.BaseService.OnStop()
+	l.listener.Close()
+}
+
+// Accept connections and pass on the channel
+func (l *DefaultListener) listenRoutine() {
+	for {
+		conn, err := l.listener.Accept()
+
+		if !l.IsRunning() {
+			break // Go to cleanup
+		}
+
+		// listener wasn't stopped,
+		// yet we encountered an error.
+		if err != nil {
+			cmn.PanicCrisis(err)
+		}
+
+		l.connections <- conn
+	}
+
+	// Cleanup
+	close(l.connections)
+	for range l.connections {
+		// Drain
+	}
+}
+
+// A channel of inbound connections.
+// It gets closed when the listener closes.
+func (l *DefaultListener) Connections() <-chan net.Conn {
+	return l.connections
+}
+
+func (l *DefaultListener) InternalAddress() *NetAddress {
+	return l.intAddr
+}
+
+func (l *DefaultListener) ExternalAddress() *NetAddress {
+	return l.extAddr
+}
+
+// NOTE: The returned listener is already Accept()'ing.
+// So it's not suitable to pass into http.Serve().
+func (l *DefaultListener) NetListener() net.Listener {
+	return l.listener
+}
+
+func (l *DefaultListener) String() string {
+	return fmt.Sprintf("Listener(@%v)", l.extAddr)
+}
+
+/* external address helpers */
+
+// UPNP external address discovery & port mapping
+func getUPNPExternalAddress(externalPort, internalPort int, logger log.Logger) *NetAddress {
+	logger.Info("Getting UPNP external address")
+	nat, err := upnp.Discover()
+	if err != nil {
+		logger.Info("Could not perform UPNP discover", "err", err)
+		return nil
+	}
+
+	ext, err := nat.GetExternalAddress()
+	if err != nil {
+		logger.Info("Could not get UPNP external address", "err", err)
+		return nil
+	}
+
+	// UPnP can't seem to get the external port, so let's just be explicit.
+	if externalPort == 0 {
+		externalPort = defaultExternalPort
+	}
+
+	externalPort, err = nat.AddPortMapping("tcp", externalPort, internalPort, "tendermint", 0)
+	if err != nil {
+		logger.Info("Could not add UPNP port mapping", "err", err)
+		return nil
+	}
+
+	logger.Info("Got UPNP external address", "address", ext)
+	return NewNetAddressIPPort(ext, uint16(externalPort))
+}
+
+// TODO: use syscalls: http://pastebin.com/9exZG4rh
+func getNaiveExternalAddress(port int) *NetAddress {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		cmn.PanicCrisis(cmn.Fmt("Could not fetch interface addresses: %v", err))
+	}
+
+	for _, a := range addrs {
+		ipnet, ok := a.(*net.IPNet)
+		if !ok {
+			continue
+		}
+		v4 := ipnet.IP.To4()
+		if v4 == nil || v4[0] == 127 {
+			continue
+		} // loopback
+		return NewNetAddressIPPort(ipnet.IP, uint16(port))
+	}
+	return nil
+}

p2p/listener_test.go

@@ -0,0 +1,42 @@
+package p2p
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tendermint/tmlibs/log"
+)
+
+func TestListener(t *testing.T) {
+	// Create a listener
+	l := NewDefaultListener("tcp", ":8001", true, log.TestingLogger())
+
+	// Dial the listener
+	lAddr := l.ExternalAddress()
+	connOut, err := lAddr.Dial()
+	if err != nil {
+		t.Fatalf("Could not connect to listener address %v", lAddr)
+	} else {
+		t.Logf("Created a connection to listener address %v", lAddr)
+	}
+	connIn, ok := <-l.Connections()
+	if !ok {
+		t.Fatalf("Could not get inbound connection from listener")
+	}
+
+	msg := []byte("hi!")
+	go connIn.Write(msg)
+	b := make([]byte, 32)
+	n, err := connOut.Read(b)
+	if err != nil {
+		t.Fatalf("Error reading off connection: %v", err)
+	}
+
+	b = b[:n]
+	if !bytes.Equal(msg, b) {
+		t.Fatalf("Got %s, expected %s", b, msg)
+	}
+
+	// Close the server, no longer needed.
+	l.Stop()
+}

p2p/netaddress.go

@@ -0,0 +1,252 @@
+// Modified for Tendermint
+// Originally Copyright (c) 2013-2014 Conformal Systems LLC.
+// https://github.com/conformal/btcd/blob/master/LICENSE
+
+package p2p
+
+import (
+	"errors"
+	"flag"
+	"net"
+	"strconv"
+	"time"
+
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+// NetAddress defines information about a peer on the network
+// including its IP address, and port.
+type NetAddress struct {
+	IP   net.IP
+	Port uint16
+	str  string
+}
+
+// NewNetAddress returns a new NetAddress using the provided TCP
+// address. When testing, other net.Addr (except TCP) will result in
+// using 0.0.0.0:0. When normal run, other net.Addr (except TCP) will
+// panic.
+// TODO: socks proxies?
+func NewNetAddress(addr net.Addr) *NetAddress {
+	tcpAddr, ok := addr.(*net.TCPAddr)
+	if !ok {
+		if flag.Lookup("test.v") == nil { // normal run
+			cmn.PanicSanity(cmn.Fmt("Only TCPAddrs are supported. Got: %v", addr))
+		} else { // in testing
+			return NewNetAddressIPPort(net.IP("0.0.0.0"), 0)
+		}
+	}
+	ip := tcpAddr.IP
+	port := uint16(tcpAddr.Port)
+	return NewNetAddressIPPort(ip, port)
+}
+
+// NewNetAddressString returns a new NetAddress using the provided
+// address in the form of "IP:Port". Also resolves the host if host
+// is not an IP.
+func NewNetAddressString(addr string) (*NetAddress, error) {
+
+	host, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	ip := net.ParseIP(host)
+	if ip == nil {
+		if len(host) > 0 {
+			ips, err := net.LookupIP(host)
+			if err != nil {
+				return nil, err
+			}
+			ip = ips[0]
+		}
+	}
+
+	port, err := strconv.ParseUint(portStr, 10, 16)
+	if err != nil {
+		return nil, err
+	}
+
+	na := NewNetAddressIPPort(ip, uint16(port))
+	return na, nil
+}
+
+// NewNetAddressStrings returns an array of NetAddress'es build using
+// the provided strings.
+func NewNetAddressStrings(addrs []string) ([]*NetAddress, error) {
+	netAddrs := make([]*NetAddress, len(addrs))
+	for i, addr := range addrs {
+		netAddr, err := NewNetAddressString(addr)
+		if err != nil {
+			return nil, errors.New(cmn.Fmt("Error in address %s: %v", addr, err))
+		}
+		netAddrs[i] = netAddr
+	}
+	return netAddrs, nil
+}
+
+// NewNetAddressIPPort returns a new NetAddress using the provided IP
+// and port number.
+func NewNetAddressIPPort(ip net.IP, port uint16) *NetAddress {
+	na := &NetAddress{
+		IP:   ip,
+		Port: port,
+		str: net.JoinHostPort(
+			ip.String(),
+			strconv.FormatUint(uint64(port), 10),
+		),
+	}
+	return na
+}
+
+// Equals reports whether na and other are the same addresses.
+func (na *NetAddress) Equals(other interface{}) bool {
+	if o, ok := other.(*NetAddress); ok {
+		return na.String() == o.String()
+	}
+
+	return false
+}
+
+func (na *NetAddress) Less(other interface{}) bool {
+	if o, ok := other.(*NetAddress); ok {
+		return na.String() < o.String()
+	}
+
+	cmn.PanicSanity("Cannot compare unequal types")
+	return false
+}
+
+// String representation.
+func (na *NetAddress) String() string {
+	if na.str == "" {
+		na.str = net.JoinHostPort(
+			na.IP.String(),
+			strconv.FormatUint(uint64(na.Port), 10),
+		)
+	}
+	return na.str
+}
+
+// Dial calls net.Dial on the address.
+func (na *NetAddress) Dial() (net.Conn, error) {
+	conn, err := net.Dial("tcp", na.String())
+	if err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+// DialTimeout calls net.DialTimeout on the address.
+func (na *NetAddress) DialTimeout(timeout time.Duration) (net.Conn, error) {
+	conn, err := net.DialTimeout("tcp", na.String(), timeout)
+	if err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+// Routable returns true if the address is routable.
+func (na *NetAddress) Routable() bool {
+	// TODO(oga) bitcoind doesn't include RFC3849 here, but should we?
+	return na.Valid() && !(na.RFC1918() || na.RFC3927() || na.RFC4862() ||
+		na.RFC4193() || na.RFC4843() || na.Local())
+}
+
+// For IPv4 these are either a 0 or all bits set address. For IPv6 a zero
+// address or one that matches the RFC3849 documentation address format.
+func (na *NetAddress) Valid() bool {
+	return na.IP != nil && !(na.IP.IsUnspecified() || na.RFC3849() ||
+		na.IP.Equal(net.IPv4bcast))
+}
+
+// Local returns true if it is a local address.
+func (na *NetAddress) Local() bool {
+	return na.IP.IsLoopback() || zero4.Contains(na.IP)
+}
+
+// ReachabilityTo checks whenever o can be reached from na.
+func (na *NetAddress) ReachabilityTo(o *NetAddress) int {
+	const (
+		Unreachable = 0
+		Default     = iota
+		Teredo
+		Ipv6_weak
+		Ipv4
+		Ipv6_strong
+	)
+	if !na.Routable() {
+		return Unreachable
+	} else if na.RFC4380() {
+		if !o.Routable() {
+			return Default
+		} else if o.RFC4380() {
+			return Teredo
+		} else if o.IP.To4() != nil {
+			return Ipv4
+		} else { // ipv6
+			return Ipv6_weak
+		}
+	} else if na.IP.To4() != nil {
+		if o.Routable() && o.IP.To4() != nil {
+			return Ipv4
+		}
+		return Default
+	} else /* ipv6 */ {
+		var tunnelled bool
+		// Is our v6 is tunnelled?
+		if o.RFC3964() || o.RFC6052() || o.RFC6145() {
+			tunnelled = true
+		}
+		if !o.Routable() {
+			return Default
+		} else if o.RFC4380() {
+			return Teredo
+		} else if o.IP.To4() != nil {
+			return Ipv4
+		} else if tunnelled {
+			// only prioritise ipv6 if we aren't tunnelling it.
+			return Ipv6_weak
+		}
+		return Ipv6_strong
+	}
+}
+
+// RFC1918: IPv4 Private networks (10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12)
+// RFC3849: IPv6 Documentation address  (2001:0DB8::/32)
+// RFC3927: IPv4 Autoconfig (169.254.0.0/16)
+// RFC3964: IPv6 6to4 (2002::/16)
+// RFC4193: IPv6 unique local (FC00::/7)
+// RFC4380: IPv6 Teredo tunneling (2001::/32)
+// RFC4843: IPv6 ORCHID: (2001:10::/28)
+// RFC4862: IPv6 Autoconfig (FE80::/64)
+// RFC6052: IPv6 well known prefix (64:FF9B::/96)
+// RFC6145: IPv6 IPv4 translated address ::FFFF:0:0:0/96
+var rfc1918_10 = net.IPNet{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(8, 32)}
+var rfc1918_192 = net.IPNet{IP: net.ParseIP("192.168.0.0"), Mask: net.CIDRMask(16, 32)}
+var rfc1918_172 = net.IPNet{IP: net.ParseIP("172.16.0.0"), Mask: net.CIDRMask(12, 32)}
+var rfc3849 = net.IPNet{IP: net.ParseIP("2001:0DB8::"), Mask: net.CIDRMask(32, 128)}
+var rfc3927 = net.IPNet{IP: net.ParseIP("169.254.0.0"), Mask: net.CIDRMask(16, 32)}
+var rfc3964 = net.IPNet{IP: net.ParseIP("2002::"), Mask: net.CIDRMask(16, 128)}
+var rfc4193 = net.IPNet{IP: net.ParseIP("FC00::"), Mask: net.CIDRMask(7, 128)}
+var rfc4380 = net.IPNet{IP: net.ParseIP("2001::"), Mask: net.CIDRMask(32, 128)}
+var rfc4843 = net.IPNet{IP: net.ParseIP("2001:10::"), Mask: net.CIDRMask(28, 128)}
+var rfc4862 = net.IPNet{IP: net.ParseIP("FE80::"), Mask: net.CIDRMask(64, 128)}
+var rfc6052 = net.IPNet{IP: net.ParseIP("64:FF9B::"), Mask: net.CIDRMask(96, 128)}
+var rfc6145 = net.IPNet{IP: net.ParseIP("::FFFF:0:0:0"), Mask: net.CIDRMask(96, 128)}
+var zero4 = net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: net.CIDRMask(8, 32)}
+
+func (na *NetAddress) RFC1918() bool {
+	return rfc1918_10.Contains(na.IP) ||
+		rfc1918_192.Contains(na.IP) ||
+		rfc1918_172.Contains(na.IP)
+}
+func (na *NetAddress) RFC3849() bool { return rfc3849.Contains(na.IP) }
+func (na *NetAddress) RFC3927() bool { return rfc3927.Contains(na.IP) }
+func (na *NetAddress) RFC3964() bool { return rfc3964.Contains(na.IP) }
+func (na *NetAddress) RFC4193() bool { return rfc4193.Contains(na.IP) }
+func (na *NetAddress) RFC4380() bool { return rfc4380.Contains(na.IP) }
+func (na *NetAddress) RFC4843() bool { return rfc4843.Contains(na.IP) }
+func (na *NetAddress) RFC4862() bool { return rfc4862.Contains(na.IP) }
+func (na *NetAddress) RFC6052() bool { return rfc6052.Contains(na.IP) }
+func (na *NetAddress) RFC6145() bool { return rfc6145.Contains(na.IP) }